V4.2022.02.2900 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
-Abonti
-Aboundexbot
-AiHitBot
+01h4x.com
+360Spider
+404checker
+80legs
+AIBOT
+ASPSeek
+Aboundex
 Aipbot
 Alexibot
-Alligator
 AlphaBot
-Anthill
-AwarioRssBot
-AwarioSmartBot
-BDFetch
+Anarchie
+Anarchy99
+Asterias
+Attach
+BBBike
+BDCbot
 BackWeb
 BacklinkCrawler
-Bigfoot
-Black Hole
+Bitacle
 Blackboard
 BlowFish
-Boardreader
-Brandwatch
-BuzzSumo
+Bolt
+BotALot
+Brandprotect
+Buddy
+Bullseye
+CCBot
 CODE87
-CheTeam
-CheeseBot
-CherryPicker
-Chlooe
+CSHttp
+CensysInspect
+ChinaClaw
+Claritybot
+Cloud mapping
 Cocolyzebot
-Collector
-Copier
-Copyscape
-Cosmos
-Crescent
+Cogentbot
+Crawling at Home Project
 CrunchBot
-Curious
-Custo
-CyotekWebCopy
-DBLBot
-DIIbot
-Demon
+DSearch
+DataCha0s
 Devil
 DigitalPebble
-Dirbuster
-Disco
-DittoSpyder
+Discoverybot
+DnBCrawler-Analytics
+DomCopBot
+DomainAppender
 DomainCrawler
+DomainSigmaCrawler
+DomainStatsBot
+Domains Project
 Dotbot
 Download Wonder
 Dragonfly
-Drip
-EMail Wolf
+ECCP/1.0
 EasyDL
-Ebingbong
-Ecxi
 EroCrawler
+Evil
 Exabot
+Express WebPictures
 ExtLinksBot
-ExtractorPro
+Extractor
 Extreme Picture Finder
-EyeNetIE
-FemtosearchBot
-Firefox/7.0
-FlashGet
-Fuzz
+FHscan
+Foobot
+Freeuploader
 FyberSpider
-Fyrebot
 GT::WWW
 GalaxyBot
+GermCrawler
 GetRight
 GetWeb
 Gigabot
-Go!Zilla
-Gotit
-GrapeFX
-GrapeshotCrawler
-HEADMasterSEO
-HMView
-HaosouSpider
-Harvest
+Go-Ahead-Got-It
+GridBot
+HTTP::Lite
+HTTrack
 Havij
 HonoluluBot
-HybridBot
-IDBTE4M
-IlseBot
+Id-search
+Image Fetch
+Image Sucker
+IndeedBot
+Indy Library
+InfoNaviRobot
 Intelliseek
-InterGET
-Internet Ninja
-Iria
 IstellaBot
-JOC Web Spider
-Jbrofuzz
 JetCar
-JikeSpider
-Jorgee
-Keyword Density
-Kinza
+JustView
 Kozmosbot
-LexiBot
-Libwhisker
-LieBaoFast
-LinkScan
-Linkdexbot
-LinkextractorPro
-LinqiaRSSBot
-Lipperhey
+LWP::Simple
+Lanshanbot
+Larbin
+Leap
+LeechFTP
+Lftp
+LibWeb
+Likse
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
 MIDown tool
+MIIxpc
 MJ12bot
-MSFrontPage
 MSIECrawler
 MTRobot
 Magnet
-MarkMonitor
+Mail.RU_Bot
+Majestic12
 MarkWatch
-Masscan
-Mb2345Browser
 MeanPath Bot
 Meanpathbot
-Mediatoolkitbot
 MegaIndex.ru
 Metauri
-Microsoft Data Access
-Minefield
+Microsoft URL Control
 Moblie Safari
 Mojeek
+Mojolicious
 MolokaiBot
 Morfeus Fucking Scanner
-Msrabot
-Musobot
-Nameprotect
-NearSite
+Mozlila
+NPbot
+Name Intelligence
+Navroad
 Needle
-Net Vampire
-NetAnts
-NetMechanic
-Nibbler
-Niki-bot
-Nikto
-NimbleCrawler
-Nmap
-Not
+Nessus
+NetZIP
+Netcraft
+Nimbostratus
 Nutch
 OnCrawl
 OpenLinkProfiler
-Openfind
-PHPCrawl
+OutclicksBot
 POE-Component-Client-HTTP
-PageThing.com
-Panscient
-Pavuk
-Petalbot
-Pi-Monster
-Picscout
+Page Analyzer
+PageAnalyzer
+PageGrabber
+PageScorer
+Papa Foto
+PeoplePal
 Picsearch
-Pu_iN
-PxBroker
-RankActive
+Piepmatz
+Pimonster
+Pixray
+Pockey
+ProPowerBot
+Psbot
+Pump
+Quick-Crawler
+RankFlex
 Rankivabot
-RankurBot
-RealDownload
+Re-re
 Reaper
 Recorder
-RepoMonkey
 Rogerbot
-SBIder
-SEOkicks
-SEOkicks-Robot
+SEOlyticsCrawler
+SISTRIX
+SMTBot
 SalesIntelligent
-ScoutJet
-Screaming
-ScreenerBot
-ScrepyBot
-Searchestate
-SearchmetricsBot
+ScanAlert
+Semrush
 SemrushBot
-SentiBot
 SeoSiteCheckup
 SeobilityBot
-SiteExplorer
+Seomoz
 SiteLockSpider
 SiteSnagger
-SiteSucker
+Sitebeam
 Siteimprove
 SmartDownload
 Snake
 Snapbot
-Snoopy
-Sottopop
-SpankBot
-Spanner
-Spinn3r
-Sucuri
-SuperBot
+SocialRankIOBot
+Sociscraper
+Sogou web spider
+SpaceBison
+Spbot
+Sqlworm
+Sqworm
 SuperHTTP
 Surfbot
-Suzuran
-Swiftbot
 Teleport
-The Intraformant
+Thumbor
 TightTwatBot
-Titan
-Tracemyfile
-TwengaBot
-Twice
+Toata
+Toweyabot
+Trendiction
+True_Robot
+Turingos
 Typhoeus
-V-BOT
-VB Project
-Vacuum
-WPScan
-WWW-Collector-E
-WWW-Mechanize
-Web Fetch
-Web Sucker
-WebCollage
-WebFetch
+URLy Warning
+VelenPublicWebCrawler
+VoidEYE
+Voltron
+WASALive-Bot
+WEBDAV
+Web Fuck
+Web Sauger
+WebAuto
+WebBandit
+WebEnhancer
 WebGo IS
 WebLeacher
+WebPix
+WebStripper
 WebSucker
 WebZIP
-WebsiteQuester
+Webalta
+WebmasterWorldForumBot
+Website Quester
+WebsiteExtractor
 Whack
+Whacker
+Whatweb
 Who.is Bot
-Woobot
+Wonderbot
+Wprecon
 Xaldon WebSpider
 YoudaoBot
 Zade
-ZoominfoBot
-ZyBorg
+Zauba
+Zermelo
+Zeus
 adscanner
-autoemailspider
+archive.org_bot
+backlink-check
+clark-crawler
 cognitiveseo
+com.plumanalytics
+crawler.feedback
 crawler4j
-dataforseo.com
-demandbase-bot
-heritrix
+domainsproject.org
+facebookscraper
 instabid
-isitwp.com
+ips-agent
 lwp-request
+mediawords
+muhstik-scan
 netEstate NE Crawler
-pcBrowser
-polaris version
+page scorer
+plumanalytics
 probe-image-size
 ripz
-scalaj-http
-sexsearcher
-sitechecker.pro
+seoscanners
+sogouspider
 spyfu
-trendiction.com
-webprosbot
-x22Mozilla
+trendiction.de
+ubermetrics-technologies.com
+webgains-bot
+webmeup-crawler
+webpros.com
+x09Mozilla
 xpymep1.exe
-zauba.io