V4.2021.10.2661 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
-80legs
-AIBOT
-Aboundexbot
+ALittle Client
+ASPSeek
 Acunetix
-Alexibot
-AllSubmitter
-Alligator
-AlphaBot
+AiHitBot
+Anarchie
 Anarchy
-Anarchy99
 Anthill
 Aspiegel
-Asterias
-AwarioSmartBot
+BBBike
+BDFetch
+BackDoorBot
 BackWeb
-Badass
 Bandit
-BatchFTP
-BetaBot
+Barkrowler
+Battleztar Bazinga
+Bigfoot
+Bitacle
+Black Hole
+Blackboard
 Blow
-Brandwatch
+Brandprotect
 Buck
-BuiltWith
-Bullseye
 BunnySlippers
 CATExplorador
-CCBot
-CSHttp
-CensysInspect
-Claritybot
-Cloud mapping
+Cegbfeieh
+CheeseBot
+CherryPicker
+Cliqzbot
 Cocolyzebot
+Cogentbot
 Collector
-CopyRightCheck
+Copier
 Copyscape
 Cosmos
+Crawling at Home Project
 Crescent
-CrunchBot
-Custo
 DSearch
-DTS Agent
 DatabaseDriverMysqli
-DigitalPebble
+Demon
 Dirbuster
+Discobot
+Dispatch
+DnyzBot
 DomainAppender
-Dragonfly
+Dotbot
 Drip
 ECCP/1.0
-EMail Siphon
 EMail Wolf
-Ebingbong
-Ecxi
+EasyDL
 EroCrawler
 Evil
 Extractor
-Ezooms
-Fimap
+EyeNetIE
+FDM
+FemtosearchBot
 Firefox/7.0
+FlashGet
 Flunky
 Foobot
 Fuzz
-FyberSpider
-GT::WWW
-GalaxyBot
-Genieo
-GetRight
+Fyrebot
+GermCrawler
 GetWeb
-Go!Zilla
-Gotit
-GridBot
-HEADMasterSEO
+Getintent
+Gigabot
+GoZilla
+Grafula
+GrapeFX
 HMView
-HTTP::Lite
 Haansoft
-HaosouSpider
+Harvest
+Havij
+Hloader
+IDBot
+IRLbot
+Iblog
+Id-search
 IlseBot
 Image Fetch
-IndeedBot
-InfoNaviRobot
-InterGET
 Internet Ninja
 JOC Web Spider
 JamesBOT
+Jbrofuzz
 JennyBot
+JetCar
 Jetty
-Kozmosbot
+Joomla
+Kenjin Spider
 LWP::Simple
 Lanshanbot
 Larbin
-Leap
-Lftp
-Linkdexbot
-LinkextractorPro
+LeechFTP
+LexiBot
+LieBaoFast
+Likse
+LinkScan
+LinkWalker
 LinksManager
-LinqiaRSSBot
 LinqiaScrapeBot
-Litemage_walker
-Ltx71
+Lipperhey
+Lmspider
 MIDown tool
-MQQBrowser
 MSIECrawler
-MTRobot
-Mag-Net
-Mail.RU_Bot
+Magnet
+Majestic-SEO
 Masscan
-MauiBot
+Mata Hari
 MeanPath Bot
-Meanpathbot
 MegaIndex.ru
-Metauri
 Microsoft Data Access
 Mister PiX
+Morfeus Fucking Scanner
+Mr.4x3
 Msrabot
 Musobot
-NPbot
+Nameprotect
+Navroad
 NetAnts
 NetLyzer
-NetMechanic
-NetSpider
-NetZIP
-Netvibes
+NextGenSearchBot
+Nibbler
 Niki-bot
 Nikto
-NimbleCrawler
-Nmap
+Nimbostratus
+Ninja
 Nuclei
-Nutch
-Octopus
-Openfind
-OrangeSpider
+Offline Explorer
+OrangeBot
 PECL::HTTP
-POE-Component-Client-HTTP
 Page Analyzer
-PageGrabber
-PageScorer
+PageAnalyzer
 PageThing.com
 Pandalytics
 Papa Foto
-Petalbot
-Picscout
+Pi-Monster
+PictureFinder
+Pixray
+PleaseCrawl
 Pockey
-ProPowerBot
-ProWebWalker
-Psbot
-Pu_iN
-PxBroker
+Probethenet
 PyCurl
 RankActive
 RankActiveLinkBot
-RankingBot
-RankingBot2
 RankurBot
 Re-re
-RebelMouse
-RocketCrawler
+RealDownload
+Ripper
+Rogerbot
 SEOkicks-Robot
 SEOlyticsCrawler
-SEOprofiler
-SISTRIX
-SMTBot
-SalesIntelligent
+SEOstats
 Scanbot
-ScoutJet
-Scrapy
 ScrepyBot
-SearchmetricsBot
-Semrush
-SeobilityBot
-Site Sucker
+SemrushBot
+SentiBot
+Seomoz
+Shodan
 SiteCheckerBotCrawler
 SiteLockSpider
-Sitebeam
-SocialRankIOBot
+SiteSnagger
+SlySearch
+SmartDownload
+Snoopy
 Sociscraper
-Sottopop
 SpaceBison
+Spammen
 SpankBot
-Spinn3r
-Sqlworm
-Sucuri
-Suzuran
+SputnikBot
+Sqworm
+Stripper
+SuperBot
+Surfbot
+SurveyBot
 Szukacz
-T0PHackTeam
-Telesphoreo
-The Intraformant
-TheNomad
+Telesoft
+Thumbor
+Titan
 Trendiction
-True_Robot
-URLy.Warning
+Trendictionbot
+TwengaBot
+Twice
 UnisterBot
-Upflow
+V-BOT
 VB Project
 VCI
-Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
 Virusdie
-Voltron
+VoidEYE
+Voil
+WASALive-Bot
+WEBDAV
 WPScan
 WWW-Mechanize
-WWW::Mechanize
 Wallpapers/3.0
-Web Fetch
-Web Fuck
-WebAuto
-WebSucker
+WallpapersHD
+Web Auto
+Web Collage
+Web Enhancer
+Web Pix
+Web Sauger
+Web Sucker
+WebCopier
+WebEnhancer
+WebPix
+WebWhacker
 WebZIP
-Webster
+Webalta
+WebmasterWorldForumBot
+Webshag
 Whack
 Whacker
-Who.is Bot
 WinHTTrack
-WiseGuys Robot
 Wonderbot
 Xaldon WebSpider
-Zitebot
-ZoominfoBot
-ZumBot
+Xenu
+YoudaoBot
+Zade
+Zauba
+Zermelo
+Zeus
+ZmEu
+ZoomBot
+ZyBorg
+archive.org_bot
+arquivo-web-crawler
 arquivo.pt
 autoemailspider
-crawler.feedback
-domainsproject.org
-facebookscraper
-gopher
-internetVista monitor
+check1.exe
+coccocbot-web
+crawl.sogou.com
+eCatch
+instabid
 ips-agent
-isitwp.com
 lwp-trivial
-meanpathbot
+magpie-crawler
 mediawords
-muhstik-scan
-netEstate NE Crawler
-oBot
 page scorer
-pcBrowser
-plumanalytics
-probe-image-size
+polaris version
 s1z.ru
-seobility
+scalaj-http
+scan.lol
 seoscanners
-serpstatbot
+sexsearcher
+sitechecker.pro
 siteripz
-trendiction.com
+sogouspider
+sp_auditbot
+tAkeOut
 trendiction.de
-webmeup-crawler
 webpros.com
-x09Mozilla