V4.2021.09.2617 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
+01h4x.com
+404enemy
 80legs
 ALittle Client
-Aboundexbot
 Acunetix
 AfD-Verbotsverfahren
 AhrefsBot
-AllSubmitter
-Anarchy99
+Alexibot
+AlphaBot
+Anarchie
 Ankit
-Apexoo
+Asterias
 Attach
 AwarioSmartBot
-BBBike
-BDCbot
-BDFetch
+BackDoorBot
+Backlink-Ceck
 BacklinkCrawler
 Badass
 Bandit
-BetaBot
+Barkrowler
 Bigfoot
-Bitacle
-Black Hole
+Blackboard
+BlowFish
 Boardreader
-Bolt
-Brandwatch
 BuiltBotTough
+BuiltWith
 BunnySlippers
-BuzzSumo
-CATExplorador
-CCBot
-CODE87
-CazoodleBot
-CensysInspect
-CheTeam
+CSHttp
 ChinaClaw
-Claritybot
 Cliqzbot
-Cocolyzebot
 Cogentbot
-Collector
 CopyRightCheck
-Copyscape
-Crawling at Home Project
-CrazyWebCrawler
-CrunchBot
-DTS Agent
-Demon
-Deusu
-Digincore
-DigitalPebble
+CyotekWebCopy
+DIIbot
+Devil
+Dirbuster
+Disco
 Discobot
 Discoverybot
-DittoSpyder
-DnyzBot
-DomainAppender
-DomainCrawler
-DomainSigmaCrawler
-DomainStatsBot
-Domains Project
-Dotbot
-Dragonfly
+Dispatch
+DomCopBot
+Download Wonder
+Drip
+ECCP/1.0
 EMail Siphon
-EMail Wolf
-Ebingbong
+EirGrabber
 Exabot
-ExtractorPro
-EyeNetIE
+Express WebPictures
 Ezooms
+FDM
+FHscan
 FemtosearchBot
-Flunky
+FlashGet
 Foobot
 Freeuploader
-Fuzz
-GT::WWW
-Go-Ahead-Got-It
+FyberSpider
+Fyrebot
+G-i-g-a-b-o-t
+Genieo
+GermCrawler
+GetRight
+GetWeb
+Getintent
 GoZilla
+GrabNet
 Grafula
+GrapeFX
 GrapeshotCrawler
-GridBot
-Haansoft
+HMView
+HTTP::Lite
+HTTrack
 HaosouSpider
 Harvest
-Heritrix
 Humanlinks
 HybridBot
-IDBot
+IDBTE4M
+IRLbot
 Image Sucker
-Indy Library
-InterGET
-Internet Ninja
+IndeedBot
+InfoTekies
 Iria
-Iskanie
-JamesBOT
-JennyBot
-JetCar
+IstellaBot
+JOC Web Spider
+Jbrofuzz
+Joomla
 Jorgee
 Jyxobot
-LNSpiderguy
-Lanshanbot
+Keyword Density
+Kozmosbot
 Larbin
-LeechGet
+LeechFTP
+LexiBot
 Lftp
 LibWeb
-Lightspeedsystems
-Likse
-LinkWalker
-Linkbot
+LinkpadBot
+LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
-MIDown tool
-MJ12bot
+LinqiaScrapeBot
+Lipperhey
+Litemage_walker
+Ltx71
+MIIxpc
 MQQBrowser
 MSFrontPage
-Magnet
+MSIECrawler
+Mail.RU_Bot
+Majestic12
+MarkWatch
 Mata Hari
-MauiBot
 MeanPath Bot
+Mediatoolkitbot
 MegaIndex.ru
-Metauri
-Mozlila
+MicroMessenger
+Microsoft URL Control
+Moblie Safari
+Morfeus Fucking Scanner
 Msrabot
-NICErsPRO
-NetAnts
-NetSpider
-Nettrack
-Nibbler
+Nessus
+Netvibes
+NextGenSearchBot
+Niki-bot
 Nikto
-Nmap
+Nuclei
 Nutch
 Offline Explorer
 Offline Navigator
-OnCrawl
 OpenLinkProfiler
 OpenVAS
-OrangeSpider
-OutclicksBot
-PHPCrawl
-Page Analyzer
+Openvas
+OrangeBot
+OutfoxBot
+PECL::HTTP
+PageAnalyzer
+PageGrabber
 PageThing.com
-Papa Foto
-PeoplePal
-Pi-Monster
+Picscout
 Piepmatz
 Pimonster
-Pixray
 PleaseCrawl
 Pockey
-ProWebWalker
-Probethenet
-Pump
+Psbot
+PxBroker
+QueryN Metasearch
+RankingBot2
+Rankivabot
 RankurBot
+Re-re
 RealDownload
+Recorder
 RedesScrapy
-SEOkicks
 SEOkicks-Robot
-SEOstats
+SEOlyticsCrawler
+SMTBot
+SalesIntelligent
+ScanAlert
 ScoutJet
-Screaming
-Searchestate
 SearchmetricsBot
-Semrush
+SeoSiteCheckup
 SeobilityBot
-Shodan
+Seomoz
+Siphon
 SiteCheckerBotCrawler
 SiteExplorer
-SiteSnagger
 SiteSucker
+Sitebeam
+Siteimprove
+Sitevigil
 SmartDownload
-Snapbot
-Snoopy
-Sociscraper
-Sottopop
+SocialRankIOBot
+SpankBot
 Spanner
-Spinn3r
-Sqlmap
 Sqlworm
-Surfbot
-Suzuran
-T0PHackTeam
-T8Abot
+Steeler
+Stripper
+Szukacz
 Telesphoreo
+Telesphorep
 The Intraformant
+TheNomad
+Thumbor
+Titan
 Toata
-Trendiction
+Trendictionbot
 True_Robot
 Turingos
-TwengaBot
-Typhoeus
-URLy.Warning
-Upflow
+Turnitin
+TurnitinBot
 VB Project
-VCI
 Vacuum
 VelenPublicWebCrawler
-VidibleScraper
-VoidEYE
+VeriCiteCrawler
+Voil
 Voltron
-WASALive-Bot
-WBSearchBot
 WEBDAV
-WWW-Collector-E
-WWW::Mechanize
-WWWOFFLE
-Wallpapers/3.0
-WeSEE
-Web Sauger
-WebAuto
-WebCopier
+WISENutbot
+WWW-Mechanize
+Wallpapers
+Web Collage
+Web Fetch
+Web Fuck
+WebCollage
 WebFuck
-WebSauger
 WebSucker
-WebmasterWorldForumBot
+Webalta
+Webshag
 Website Quester
-Whatweb
+WebsiteExtractor
+Webster
 Who.is Bot
+Widow
 Wonderbot
-Wprecon
-Xenu
-Zeus
+Woobot
+Wotbox
+Xaldon WebSpider
+Xaldon_WebSpider
+Zade
+Zauba
+Zitebot
+ZmEu
 ZoomBot
-ZoominfoBot
-ZumBot
-archive.org_bot
 arquivo-web-crawler
+arquivo.pt
 cah.io.community
+check1.exe
 clark-crawler
+coccocbot-web
 cognitiveseo
-crawl.sogou.com
-crawler.feedback
-dataforseo.com
-demandbase-bot
-instabid
+com.plumanalytics
+crawler4j
+domainsproject.org
+eCatch
+evc-batch
+gopher
+internetVista monitor
 ips-agent
-mediawords
-oBot
-ripz
-s1z.ru
+isitwp.com
+lwp-trivial
+magpie-crawler
+meanpathbot
+muhstik-scan
+pcBrowser
+polaris version
+probe-image-size
 scalaj-http
-scan.lol
-seoscanners
 serpstatbot
-sexsearcher
 sitechecker.pro
-siteripz
-sogouspider
-sp_auditbot
-sysscan
-trendiction.de
-webmeup-crawler
+ubermetrics-technologies.com
+webprosbot
+x09Mozilla
+x22Mozilla