V4.2021.12.2768 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-ADmantX
-ALittle Client
-Aboundex
-Aboundexbot
+01h4x.com
+360Spider
+404enemy
+80legs
+ASPSeek
 AiHitBot
-AllSubmitter
-AlphaBot
-Anarchy
-Anarchy99
-Apexoo
-Aspiegel
-Attach
-BBBike
-BDFetch
-BLEXBot
-BackDoorBot
-BackStreet
-BackWeb
-Backlink-Ceck
-BacklinkCrawler
+Aipbot
+Alexibot
+Anthill
+BDCbot
 Bandit
 Barkrowler
-Bitacle
+Black Hole
 Blackboard
+Bolt
+BotALot
+Brandprotect
 Brandwatch
-Buddy
-BuiltBotTough
 BuiltWith
-BunnySlippers
+Bullseye
+BuzzSumo
 CATExplorador
+CCBot
 CODE87
-Calculon
+CazoodleBot
 CensysInspect
-CheTeam
-CheeseBot
+CherryPicker
 ChinaClaw
+Chlooe
 Claritybot
-CopyRightCheck
+Cliqzbot
+Collector
+Cosmos
+Craftbot
 Crawling at Home Project
+CrazyWebCrawler
 CrunchBot
-CyotekWebCopy
-Demon
+Custo
+Deusu
 Devil
-Dirbuster
-Discobot
-DnBCrawler-Analytics
-DnyzBot
-DomainSigmaCrawler
+Digincore
+Discoverybot
+DomCopBot
 Domains Project
-Drip
-ECCP/1.0
+Download Wonder
 EMail Wolf
-Ebingbong
+EasyDL
 EirGrabber
-EroCrawler
-Exabot
-Extreme Picture Finder
-EyeNetIE
-FHscan
-Firefox/7.0
+Evil
+ExtLinksBot
+Extractor
+ExtractorPro
+Ezooms
+FemtosearchBot
 Flunky
-Fyrebot
+Foobot
+Fuzz
 G-i-g-a-b-o-t
+GT::WWW
+GalaxyBot
+Genieo
+GetRight
+GetWeb
 Gigabot
-Go-Ahead-Got-It
-Grabber
-GrapeFX
-GrapeshotCrawler
-GridBot
-HEADMasterSEO
+Go!Zilla
+GrabNet
+Grafula
 HMView
-HTTP::Lite
+HTMLparser
 HTTrack
-Haansoft
-HaosouSpider
-Hloader
+Heritrix
 HybridBot
 IDBot
-Iblog
-Image Fetch
-Indy Library
+IlseBot
+Image Sucker
 InfoNaviRobot
-InfoTekies
 Intelliseek
 InterGET
 Internet Ninja
-Iria
-JOC Web Spider
+JamesBOT
 Jbrofuzz
-JennyBot
+Jetty
 JikeSpider
 JustView
-Jyxobot
-Kinza
+Keyword Density
+Kozmosbot
 LNSpiderguy
-Lanshanbot
-LeechFTP
+LWP::Simple
+Larbin
+LibWeb
+Libwhisker
 LieBaoFast
-LinkScan
+Lightspeedsystems
 Linkbot
-LinksManager
+Linkdexbot
+LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
 LinqiaRSSBot
 LinqiaScrapeBot
+Lipperhey
+Lipperhey Spider
+Litemage_walker
 Ltx71
 MIDown tool
-MJ12bot
-Magnet
-Majestic12
-MarkWatch
-Mb2345Browser
-Microsoft Data Access
-Mojolicious
-Mr.4x3
-Msrabot
+MQQBrowser
+MTRobot
+Mail.RU_Bot
+Majestic-SEO
+Masscan
+MeanPath Bot
+Meanpathbot
+Metauri
+Mojeek
+Morfeus Fucking Scanner
 Name Intelligence
-Nameprotect
-Navroad
 NearSite
 Needle
-Nettrack
+Net Vampire
+NetAnts
+NetZIP
+Netcraft
+Netvibes
+NextGenSearchBot
 Nibbler
 NimbleCrawler
-Ninja
-Octopus
+Nmap
+Not
+Nuclei
+Nutch
+Offline Navigator
+OnCrawl
+OpenLinkProfiler
 OpenVAS
-Openvas
 OrangeSpider
-PHPCrawl
-POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
-PageThing.com
-Panscient
+PageGrabber
+PageScorer
+Pavuk
 Pi-Monster
-PictureFinder
-Piepmatz
-Pimonster
+Picscout
 Pixray
 Pockey
-Psbot
 Pu_iN
-PyCurl
-Quick-Crawler
+PxBroker
+QueryN Metasearch
+RSSingBot
+RankActive
+RankFlex
 RankingBot
-RankurBot
-ReGet
+RealDownload
+Recorder
 RepoMonkey
-RocketCrawler
-SBIder
+SEOkicks
 SEOkicks-Robot
-SEOprofiler
-SEOstats
+SEOlyticsCrawler
+SalesIntelligent
 ScrepyBot
-Searchestate
-SearchmetricsBot
 SemanticJuice
+SemrushBot
 SeoSiteCheckup
-SeobilityBot
+Seomoz
 Shodan
-SiteLockSpider
-Snoopy
-Sociscraper
-Sogou web spider
-Spanner
-Spbot
-Spinn3r
-Sqlmap
+Site Sucker
+Snake
+SpankBot
+SputnikBot
 Sqworm
 Steeler
 Stripper
 Sucker
 Sucuri
 SuperBot
+SuperHTTP
 Surfbot
-Suzuran
-T8Abot
-TeleportPro
+SurveyBot
+Teleport
+Telesoft
 Telesphoreo
 Telesphorep
 The Intraformant
-Thumbor
+TheNomad
 TightTwatBot
-Toweyabot
-Tracemyfile
-Turnitin
-TurnitinBot
-Twice
-Upflow
-Vacuum
-Vagabondo
-VelenPublicWebCrawler
+Titan
+Toata
+Trendictionbot
+Turingos
+TwengaBot
+Typhoeus
+URLy.Warning
 VeriCiteCrawler
 VidibleScraper
-Virusdie
-Voil
 Voltron
-WWWOFFLE
+WASALive-Bot
+WISENutbot
+WWW-Mechanize
+Wallpapers/3.0
 WallpapersHD
+WeSEE
 Web Auto
-Web Fuck
+Web Enhancer
+Web Pix
 Web Sauger
+WebCollage
+WebFetch
+WebFuck
 WebLeacher
-WebPix
-WebStripper
+WebReaper
 WebSucker
-WebWhacker
 WebZIP
-Webalta
-Website Quester
-WebsiteExtractor
-WebsiteQuester
-Webster
-Widow
-Wonderbot
-Xaldon WebSpider
+Webshag
+Whacker
+Who.is Bot
+WiseGuys Robot
+Wprecon
+Xaldon_WebSpider
+Xenu
+Zade
+Zauba
 Zermelo
+Zeus
 Zitebot
-ZyBorg
+ZmEu
+ZumBot
 archive.org_bot
 autoemailspider
-check1.exe
-clark-crawler
-cognitiveseo
-com.plumanalytics
-crawler4j
+backlink-check
+crawl.sogou.com
 dataforseo.com
-demandbase-bot
-domainsproject.org
-evc-batch
-heritrix
-ips-agent
-isitwp.com
-lwp-request
-lwp-trivial
-meanpathbot
+eCatch
+instabid
+magpie-crawler
 mediawords
-muhstik-scan
 netEstate NE Crawler
+plumanalytics
+polaris version
 probe-image-size
+ripz
 seobility
-seocompany.store
 serpstatbot
 siteripz
-sogouspider
 sp_auditbot
+tAkeOut
 trendiction.de
+ubermetrics-technologies.com
 voyagerx.com
-webgains-bot
-zauba.io
+webpros.com
+webprosbot
+xpymep1.exe
+zgrab