V4.2024.02.4292 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
-ALittle Client
-AdsTxtCrawlerTP
-Alligator
-Asterias
-Attach
-AwarioRssBot
-BBBike
+ASPSeek
+Aboundexbot
+Acunetix
+AhrefsBot
+Aipbot
+Anarchy
+Anarchy99
+Aspiegel
+Atomseobot
+BackDoorBot
 BackWeb
-Black Hole
-Buck
-BuiltWith
-CCBot
-CensysInspect
-Collector
-Copier
+Boardreader
+CheTeam
+ChinaClaw
+Cogentbot
 Copyscape
-CrunchBot
-Custo
-DittoSpyder
-Dragonfly
+Cosmos
+DBLBot
+Demon
+EMail Siphon
 Ecxi
 Exabot
-FDM
-FacebookBot
-FlashGet
-Freeuploader
+Extractor
+GT::WWW
 Google-Extended
-GrabNet
-Image Fetch
-Iria
-Jbrofuzz
-LNSpiderguy
-Larbin
-LeechFTP
-MIDown tool
-MSFrontPage
-MSIECrawler
-MTRobot
-Mb2345Browser
-Mojeek
-Name Intelligence
+Gotit
+HaosouSpider
+Havij
+Heritrix
+IlseBot
+IndeedBot
+Iskanie
+JikeSpider
+Kinza
+LeechGet
+LinkextractorPro
+Lmspider
+Majestic-SEO
+Majestic12
+MolokaiBot
+Musobot
+NetAnts
 Netcraft
-Nutch
+Ninja
 Offline Explorer
-OrangeBot
-Picscout
-PictureFinder
-Pixray
-Proximic
-Rainbot
+Offline Navigator
+OnCrawl
+OpenLinkProfiler
+OrangeSpider
+Page Analyzer
+Pandalytics
+PxBroker
+RSSingBot
+RankActive
+RankActiveLinkBot
+RankingBot2
 Re-re
+Reaper
 RebelMouse
-RocketCrawler
-SEOlyticsCrawler
-SalesIntelligent
-Scrapy
-Screaming
-SenutoBot
-Site Sucker
-SiteExplorer
-SiteSucker
+Rogerbot
+SBIder
+SEOstats
+SMTBot
+Scanbot
+Searchestate
+SeekportBot
+SeoSiteCheckup
+Seomoz
+Shodan
+SiteSnagger
+SmartDownload
+Snapbot
+SocialRankIOBot
+Sociscraper
 SpankBot
-Sqworm
+Spinn3r
 Stripper
-SuperBot
-Swiftbot
-TeleportPro
 TightTwatBot
-Tracemyfile
-Trendiction
 Twice
-URLy.Warning
-VidibleScraper
+V-BOT
+Vagabondo
+VeriCiteCrawler
 VoidEYE
-Voil
-Voltron
-WPScan
-WWW-Collector-E
-WWWOFFLE
-Wallpapers
-Web Fetch
-Web Fuck
-WebGo IS
+WWW-Mechanize
+WWW::Mechanize
+Web Auto
+WebImageCollector
+WebSucker
+WebZIP
 WebsiteQuester
-WiseGuys Robot
-Xenu
+Zeus
+ZoomBot
 anthropic-ai
-clark-crawler
-cohere-ai
-demandbase-bot
-isitwp.com
-mediawords
-seobility
-seocompany.store
+dataforseo.com
+eCatch
+meanpathbot
+seoscanners
 seostar
-siteripz
-webpros.com
-x22Mozilla
-zauba.io
+x09Mozilla
+zgrab