V4.2024.02.4293 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-ASPSeek
-Aboundexbot
+ALittle Client
 Acunetix
-AhrefsBot
-Aipbot
-Anarchy
-Anarchy99
-Aspiegel
-Atomseobot
+Attach
 BackDoorBot
-BackWeb
-Boardreader
-CheTeam
-ChinaClaw
-Cogentbot
-Copyscape
-Cosmos
-DBLBot
-Demon
-EMail Siphon
-Ecxi
-Exabot
-Extractor
+BetaBot
+CODE87
+Cloud mapping
+Cocolyzebot
+Collector
+Crescent
+Curious
+DigitalPebble
+DittoSpyder
+ExtractorPro
+FDM
+Flunky
+Foobot
+Fyrebot
 GT::WWW
-Google-Extended
-Gotit
-HaosouSpider
-Havij
+Grafula
+GridBot
+HTMLparser
 Heritrix
-IlseBot
-IndeedBot
-Iskanie
-JikeSpider
-Kinza
-LeechGet
-LinkextractorPro
-Lmspider
-Majestic-SEO
-Majestic12
-MolokaiBot
+HybridBot
+IDBTE4M
+InternetSeer
+Joomla
+LNSpiderguy
+LibWeb
+LieBaoFast
+Linkbot
+LinqiaMetadataDownloaderBot
+Majestic SEO
+MicroMessenger
+Msrabot
 Musobot
+NICErsPRO
+Net Vampire
 NetAnts
 Netcraft
-Ninja
-Offline Explorer
-Offline Navigator
-OnCrawl
-OpenLinkProfiler
-OrangeSpider
+Nibbler
+OpenVAS
 Page Analyzer
-Pandalytics
+Picscout
+Piepmatz
 PxBroker
-RSSingBot
-RankActive
-RankActiveLinkBot
-RankingBot2
+RankFlex
+RankingBot
 Re-re
-Reaper
 RebelMouse
-Rogerbot
-SBIder
-SEOstats
+RedesScrapy
 SMTBot
-Scanbot
-Searchestate
+SalesIntelligent
+Screaming
+ScreenerBot
+Seekport
 SeekportBot
-SeoSiteCheckup
-Seomoz
-Shodan
-SiteSnagger
-SmartDownload
-Snapbot
+SlySearch
+Snoopy
 SocialRankIOBot
-Sociscraper
+Sogou web spider
 SpankBot
-Spinn3r
+Spanner
+Steeler
 Stripper
-TightTwatBot
-Twice
-V-BOT
-Vagabondo
-VeriCiteCrawler
+Surfbot
+SurveyBot
+Szukacz
+TinyTestBot
+Trendictionbot
+Turnitin
+TurnitinBot
+Typhoeus
+VB Project
+Vacuum
+VelenPublicWebCrawler
 VoidEYE
 WWW-Mechanize
-WWW::Mechanize
 Web Auto
-WebImageCollector
-WebSucker
+WebCopier
+WebFuck
+WebWhacker
 WebZIP
-WebsiteQuester
-Zeus
-ZoomBot
-anthropic-ai
-dataforseo.com
-eCatch
-meanpathbot
-seoscanners
-seostar
-x09Mozilla
+Whatweb
+WiseGuys Robot
+Zermelo
+Zitebot
+adscanner
+cognitiveseo
+crawler4j
+facebookscraper
+gopher
+openai.com
+probe-image-size
+siteripz
+spyfu
+trendiction.com
+ubermetrics-technologies.com
+webprosbot
 zgrab