V4.2024.03.4347 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-Aboundex
-Aboundexbot
-AwarioRssBot
-BackDoorBot
-Bandit
-BatchFTP
-BetaBot
+360Spider
+BackStreet
+Backlink-Ceck
+Bigfoot
+Bitacle
 Blackboard
-CheTeam
-Cogentbot
-Collector
-Copier
-CopyRightCheck
-Craftbot
-CyotekWebCopy
-Discobot
-DittoSpyder
-DomainAppender
-Drip
-Ebingbong
-EroCrawler
-Express WebPictures
-Ezooms
-FacebookBot
-Firefox/7.0
+BlowFish
+Boardreader
+BotALot
+Bytespider
+CazoodleBot
+Cocolyzebot
+Cosmos
+DIIbot
+Digincore
+Disco
+DomCopBot
+Download Wonder
+Ecxi
+Evil
+FHscan
+Fimap
+Flunky
 Foobot
-FrontPage
-Fuzz
-GalaxyBot
-Genieo
+Fyrebot
 GetRight
-Go!Zilla
-Go-Ahead-Got-It
-GrabNet
+GoZilla
 HaosouSpider
-Harvest
-IRLbot
-InterGET
-Internet Ninja
-Iria
-LibWeb
-LinkextractorPro
-Litemage_walker
-Meanpathbot
+HybridBot
+IDBot
+Indy Library
+InfoNaviRobot
+Iskanie
+JOC Web Spider
+JetCar
+Jetty
+Jorgee
+LeechGet
+LinkScan
+LinqiaScrapeBot
+MSFrontPage
+Mag-Net
+Magnet
+Metauri
+Minefield
 Musobot
-NetLyzer
-NetMechanic
-NetZIP
-Netvibes
-Nibbler
-Offline Navigator
-PageAnalyzer
-Panscient
-Pavuk
-Picsearch
-Piepmatz
-PleaseCrawl
-Pockey
-ProWebWalker
-Pu_iN
-Pump
+Nikto
+OpenLinkProfiler
+OutfoxBot
+PECL::HTTP
+Pixray
+Quick-Crawler
+RSSingBot
+RankActive
 RankFlex
-RankurBot
 Re-re
-RedesScrapy
-SEOprofiler
-SalesIntelligent
-ScoutJet
+ReGet
+RepoMonkey
+Rogerbot
+ScanAlert
+Scrapy
+ScrepyBot
 Seekport
-SeoSiteCheckup
-SocialRankIOBot
-Sosospider
-Sqlworm
-Sqworm
-T8Abot
+SemanticJuice
+SiteCheckerBotCrawler
+Sogou web spider
+Spanner
+Suzuran
 Telesphoreo
 Telesphorep
+Titan
 Toata
-TwengaBot
-Vagabondo
-WWW-Collector-E
-WebCopier
-WebImageCollector
+Tracemyfile
+Twice
+VidibleScraper
+WWW::Mechanize
+WWWOFFLE
+Web Pix
+WebGo IS
 WebStripper
-WebZIP
-WebmasterWorldForumBot
-WebsiteQuester
-Xaldon_WebSpider
-Zauba
-ZyBorg
-anthropic-ai
-crawl.sogou.com
-crawler4j
-gopher
-lwp-trivial
+WebSucker
+WebWhacker
+Webster
+Widow
+WinHTTrack
+Wonderbot
+ZumBot
+arquivo-web-crawler
+cah.io.community
+domainsproject.org
+imagesift.com
+instabid
 magpie-crawler
-webpros.com
-webprosbot
+meanpathbot
+openai
+ripz
+seocompany.store
 x22Mozilla
 xpymep1.exe
+zgrab