V4.2024.05.4517 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-360Spider
-404checker
-404enemy
-80legs
-Aboundex
-Aboundexbot
-Alexibot
+ASPSeek
+Anarchy99
 Attach
-BBBike
-BDCbot
-BackDoorBot
-Bandit
-BetaBot
-BlackWidow
+Buddy
 BuiltBotTough
-CODE87
-ChinaClaw
+Bytespider
+CSHttp
+Chlooe
 Citoid
 Claritybot
 Cocolyzebot
-Copier
 Crawling at Home Project
-Curious
-DTS Agent
+CyotekWebCopy
+DIIbot
+DataCha0s
 Devil
-Disco
-DomainAppender
 Domains Project
-Download Wonder
 EroCrawler
 FDM
+FHscan
 FacebookBot
-GalaxyBot
-Getintent
-HTTrack
-HaosouSpider
-Hloader
+FrontPage
+Genieo
+GermCrawler
+Gigabot
+GrabNet
+HEADMasterSEO
 Humanlinks
-IRLbot
+HybridBot
+Iblog
 Image Sucker
-ImagesiftBot
-InterGET
-InternetMeasurement
-InternetSeer
-Leap
-LinkpadBot
-LinksManager
-Lipperhey
-MIIxpc
-MSFrontPage
-MSIECrawler
-Mag-Net
-Microsoft URL Control
-MolokaiBot
-NetLyzer
-NetMechanic
-Nimbostratus
-Octopus
-OrangeSpider
-Pavuk
-Pi-Monster
-Pump
+Internet Ninja
+JetCar
+Jorgee
+Jyxobot
+Kenjin Spider
+Keybot Translation-Search-Machine
+LNSpiderguy
+Larbin
+LexiBot
+LinkScan
+LinqiaRSSBot
+Ltx71
+MTRobot
+Magnet
+Mail.RU_Bot
+Majestic-SEO
+Masscan
+Mata Hari
+Name Intelligence
+NetZIP
+NextGenSearchBot
+PECL::HTTP
+PHPCrawl
+Papa Foto
+Picscout
+Probethenet
+Psbot
 PyCurl
-Rainbot
-SISTRIX
-SMTBot
-SalesIntelligent
-Screaming
+RankurBot
+RepoMonkey
+ScoutJet
 SemanticJuice
-Semrush
-Sitebeam
-SmartDownload
+SeobilityBot
+Shodan
+Snapbot
 SocialRankIOBot
-Spinn3r
-Sqworm
-Sucuri
-Szukacz
-Toweyabot
-TwengaBot
-Vacuum
-WEBDAV
-Web Fetch
-WebsiteExtractor
-Whatweb
-YaK
-anthropic-ai
-cognitiveseo
+Sogou web spider
+Sottopop
+Spammen
+Steeler
+SuperBot
+T0PHackTeam
+Typhoeus
+V-BOT
+VeriCiteCrawler
+Voltron
+WASALive-Bot
+WebCollage
+WebmasterWorldForumBot
+Website Quester
+WebsiteQuester
+Webster
+Who.is Bot
+Woobot
+Zauba
+ZyBorg
+adscanner
+archive.org_bot
+cohere-ai
 com.plumanalytics
-dataforseo.com
-dataforseobot
-facebookscraper
-linkfluence
-meanpathbot
-pcBrowser
+heritrix
+omgili
+openai
+openai.com
+plumanalytics
+polaris version
+ripz
 scalaj-http
-sogouspider
-sp_auditbot
-spyfu
-x09Mozilla
-zgrab
+seoscanners