V4.2024.09.4728 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
+01h4x.com
 404checker
-Aboundexbot
-Alligator
-Apexoo
-BLEXBot
-BackWeb
-Badass
-Bandit
-Buck
-CazoodleBot
-Citoid
-Cloud mapping
-Collector
-Crawling at Home Project
-CrazyWebCrawler
-Custo
-DIIbot
-Demon
-Deusu
-DigitalPebble
-Domains Project
-Dotbot
-Download Wonder
-Dragonfly
-Exabot
-ExtractorPro
-Extreme Picture Finder
-Foobot
-FyberSpider
-GT::WWW
+ASPSeek
+Anarchy
+Ankit
+Asterias
+BotALot
+Buddy
+CherryPicker
+Cliqzbot
+Cocolyzebot
+DataCha0s
+Devil
+DittoSpyder
+Drip
+EMail Siphon
+EasyDL
+EyeNetIE
+FHscan
+FacebookBot
+Flunky
+GPTBot
 GetWeb
-GoZilla
-GrapeFX
-HaosouSpider
-IDBTE4M
-IndeedBot
+GridBot
+HTMLparser
+Hloader
+IRLbot
+Iblog
+Intelliseek
 InterGET
-InternetMeasurement
-Iria
-Jbrofuzz
 JetCar
-Jorgee
-LinkScan
-Mag-Net
-MarkMonitor
-MegaIndex.ru
+Kenjin Spider
+LWP::Simple
+Linkbot
+MIDown tool
+MarkWatch
 Metauri
-Musobot
-NPbot
-Name Intelligence
-Nikto
-Nimbostratus
-Nmap
-OrangeBot
-OrangeSpider
-OutfoxBot
+Microsoft URL Control
+Nameprotect
+NetMechanic
+Nibbler
+Niki-bot
+Offline Explorer
+OnCrawl
+OutclicksBot
+PECL::HTTP
+ProPowerBot
 ProWebWalker
+Psbot
+PxBroker
 Quick-Crawler
-RSSingBot
-RankingBot
 RankingBot2
-RealDownload
-Recorder
-ScreenerBot
-Semrush
-SeobilityBot
-Seomoz
-SiteCheckerBotCrawler
+Rankivabot
+SEOprofiler
+SEOstats
+ScrepyBot
+SemanticJuice
+SiteExplorer
+SiteLockSpider
+SmartDownload
+Snapbot
+Sociscraper
+Sogou web spider
 SpaceBison
-Sucker
-SuperHTTP
-Swiftbot
+Spanner
+Sqworm
 TeleportPro
-Telesoft
-Telesphoreo
-Toata
-Toweyabot
-TurnitinBot
-Typhoeus
-VeriCiteCrawler
+Telesphorep
+Thumbor
+Tracemyfile
+Trendictionbot
+Turnitin
+Voil
 WPScan
-WallpapersHD
-WeSEE
-Web Auto
-Web Sauger
+Web Sucker
+WebWhacker
 WebsiteExtractor
-Whack
+WebsiteQuester
+Webster
+Wotbox
+Xaldon WebSpider
+Xenu
+ZoomBot
 ZumBot
+ZyBorg
+arquivo-web-crawler
 arquivo.pt
-com.plumanalytics
+crawler.feedback
+dataforseo.com
+dataforseobot
+gopher
 instabid
 openai.com
-probe-image-size
-satoristudio.net
-scalaj-http
-seobility
-sexsearcher
-sitechecker.pro
-ubermetrics-technologies.com
-webgains-bot
+pcBrowser
+ripz
+scan.lol
+serpstatbot
+sp_auditbot
+sysscan
+zauba.io