V4.2024.05.4495 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
+360Spider
 404checker
-ALittle Client
-Aboundex
-Aboundexbot
-Acunetix
-AhrefsBot
-Anarchy99
-AwarioBot
-Bandit
-Blow
-Brandprotect
-CODE87
-Cogentbot
-Copyscape
+404enemy
+ADmantX
+BDFetch
+Backlink-Ceck
+BetaBot
+Blackboard
+BotALot
+Brandwatch
+Chlooe
+Claritybot
+DBLBot
+DTS Agent
 DigitalPebble
-EroCrawler
-Extreme Picture Finder
-EyeNetIE
-FlashGet
+DittoSpyder
+DomainStatsBot
+Dragonfly
+EMail Siphon
+ExtLinksBot
+FDM
+Foobot
 GetRight
-Gigabot
-Havij
-Hloader
-Id-search
-Image Fetch
-InternetMeasurement
+GetWeb
+Getintent
+Go-Ahead-Got-It
+GoZilla
+GrabNet
+Iblog
+IlseBot
+Image Sucker
+JustView
 Jyxobot
-Kenjin Spider
-Kinza
+Kozmosbot
 LWP::Simple
+Lanshanbot
+Larbin
+LeechFTP
+LibWeb
+LinkScan
 LinqiaScrapeBot
-Lipperhey Spider
-Ltx71
-Microsoft Data Access
+Magnet
+Mass Downloader
+MauiBot
 Minefield
-Mister PiX
-Mr.4x3
+Mojeek
+NICErsPRO
 NetAnts
-NetZIP
-Nikto
-Nimbostratus
-Nutch
-PHPCrawl
-PageAnalyzer
-Picsearch
-PictureFinder
+Netvibes
+Openvas
+PECL::HTTP
+POE-Component-Client-HTTP
+Pavuk
+Petalbot
 Piepmatz
 Pimonster
-Pockey
-Proximic
-QueryN Metasearch
-Quick-Crawler
-RankActive
-RealDownload
-RebelMouse
-Ripper
-RocketCrawler
-SEOlyticsCrawler
-SISTRIX
-ScrepyBot
-SeekportBot
-Seomoz
-Siphon
-SiteSnagger
+PleaseCrawl
+Pump
+Re-re
+RedesScrapy
+SEOstats
+Scrapy
+SiteCheckerBotCrawler
 SiteSucker
-SmartDownload
-Sosospider
-Sqlworm
-Sqworm
-TightTwatBot
-URLy.Warning
-UnisterBot
+Sitebeam
+Siteimprove
+SocialRankIOBot
+SpankBot
+Stripper
+Surfbot
+Turingos
+TurnitinBot
+TwengaBot
 VB Project
-Vagabondo
-WallpapersHD
-WebEnhancer
+VCI
+VeriCiteCrawler
+WEBDAV
+WWW-Collector-E
+Web Auto
+Web Sucker
 WebLeacher
-WebPix
-Website Quester
-Whack
-Whatweb
-Who.is Bot
-Zitebot
-ZumBot
+WebStripper
+WebSucker
+WebmasterWorldForumBot
+WebsiteExtractor
+Widow
+WiseGuys Robot
+Xaldon WebSpider
 adscanner
-arquivo-web-crawler
-autoemailspider
-coccocbot
-cognitiveseo
-demandbase-bot
-iubenda-radar
+anthropic-ai
+crawl.sogou.com
+crawler4j
+imagesift.com
+internetVista monitor
 magpie-crawler
+muhstik-scan
 netEstate NE Crawler
-omgili
-openai
-openai.com
-polaris version
-seocompany.store
-sysscan
-webpros.com
+seobility
+ubermetrics-technologies.com
+x22Mozilla