V4.2023.08.3926 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
-404checker
-AiHitBot
-Anthill
-Asterias
+404enemy
+AdsTxtCrawlerTP
+AfD-Verbotsverfahren
+AhrefsBot
 AwarioSmartBot
-BackStreet
-Bandit
-Boardreader
-CheeseBot
+BDCbot
+BacklinkCrawler
+Brandwatch
+BuiltBotTough
+Bullseye
+BuzzSumo
+CODE87
+Calculon
 Cloud mapping
-Cocolyzebot
-Crawling at Home Project
-Custo
-DSearch
-Digincore
+Cosmos
+Crescent
+Dispatch
+DittoSpyder
+DomainSigmaCrawler
+Dotbot
 Drip
-EasyDL
-FlashGet
-FrontPage
-FyberSpider
+EMail Wolf
+EirGrabber
 Fyrebot
+Gigabot
 Go!Zilla
-Grafula
-GrapeFX
+GoZilla
 GridBot
-Id-search
+HTMLparser
+Harvest
+Havij
 InfoTekies
-Internet Ninja
-Jyxobot
-Kenjin Spider
+Jbrofuzz
 Kozmosbot
-Larbin
-LeechFTP
-LinkextractorPro
-LinksManager
-LinqiaRSSBot
-Lipperhey
-Mata Hari
-Mediatoolkitbot
-Mr.4x3
-Nettrack
-Netvibes
-Nibbler
-Nimbostratus
-Pandalytics
+LexiBot
+Lftp
+LibWeb
+Libwhisker
+Lightspeedsystems
+Lipperhey Spider
+Litemage_walker
+MIIxpc
+MSFrontPage
+Majestic-SEO
+MarkMonitor
+MeanPath Bot
+Metauri
+Microsoft Data Access
+Mojeek
+MolokaiBot
+Musobot
+NearSite
+NetLyzer
+Nuclei
+OrangeSpider
+PECL::HTTP
+Papa Foto
+ProPowerBot
 Proximic
-PxBroker
-Rankivabot
-SEOlyticsCrawler
-SEOstats
+ReGet
+SEOkicks-Robot
 SISTRIX
-SMTBot
-ScoutJet
-Searchestate
-SentiBot
-SeobilityBot
-Shodan
-Siphon
-Site Sucker
-SmartDownload
+ScanAlert
+Seekport
+Seomoz
+SiteCheckerBotCrawler
+SiteExplorer
+Siteimprove
 SocialRankIOBot
-SpaceBison
-Spanner
+Sottopop
+SpankBot
 SputnikBot
-Sucker
 Surfbot
-Suzuran
-Titan
-TwengaBot
-Twice
-Vagabondo
-Virusdie
-Web Enhancer
-Web Pix
-WebGo IS
+Teleport
+Toata
+Typhoeus
+URLy Warning
+Vacuum
+WEBDAV
 WebLeacher
-WebSauger
-WebsiteExtractor
-Xaldon WebSpider
-Xenu
-YoudaoBot
-Zermelo
-arquivo-web-crawler
-cah.io.community
-clark-crawler
-coccocbot
-demandbase-bot
-heritrix
-iubenda-radar
-openai.com
-pcBrowser
-seocompany.store
-seostar
-sogouspider
-sp_auditbot
-spyfu
-tAkeOut
-webpros.com
-zauba.io
+WebSucker
+WebWhacker
+WebZIP
+WebmasterWorldForumBot
+Whack
+WinHTTrack
+Zeus
+ZmEu
+ZoomBot
+ZoominfoBot
+com.plumanalytics
+crawler.feedback
+ips-agent
+lwp-trivial
+openai
+polaris version
+s1z.ru
+scalaj-http
+seobility
+sysscan