V4.2024.05.4496 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
 360Spider
-404checker
-404enemy
-ADmantX
-BDFetch
-Backlink-Ceck
-BetaBot
-Blackboard
-BotALot
+AIBOT
+AfD-Verbotsverfahren
+Anarchy99
+Asterias
+BDCbot
+Barkrowler
 Brandwatch
-Chlooe
-Claritybot
-DBLBot
-DTS Agent
-DigitalPebble
+BuiltBotTough
+Cegbfeieh
+Crescent
+Discoverybot
 DittoSpyder
+DnyzBot
 DomainStatsBot
-Dragonfly
-EMail Siphon
-ExtLinksBot
-FDM
-Foobot
-GetRight
-GetWeb
-Getintent
-Go-Ahead-Got-It
-GoZilla
-GrabNet
-Iblog
-IlseBot
+Flunky
+Fuzz
+GalaxyBot
+Go!Zilla
+Google-Extended
+Gotit
+HMView
+HTTP::Lite
+IDBot
+IRLbot
+Image Fetch
 Image Sucker
-JustView
-Jyxobot
-Kozmosbot
-LWP::Simple
-Lanshanbot
-Larbin
-LeechFTP
-LibWeb
-LinkScan
-LinqiaScrapeBot
-Magnet
-Mass Downloader
-MauiBot
-Minefield
-Mojeek
-NICErsPRO
-NetAnts
-Netvibes
-Openvas
-PECL::HTTP
-POE-Component-Client-HTTP
-Pavuk
-Petalbot
-Piepmatz
-Pimonster
+Iria
+Jbrofuzz
+Keyword Density
+Kinza
+Lftp
+LinkextractorPro
+LinqiaRSSBot
+Ltx71
+MFC_Tear_Sample
+MIDown tool
+MegaIndex.ru
+Mr.4x3
+Msrabot
+Musobot
+Navroad
+Needle
+Nessus
+Net Vampire
+NetSpider
+NextGenSearchBot
+NimbleCrawler
+Octopus
+OpenLinkProfiler
+OutclicksBot
+PageScorer
+PageThing.com
+Panscient
+Pixray
 PleaseCrawl
-Pump
-Re-re
-RedesScrapy
-SEOstats
-Scrapy
-SiteCheckerBotCrawler
-SiteSucker
+ProWebWalker
+Proximic
+RankActive
+SEOlyticsCrawler
+SISTRIX
+Scanbot
+Semrush
+SemrushBot
 Sitebeam
 Siteimprove
-SocialRankIOBot
+Sogou web spider
+Sottopop
+Spammen
 SpankBot
-Stripper
-Surfbot
+SurveyBot
+Trendictionbot
 Turingos
-TurnitinBot
-TwengaBot
-VB Project
-VCI
-VeriCiteCrawler
 WEBDAV
-WWW-Collector-E
-Web Auto
-Web Sucker
-WebLeacher
-WebStripper
-WebSucker
-WebmasterWorldForumBot
+Wallpapers/3.0
+Web Sauger
+WebBandit
+WebCollage
+WebZIP
 WebsiteExtractor
-Widow
-WiseGuys Robot
+Whack
+Whacker
+Woobot
 Xaldon WebSpider
-adscanner
-anthropic-ai
-crawl.sogou.com
-crawler4j
-imagesift.com
-internetVista monitor
+Xaldon_WebSpider
+arquivo.pt
+autoemailspider
+cohere-ai
+linkfluence
 magpie-crawler
 muhstik-scan
-netEstate NE Crawler
+omgili
+scalaj-http
 seobility
-ubermetrics-technologies.com
-x22Mozilla
+siteripz
+sysscan
+trendiction.de
+x09Mozilla
+xpymep1.exe
+zgrab