V4.2024.01.4255 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-AfD-Verbotsverfahren
-AhrefsBot
-Anarchy99
-AwarioSmartBot
-BacklinkCrawler
-Badass
-Bolt
-BotALot
-Claritybot
-CopyRightCheck
-CrunchBot
-Custo
+80legs
+ALittle Client
+ASPSeek
+Aboundex
+AlphaBot
+Attach
+AwarioRssBot
+BBBike
+BDFetch
+BackDoorBot
+BlowFish
+Brandwatch
+Buddy
+BuiltWith
+Bullseye
+Bytespider
+Collector
+Copier
+Crescent
+DIIbot
 Demon
-Digincore
-Discobot
-Domains Project
+DittoSpyder
+DomainCrawler
 Download Wonder
 Dragonfly
-Drip
 EroCrawler
-Extractor
-EyeNetIE
-FHscan
-GetRight
-Gotit
-HTTP::Lite
-Harvest
-IDBTE4M
+FyberSpider
+GT::WWW
+GrabNet
+HEADMasterSEO
+Hloader
+Humanlinks
+IDBot
+IRLbot
 IlseBot
-IndeedBot
-Iria
-Larbin
-LeechGet
-LexiBot
-Likse
-Linkbot
-LinqiaRSSBot
+Indy Library
+LWP::Simple
+Lanshanbot
+LinkScan
+LinkWalker
+LinksManager
+LinqiaScrapeBot
 Lmspider
-MJ12bot
-Mass Downloader
-Microsoft Data Access
-Microsoft URL Control
-Mojolicious
-NearSite
-Needle
-Nessus
+MIDown tool
+MIIxpc
+MQQBrowser
+Mag-Net
+Mail.RU_Bot
+Majestic SEO
+MegaIndex.ru
+Metauri
+Musobot
+Name Intelligence
+NetZIP
 Netcraft
-Nettrack
 Niki-bot
-OpenLinkProfiler
-OpenVAS
-Openvas
-POE-Component-Client-HTTP
-Panscient
-PictureFinder
-Psbot
-RankFlex
-RankingBot2
-Re-re
-RealDownload
-RepoMonkey
-Ripper
-ScrepyBot
-Searchestate
+NimbleCrawler
+Nutch
+Offline Explorer
+PECL::HTTP
+Pi-Monster
+Pockey
+ProPowerBot
+Pu_iN
+PxBroker
+RankActiveLinkBot
+ReGet
+SEOprofiler
+SMTBot
+ScreenerBot
 SearchmetricsBot
-SemanticJuice
 Semrush
-SiteLockSpider
-SiteSnagger
 Snake
-Sociscraper
-SuperHTTP
-Telesoft
-Toweyabot
-Turnitin
-TurnitinBot
-Twice
-VoidEYE
-WISENutbot
-WWW-Mechanize
-Wallpapers/3.0
-WebCollage
-WebsiteExtractor
-WinHTTrack
-WiseGuys Robot
-Xaldon WebSpider
-Zermelo
-backlink-check
-crawler.feedback
+Spinn3r
+Sucuri
+Trendiction
+Upflow
+Vagabondo
+Wallpapers
+WeSEE
+Web Auto
+Web Enhancer
+Web Fuck
+Web Pix
+Web Sucker
+WebStripper
+Webster
+ZoominfoBot
+check1.exe
+dataforseo.com
+dataforseobot
+demandbase-bot
 facebookscraper
+heritrix
 iubenda-radar
-linkdexbot
-meanpathbot
-plumanalytics
-scan.lol
-seobility
-sexsearcher
-sogouspider
-ubermetrics-technologies.com
-webprosbot
+magpie-crawler
+openai
+page scorer
+xpymep1.exe
+zgrab