V4.2025.08.5433 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-404checker
-404enemy
-Aboundex
-AdsTxtCrawlerTP
-Alexibot
-Anarchy99
-Apexoo
-Asterias
-BacklinkCrawler
-Bandit
-Bolt
-BuzzSumo
-CheeseBot
-ClaudeBot
-Cogentbot
-Copier
-Copyscape
-Dirbuster
-EroCrawler
-EyeNetIE
-Fimap
-Firefox/7.0
-Fyrebot
-GermCrawler
-GrabNet
-GrapeshotCrawler
-HEADMasterSEO
-Harvest
+01h4x.com
+80legs
+Aboundexbot
+Ai2Bot
+Aipbot
+Anarchie
+Aspiegel
+BDFetch
+BackDoorBot
+BackWeb
+Badass
+Bitacle
+CODE87
+Calculon
+Cegbfeieh
+Cliqzbot
+Cloud mapping
+DIIbot
+DnyzBot
+ECCP/1.0
+EMail Siphon
+Ecxi
+Extractor
+Extreme Picture Finder
+FacebookBot
+GeedoProductSearch
+GetWeb
+Grafula
+HTMLparser
+Heritrix
 HonoluluBot
-HybridBot
-Iblog
-Information Security Team InfraSec Scanner
+InfoTekies
+Intelliseek
 InternetMeasurement
-JetCar
-Leap
-LinkScan
-LinkpadBot
-Lipperhey Spider
-Majestic12
+JOC Web Spider
+JennyBot
+Kozmosbot
+Lftp
+LibWeb
+Lightspeedsystems
+LinkextractorPro
+LinksManager
+MSFrontPage
+Majestic-SEO
+MarkMonitor
 Mass Downloader
-MeanPath Bot
-Mediatoolkitbot
+MauiBot
 NPbot
-NetLyzer
-Netcraft
-Nettrack
+Nameprotect
+Netvibes
 NextGenSearchBot
-NimbleCrawler
-Nmap
-Offline Explorer
-OnCrawl
+Nimbostratus
+OpenLinkProfiler
 OutclicksBot
-PHPCrawl
-PageThing.com
-Pi-Monster
-Pimonster
-Pockey
-Psbot
-Pu_iN
-PxBroker
-PyCurl
-Rainbot
+Page Analyzer
+Picsearch
 RankActive
+RankingBot
 RankurBot
-SEOprofiler
-SemrushBot-BA
-SemrushBot-FT
-SemrushBot-OCOB
-SeoSiteCheckup
-Seomoz
+ReGet
+SEOlyt
+SEOlyticsCrawler
+SearchmetricsBot
+SeekportBot
 Site Sucker
-SiteSucker
 SmartDownload
-Snake
-Snoopy
-Spider_Bot/3.0
-SplitSignalBot
-Sqworm
-Sucker
-The Intraformant
-URLy Warning
-URLy.Warning
-VB Project
-WISENutbot
-WWW::Mechanize
-Whatweb
-WinHTTrack
-Wonderbot
+Sociscraper
+SuperBot
+SurveyBot
+Toata
+Vagabondo
+VelenPublicWebCrawler
+Virusdie
+WEBDAV
+Wallpapers
+WebBandit
+WebPix
+WebSauger
 Xaldon_WebSpider
-Zeus
-crawler.feedback
-everyfeed-spider
-iubenda-radar
-muhstik-scan
-page scorer
-ripz
-tAkeOut
-trendiction.com
-xpymep1.exe
+Xenu
+ZyBorg
+adscanner
+anthropic-ai
+autoemailspider
+awario.com
+cah.io.community
+cognitiveseo
+crawler4j
+dataforseobot
+eCatch
+facebookscraper
+internetVista monitor
+ips-agent
+polaris version
+scalaj-http
+sexsearcher
+sitechecker.pro
+sysscan
+webmeup-crawler
 zauba.io