V4.2025.08.5402 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-Aboundex
-Alexibot
-Aliyun
-AlphaBot
+AdsTxtCrawlerTP
 Anarchy
-Ankit
-Anthill
-Apexoo
-AwarioRssBot
-AwarioSmartBot
-BackDoorBot
-BacklinksExtendedBot
-Bandit
-BatchFTP
-Bitacle
-Buddy
-BuiltBotTough
-Calculon
-Cliqzbot
-CopyRightCheck
+BDFetch
+BacklinkCrawler
+Bolt
+BotALot
+Brandwatch
+CCBot
+CensysInspect
+ChinaClaw
+Claritybot
+Crawling at Home Project
 Curious
-Dirbuster
-Discoverybot
-DnBCrawler-Analytics
-DnyzBot
-EMail Wolf
+DSearch
+DTS Agent
+DataCha0s
+Drip
+EMail Siphon
+EasyDL
 Ecxi
-Flunky
-GT::WWW
+EroCrawler
+Firefox/7.0
+FrontPage
 GalaxyBot
-GeedoProductSearch
-Getintent
-Go!Zilla
-GoZilla
-IDBTE4M
+GermCrawler
+Gotit
+HMView
+HTTrack
+Haansoft
+Heritrix
+Humanlinks
+Iblog
+Image Fetch
 Image Sucker
-InfoNaviRobot
-InterGET
-InternetSeer
+IndeedBot
+Information Security Team InfraSec Scanner
+Iria
 JOC Web Spider
+Jetty
 JustView
-LeechFTP
-LexiBot
-LibWeb
-Linkbot
-LinkextractorPro
-LinqiaMetadataDownloaderBot
-Lipperhey
-Mail.RU_Bot
-Mr.4x3
+Libwhisker
+Lightspeedsystems
+MSIECrawler
+MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
+Mister PiX
 NICErsPRO
-Name Intelligence
-Nutch
-Offline Explorer
-OrangeBot
-Page Analyzer
+NetSpider
+OnCrawl
+Openvas
+PECL::HTTP
+PHPCrawl
 PageAnalyzer
-PageScorer
-Petalbot
+PageGrabber
 Pi-Monster
-PictureFinder
-Psbot
-Quick-Crawler
-RankActiveLinkBot
-RankFlex
-RebelMouse
-RedesScrapy
-Ripper
-SalesIntelligent
-SemrushBot-FT
-SemrushBot-OCOB
+Probethenet
+Proximic
+RankingBot
+RealDownload
+RepoMonkey
+ScoutJet
+SemanticJuice
+Semrush
+SeoCherryBot
 Seomoz
-Shodan
-Sitevigil
-SocialRankIOBot
-Spbot
-Swiftbot
-Titan
-Trendictionbot
-Twice
-VidibleScraper
-Virusdie
+Sosospider
+SpankBot
+Spider_Bot
+Thumbor
+Vagabondo
 WBSearchBot
+WPScan
 WWW-Collector-E
-WWW-Mechanize
-Web Auto
+WeSEE
+Web Sucker
+WebCopier
 WebEnhancer
 WebImageCollector
-WebsiteExtractor
-ZoomBot
-ZumBot
+WebReaper
+Webshag
+Widow
+Xaldon WebSpider
+ZyBorg
 archive.org_bot
+cah.io.community
 crawler4j
-domainsproject.org
+dataprovider
 evc-batch
-everyfeed-spider
 heritrix
-imagesift.com
-magpie-crawler
-siteripz
+linkfluence
+openai
+page scorer
+pcBrowser
+s1z.ru
+satoristudio.net
+scan.lol
+seoscanners
+sysscan
+voyagerx.com