V4.2024.05.4526 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-404checker
-Aboundex
-AdsTxtCrawlerTP
+360Spider
+ADmantX
+Acunetix
 Alexibot
-Anarchy99
-AwarioRssBot
+Anarchy
+Ankit
+AwarioSmartBot
+BDCbot
+BackStreet
+Blow
 Brandprotect
-Buck
-BuiltWith
-CCBot
-CherryPicker
-Citoid
+Buddy
+Bullseye
+Copier
 Copyscape
-Craftbot
-DIIbot
-DSearch
-DataCha0s
-Digincore
-DittoSpyder
-DnBCrawler-Analytics
-DomCopBot
-DomainSigmaCrawler
+DigitalPebble
+Dirbuster
+ECCP/1.0
+Ebingbong
+EirGrabber
+Evil
 Express WebPictures
 Extreme Picture Finder
-FlashGet
-Harvest
-Hloader
-HonoluluBot
+FDM
+GetRight
+Humanlinks
+Indy Library
 Information Security Team InfraSec Scanner
-JOC Web Spider
+Internet Ninja
 JamesBOT
-Joomla
-Kenjin Spider
-Kozmosbot
+JennyBot
+Jetty
+JustView
+Leap
 Lftp
-Linkbot
-Ltx71
-MIDown tool
-MIIxpc
-Majestic SEO
-Majestic12
-MarkWatch
-Masscan
-Mata Hari
-Mb2345Browser
-MicroMessenger
-MolokaiBot
-Navroad
-NetSpider
+LieBaoFast
+LinksManager
+LinqiaScrapeBot
+Lipperhey
+Litemage_walker
+MQQBrowser
+MSIECrawler
+Mail.RU_Bot
+MauiBot
+Metauri
+Minefield
+Name Intelligence
+Netvibes
+NextGenSearchBot
 Niki-bot
-Offline Navigator
-Pavuk
-Petalbot
-Psbot
-QueryN Metasearch
-RankingBot2
-ReGet
-ScoutJet
-Screaming
-ScrepyBot
-SearchmetricsBot
-SiteCheckerBotCrawler
+OrangeSpider
+PECL::HTTP
+Page Analyzer
+PageGrabber
+Pi-Monster
+Piepmatz
+ProPowerBot
+PyCurl
+RankingBot
+RepoMonkey
+Ripper
+RocketCrawler
+Scrapy
+Seekport
+SentiBot
+SeobilityBot
+Site Sucker
+SiteLockSpider
+Sitevigil
 SmartDownload
 Snoopy
-SputnikBot
-Sqlworm
-Suzuran
-T8Abot
-Telesoft
-Thumbor
-VB Project
-Vacuum
-VelenPublicWebCrawler
-WWW-Collector-E
-WWW-Mechanize
-Wallpapers
-Wallpapers/3.0
-WallpapersHD
-Web Auto
-Web Collage
-Whacker
-Whatweb
-Woobot
-Xenu
+Sogou web spider
+Sottopop
+Steeler
+Stripper
+Sucker
+SurveyBot
+Virusdie
+Voltron
+Web Fetch
+Web Sauger
+WebReaper
+Widow
 YoudaoBot
-cohere-ai
-crawl.sogou.com
+Zitebot
+ZmEu
+ZumBot
+ZyBorg
+crawler.feedback
 demandbase-bot
 facebookscraper
-heritrix
 lwp-trivial
-magpie-crawler
-omgili
+openai.com
+pcBrowser
 seobility
 seocompany.store
 serpstatbot
-sexsearcher
-sitechecker.pro
-webgains-bot
+sysscan
 x09Mozilla
+zauba.io