V4.2023.03.3616 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
-AIBOT
-Aboundex
-Acunetix
-AfD-Verbotsverfahren
-Apexoo
-Attach
-AwarioRssBot
-BBBike
-BDCbot
+ADmantX
+ASPSeek
+AhrefsBot
+Alexibot
+Anarchy
+Anarchy99
 BacklinkCrawler
-BatchFTP
-BetaBot
+Black Hole
 Bolt
-BuiltWith
-Bullseye
-CODE87
-CheeseBot
-Claritybot
-Cloud mapping
-Cogentbot
-Cosmos
-DittoSpyder
+BotALot
+Brandprotect
+Brandwatch
+CazoodleBot
+CensysInspect
+ChinaClaw
+Citoid
+Collector
+Crawling at Home Project
+Curious
+Custo
 DnyzBot
-DomainSigmaCrawler
-Drip
-Ebingbong
-Fyrebot
-Gotit
-HMView
-HTTrack
-HonoluluBot
-Image Sucker
-IstellaBot
-Jbrofuzz
-Keybot Translation-Search-Machine
-LWP::Simple
-LinkextractorPro
-LinqiaMetadataDownloaderBot
-Lipperhey Spider
-MIIxpc
-Mag-Net
-MarkWatch
-Meanpathbot
+EMail Siphon
+EasyDL
+Exabot
+ExtractorPro
+FyberSpider
+GalaxyBot
+GetRight
+Getintent
+HybridBot
+IDBTE4M
+InfoNaviRobot
+JOC Web Spider
+Larbin
+LeechGet
+Lftp
+Libwhisker
+Ltx71
+Majestic12
 MegaIndex.ru
 MicroMessenger
-Mojeek
-Morfeus Fucking Scanner
+Mojolicious
 Msrabot
-NetZIP
-Openfind
-OrangeBot
-RSSingBot
-SISTRIX
+Nibbler
+Niki-bot
+Nikto
+NimbleCrawler
+Nimbostratus
+OpenLinkProfiler
+PECL::HTTP
+Pandalytics
+Pavuk
+ProPowerBot
+Probethenet
+PxBroker
+RankurBot
+RocketCrawler
+SBIder
+SEOprofiler
+Scanbot
+ScrepyBot
 Searchestate
-SearchmetricsBot
-Seekport
-SemrushBot
-Shodan
-SiteLockSpider
-Sottopop
-SpaceBison
-Spbot
+SeoSiteCheckup
+SiteSnagger
+Sitevigil
+Snake
+Snoopy
+Sogou web spider
 Sqlmap
-Sqworm
-Stripper
-TeleportPro
-Telesoft
-Toweyabot
-True_Robot
-UnisterBot
-VidibleScraper
-WBSearchBot
+Steeler
+SuperBot
+SuperHTTP
+SurveyBot
+Trendictionbot
+URLy.Warning
+V-BOT
+Vagabondo
+VeriCiteCrawler
+WISENutbot
 WWW-Collector-E
-Web Pix
+WWW::Mechanize
+Web Fetch
+Web Fuck
 WebFetch
+WebLeacher
+Webalta
+Webshag
 WebsiteQuester
-Whacker
-Who.is Bot
-WiseGuys Robot
-Wonderbot
-Wotbox
-Zitebot
-ZoominfoBot
-ZumBot
-archive.org_bot
-autoemailspider
-backlink-check
-cah.io.community
+ZmEu
+ZoomBot
 clark-crawler
-dataforseo.com
-iubenda-radar
-lwp-request
-netEstate NE Crawler
-polaris version
-s1z.ru
+gopher
+magpie-crawler
+ripz
+scalaj-http
+scan.lol
 seocompany.store
+tAkeOut
 trendiction.de
-ubermetrics-technologies.com
 zgrab