V4.2023.06.3826 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-360Spider
-80legs
-AIBOT
-ALittle Client
-BackStreet
-Bitacle
-BlackWidow
-BlowFish
-Bolt
-Brandprotect
-Bullseye
-Calculon
-CazoodleBot
-CensysInspect
+Alexibot
+AllSubmitter
+Apexoo
+Atomseobot
+Attach
+Backlink-Ceck
+CODE87
+Cegbfeieh
+ChinaClaw
+Chlooe
 Cloud mapping
-Crawling at Home Project
-CrunchBot
-Custo
-DSearch
-EMail Wolf
-Extreme Picture Finder
-FHscan
-Fimap
-FlashGet
+Collector
+Copier
+DataCha0s
+Deusu
+Discobot
+DomainCrawler
+Drip
+EMail Siphon
+EasyDL
+EirGrabber
+EroCrawler
+Freeuploader
 Fuzz
 FyberSpider
-Gigabot
+GetRight
 Grabber
-HTTP::Lite
-HybridBot
-IRLbot
-JamesBOT
-JikeSpider
+HMView
+HTMLparser
+Heritrix
+Id-search
+Iskanie
+Keybot Translation-Search-Machine
 LNSpiderguy
-LieBaoFast
-LinqiaScrapeBot
+Larbin
+Leap
+Lftp
+Libwhisker
+Likse
+Lipperhey
+Magnet
+Mail.RU_Bot
 Majestic SEO
-Majestic12
-Nameprotect
-Navroad
-NetLyzer
+Majestic-SEO
+MarkMonitor
+Masscan
+Mata Hari
+MeanPath Bot
+MegaIndex.ru
+Metauri
+Mojolicious
+Nettrack
 Netvibes
-Nimbostratus
-Nmap
-Offline Explorer
-OnCrawl
-OrangeSpider
-OutfoxBot
 PHPCrawl
-Piepmatz
-ProWebWalker
-PyCurl
-RankFlex
-ReGet
-RealDownload
-RepoMonkey
-RocketCrawler
-SISTRIX
-Scrapy
+Page Analyzer
+Pandalytics
+Panscient
+Pixray
+RankurBot
+Reaper
 ScreenerBot
-SentiBot
-Shodan
-Siphon
-Site Sucker
+Semrush
 SiteCheckerBotCrawler
-Siteimprove
-Sucuri
-SuperBot
-SurveyBot
-Suzuran
+SmartDownload
+Sociscraper
+Sosospider
+Steeler
+Stripper
 Swiftbot
 T0PHackTeam
-Telesphoreo
-The Intraformant
-Turnitin
+Telesphorep
+TightTwatBot
+TwengaBot
+Typhoeus
 URLy.Warning
+VeriCiteCrawler
 Voil
-Voltron
 WWW-Mechanize
-WWWOFFLE
-Web Sauger
-WebAuto
-WebWhacker
-Webshag
+Web Auto
+Web Fuck
+WebFuck
+WebSucker
+WebmasterWorldForumBot
 Website Quester
-Whatweb
-Who.is Bot
+WebsiteExtractor
+Xaldon_WebSpider
+YoudaoBot
+Zermelo
 arquivo-web-crawler
+backlink-check
 cah.io.community
+clark-crawler
 dataforseobot
-facebookscraper
+lwp-request
 magpie-crawler
-netEstate NE Crawler
-s1z.ru
 satoristudio.net
-scan.lol
-seobility
-spyfu
-xpymep1.exe
-zgrab
+sexsearcher
+trendiction.de
+webmeup-crawler
+x09Mozilla