V4.2024.02.4287 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-404enemy
-AdsTxtCrawlerTP
-AfD-Verbotsverfahren
-AiHitBot
-Aspiegel
+ASPSeek
+Aboundexbot
+Anarchy99
 BackDoorBot
-BackWeb
-Blackboard
-CazoodleBot
-Chlooe
-Cogentbot
-CyotekWebCopy
-DTS Agent
-Demon
-Discobot
+Backlink-Ceck
+Black Hole
+Bolt
+BuiltWith
+Bytespider
+CSHttp
+ChatGPT-User
+CheeseBot
+CherryPicker
+Cliqzbot
+Crawling at Home Project
+Crescent
 DomCopBot
-DomainSigmaCrawler
+EMail Siphon
 EMail Wolf
-FDM
-GPTBot
-GoZilla
-Iblog
-Indy Library
+Ecxi
+GetRight
+HTMLparser
+Humanlinks
+HybridBot
+Id-search
+IlseBot
 InfoNaviRobot
-InternetSeer
-Iskanie
-JikeSpider
-Jyxobot
-Kozmosbot
-LieBaoFast
-Lightspeedsystems
-Linkbot
+InfoTekies
+Jorgee
+JustView
+Kinza
+LNSpiderguy
+Lanshanbot
+LinkpadBot
 Litemage_walker
-MTRobot
-Magnet
-Majestic-SEO
-Mass Downloader
-Metauri
-Msrabot
-Needle
-Net Vampire
-NextGenSearchBot
-Niki-bot
-Nikto
+MIDown tool
+MIIxpc
+Majestic SEO
+Masscan
+Mediatoolkitbot
+MegaIndex.ru
+Morfeus Fucking Scanner
+Mr.4x3
+Name Intelligence
+NetLyzer
 NimbleCrawler
+Ninja
 Nmap
-Nuclei
+Octopus
+Offline Explorer
 Openfind
-OrangeSpider
-PageAnalyzer
-PageScorer
-Papa Foto
-PeoplePal
-Pi-Monster
-Picscout
+OutfoxBot
+POE-Component-Client-HTTP
+PageThing.com
+Pandalytics
+Picsearch
+PleaseCrawl
 ProWebWalker
-PxBroker
-Quick-Crawler
-RSSingBot
-RankingBot2
+QueryN Metasearch
+RankActive
 Rankivabot
-RankurBot
-RebelMouse
-Recorder
-RepoMonkey
-SEOkicks
-SEOlyticsCrawler
+RedesScrapy
+SEOkicks-Robot
+SEOprofiler
 SEOstats
-SISTRIX
 SalesIntelligent
-Semrush
+SemanticJuice
+SiteLockSpider
 SiteSucker
 Snapbot
-SuperBot
+SocialRankIOBot
+Surfbot
 T0PHackTeam
-TeleportPro
-The Intraformant
-TinyTestBot
-Titan
-Twice
-VB Project
-Vacuum
-WWW-Collector-E
-Web Collage
-WebLeacher
-Webster
-Whacker
-WinHTTrack
-Wotbox
-ZyBorg
-com.plumanalytics
-gopher
-linkdexbot
-lwp-trivial
-magpie-crawler
-mediawords
-seoscanners
-seostar
-sogouspider
-zgrab
+Trendictionbot
+V-BOT
+Voil
+Voltron
+WASALive-Bot
+WISENutbot
+WWW-Mechanize
+WebImageCollector
+WebWhacker
+Whack
+Whatweb
+YoudaoBot
+Zeus
+ZoominfoBot
+archive.org_bot
+crawl.sogou.com
+crawler.feedback
+crawler4j
+demandbase-bot
+facebookscraper
+ips-agent
+lwp-request
+openai.com
+s1z.ru
+sp_auditbot
+webprosbot
+x22Mozilla