V4.2024.01.4278 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-404enemy
-ADmantX
-BBBike
-Black Hole
-Buddy
-CSHttp
-Copier
-Crawling at Home Project
-DIIbot
+404checker
+ASPSeek
+Aboundexbot
+AdsTxtCrawlerTP
+Anarchy99
+Anthill
+Asterias
+Bandit
+Bitacle
+Bolt
+CheeseBot
+CopyRightCheck
+Craftbot
+DBLBot
+DatabaseDriverMysqli
 Deusu
-Dirbuster
-DnyzBot
-Download Wonder
-Drip
-Extreme Picture Finder
-Ezooms
-FyberSpider
-GalaxyBot
-GetWeb
-GrapeFX
-HMView
-HTTP::Lite
+Dragonfly
+Ecxi
+Exabot
+FemtosearchBot
+Flunky
+GPTBot
+GT::WWW
+GridBot
+HTMLparser
+HTTrack
+Havij
+HonoluluBot
 IRLbot
-IlseBot
-IndeedBot
-InterGET
-Iskanie
-JennyBot
-Jetty
-JikeSpider
-Kozmosbot
+Indy Library
+JamesBOT
 LNSpiderguy
-Linkbot
-Masscan
-Mediatoolkitbot
+Larbin
+Litemage_walker
+MIDown tool
+Majestic-SEO
+MarkMonitor
+MarkWatch
+Mass Downloader
+Mb2345Browser
 MegaIndex.ru
-Metauri
-Mister PiX
-Mojeek
-Name Intelligence
+Microsoft Data Access
+Microsoft URL Control
+Minefield
+Moblie Safari
+Mojolicious
+Mr.4x3
+NPbot
 NearSite
-NetZIP
 Netcraft
-Nutch
-Offline Navigator
-OpenVAS
-OrangeBot
-OutclicksBot
-POE-Component-Client-HTTP
+Ninja
+Nuclei
+OnCrawl
+OrangeSpider
 Pandalytics
-Pixray
-Psbot
-Pu_iN
-Pump
+Piepmatz
+ProWebWalker
+Probethenet
+Proximic
 PyCurl
 QueryN Metasearch
-SMTBot
-Screaming
-ScrepyBot
+Quick-Crawler
+RankActiveLinkBot
+SBIder
+ScoutJet
+Scrapy
+ScreenerBot
+SentiBot
+SiteLockSpider
+SiteSnagger
+SiteSucker
 Sitebeam
 SmartDownload
-Snake
 Sociscraper
-SpaceBison
-Spammen
-Spanner
-Stripper
-Surfbot
-SurveyBot
-Telesoft
-The Intraformant
-TightTwatBot
-Trendiction
+Sogou web spider
+Sqlmap
+Steeler
+Sucker
+Swiftbot
+Telesphoreo
+Toata
+Trendictionbot
+Typhoeus
 URLy.Warning
+VCI
 WPScan
-Wallpapers
-Wallpapers/3.0
-WallpapersHD
-Web Collage
-Web Enhancer
-Web Fetch
-Web Sauger
-WebBandit
-WebsiteExtractor
-Xaldon WebSpider
-Xenu
-Zade
-Zauba
-Zeus
-autoemailspider
-coccocbot
-cohere-ai
-instabid
-lwp-request
-magpie-crawler
-netEstate NE Crawler
-ripz
-sogouspider
-spyfu
-x09Mozilla
+WWW::Mechanize
+Web Pix
+WebAuto
+WebImageCollector
+WebLeacher
+Whacker
+YoudaoBot
+archive.org_bot
+meanpathbot
+openai
+seoscanners
+siteripz
+sysscan
+zgrab