V4.2023.10.4058 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-01h4x.com
 80legs
 ASPSeek
 AdsTxtCrawlerTP
 Alexibot
-Apexoo
-Atomseobot
-AwarioSmartBot
-BackStreet
-BacklinkCrawler
-BuiltBotTough
-Calculon
-CheeseBot
-Collector
-Craftbot
-CrunchBot
-DIIbot
-DataCha0s
+Asterias
+Boardreader
+Bolt
+BotALot
+Brandwatch
+CheTeam
+Cocolyzebot
+Demon
 Digincore
+DigitalPebble
+Discobot
+Dispatch
+DnyzBot
+DomainAppender
+Download Wonder
+EMail Wolf
+EasyDL
+Ecxi
 Firefox/7.0
-Flunky
-FrontPage
+Foobot
 Go!Zilla
-Grabber
-Haansoft
-Harvest
-IDBTE4M
-IRLbot
-Indy Library
-Intelliseek
+Go-Ahead-Got-It
+GrapeshotCrawler
+HEADMasterSEO
+HTMLparser
+HTTP::Lite
+Havij
+Iblog
+IndeedBot
 InternetSeer
 Iskanie
-JikeSpider
-Jorgee
-Jyxobot
-Keyword Density
-LNSpiderguy
-Likse
+JustView
+Keybot Translation-Search-Machine
+Leap
+LeechGet
+LexiBot
+LinkWalker
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
+Lmspider
+MIDown tool
 MSFrontPage
-MTRobot
-Magnet
-Microsoft Data Access
-Morfeus Fucking Scanner
-Net Vampire
-NetAnts
-Nuclei
-Nutch
-Offline Navigator
-OpenVAS
-PHPCrawl
-Pandalytics
-Pimonster
-Probethenet
-Proximic
-Pu_iN
-QueryN Metasearch
-Reaper
-RebelMouse
-Rogerbot
-SEOlyticsCrawler
-Scrapy
-Screaming
-Semrush
-SemrushBot
-Shodan
-SiteLockSpider
-Sitevigil
-SocialRankIOBot
-Sociscraper
-SpaceBison
-SputnikBot
-Sqlmap
-Sucuri
+MSIECrawler
+Mister PiX
+MolokaiBot
+Musobot
+Needle
+Netcraft
+Nibbler
+Page Analyzer
+Papa Foto
+Piepmatz
+ProPowerBot
+RankActive
+RankingBot
+RankingBot2
+ReGet
+SBIder
+SeekportBot
+SemanticJuice
+SentiBot
+Seomoz
+Siteimprove
+Sogou web spider
+Sottopop
+Steeler
 SuperBot
-TightTwatBot
-Toweyabot
-Trendiction
-Turingos
-TurnitinBot
-URLy.Warning
-WWW::Mechanize
-Web Sucker
-WebFetch
-WebFuck
-WebSauger
-WebmasterWorldForumBot
-Zitebot
-ZoomBot
-check1.exe
-com.plumanalytics
-gopher
-iubenda-radar
-linkdexbot
-pcBrowser
-scalaj-http
-sysscan
+Teleport
+Telesoft
+TheNomad
+Trendictionbot
+Twice
+Vacuum
+Vagabondo
+WBSearchBot
+Web Auto
+WebAuto
+WebEnhancer
+WebSucker
+Website Quester
+Webster
+WinHTTrack
+Wotbox
+Xaldon WebSpider
+Xenu
+adscanner
+arquivo.pt
+eCatch
+facebookscraper
+lwp-request
+meanpathbot
+seoscanners
+seostar
 voyagerx.com
-webgains-bot
-webmeup-crawler
-webprosbot
+zauba.io
+zgrab