V4.2024.01.4269 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-ADmantX
-AdsTxtCrawlerTP
-Alexibot
-Bigfoot
-BlowFish
+404enemy
+Aboundex
+Alligator
+Anthill
+AwarioSmartBot
+Bandit
+BlackWidow
+Boardreader
+BotALot
 BuiltBotTough
+BunnySlippers
 CATExplorador
-CCBot
-Cogentbot
-CrazyWebCrawler
-Devil
-Digincore
+Calculon
+CazoodleBot
+ChinaClaw
+Cliqzbot
+Crawling at Home Project
+Curious
+DBLBot
+DatabaseDriverMysqli
 Discobot
-DomainStatsBot
-EasyDL
+DomCopBot
+EMail Siphon
+EMail Wolf
 ExtLinksBot
-FemtosearchBot
-Getintent
-Go!Zilla
-GrabNet
-HTMLparser
-HTTP::Lite
-Haansoft
-Heritrix
+ExtractorPro
+FHscan
+Fimap
+FlashGet
+Flunky
+Fyrebot
+IDBTE4M
 IDBot
-InfoTekies
-InterGET
-Internet Ninja
-Iria
-IstellaBot
-Keyword Density
-Kinza
-Kozmosbot
-LWP::Simple
+Iblog
+Id-search
 Larbin
-LeechFTP
-Lightspeedsystems
-LinksManager
+LinkextractorPro
 Lipperhey
-MJ12bot
-MQQBrowser
-Majestic SEO
-MarkWatch
-Minefield
-Moblie Safari
-Morfeus Fucking Scanner
-Name Intelligence
-Nessus
-Net Vampire
-Nibbler
-Nikto
+MFC_Tear_Sample
+MIDown tool
+MSFrontPage
+MarkMonitor
+Microsoft Data Access
+Mister PiX
+MolokaiBot
+Mozlila
+NPbot
+Nimbostratus
 Nmap
-OrangeSpider
-Papa Foto
-Picsearch
-Probethenet
-QueryN Metasearch
-Rainbot
-ReGet
-SEOkicks-Robot
+OnCrawl
+OpenLinkProfiler
+PageScorer
+RankingBot
+Re-re
+RedesScrapy
+RepoMonkey
+Rogerbot
 SEOprofiler
-SEOstats
-SearchmetricsBot
-Siphon
-SiteLockSpider
-Sociscraper
+SalesIntelligent
+SemanticJuice
+SemrushBot
+SentiBot
+Sitebeam
+SmartDownload
+Spanner
 SurveyBot
 Szukacz
-The Intraformant
-Thumbor
-TurnitinBot
-URLy Warning
-VCI
-Vagabondo
-WallpapersHD
-WebCollage
-WebEnhancer
+T8Abot
+Telesoft
+Turnitin
+VidibleScraper
+WEBDAV
+WWW::Mechanize
+WWWOFFLE
+Wallpapers
+Web Auto
+WebAuto
+WebBandit
+WebCopier
 WebFetch
-WebWhacker
-Whatweb
-Xenu
-Zitebot
-ZmEu
-ZumBot
-backlink-check
-coccocbot
+WebImageCollector
+WebSauger
+WebsiteQuester
+Wonderbot
+Xaldon_WebSpider
+Zade
+clark-crawler
+com.plumanalytics
 crawler.feedback
-demandbase-bot
-instabid
-iubenda-radar
-meanpathbot
-oBot
-plumanalytics
-probe-image-size
+crawler4j
+eCatch
+lwp-trivial
+mediawords
+openai.com
 ripz
-s1z.ru
-scalaj-http
-scan.lol
 seoscanners
-x09Mozilla
+spyfu
+webpros.com
+xpymep1.exe
+zauba.io