V4.2024.07.4654 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-AdsTxtCrawlerTP
-AwarioRssBot
-BDFetch
-BatchFTP
-Bigfoot
-Bitacle
-BlackWidow
-Brandwatch
-BuiltWith
-Bytespider
+360Spider
+ALittle Client
+Aboundex
+AhrefsBot
+Aipbot
+Backlink-Ceck
+CCBot
+CSHttp
 Calculon
-CherryPicker
-Chlooe
+ClaudeBot
+Copyscape
 Cosmos
-Custo
-Dispatch
-DnBCrawler-Analytics
-DomainCrawler
-DomainSigmaCrawler
-Dotbot
-EyeNetIE
-FlashGet
+CrazyWebCrawler
+DSearch
+DTS Agent
+Digincore
+DigitalPebble
+DomCopBot
+Domains Project
+Download Wonder
+EMail Siphon
+EasyDL
+Evil
+Flunky
+Genieo
 GetWeb
-Gigabot
 Go!Zilla
-Grafula
-GrapeshotCrawler
-HEADMasterSEO
-HTTP::Lite
-Haansoft
-Iblog
-Image Sucker
-Jbrofuzz
-LeechFTP
+GoogleOther
+GrapeFX
+GridBot
+Humanlinks
+Image Fetch
+Internet Ninja
+Iria
+Kozmosbot
+LNSpiderguy
 Lftp
-LibWeb
-Libwhisker
-LinkScan
-LinksManager
-Lmspider
-MSIECrawler
-Magnet
+Lightspeedsystems
+LinqiaMetadataDownloaderBot
+Litemage_walker
+MFC_Tear_Sample
+Mail.RU_Bot
 Majestic12
-Masscan
-Metauri
-Mr.4x3
-NearSite
-Nessus
-OrangeBot
+MarkWatch
+Mata Hari
+Mediatoolkitbot
+MegaIndex.ru
+Moblie Safari
+Netcraft
+Netvibes
+Ninja
+Nmap
+OpenLinkProfiler
+Openfind
 OutclicksBot
-OutfoxBot
-PageScorer
-Piepmatz
+Page Analyzer
+Papa Foto
+Pavuk
+Pi-Monster
+PictureFinder
+PleaseCrawl
 Pockey
 ProPowerBot
-Proximic
-PxBroker
-QueryN Metasearch
-RankingBot2
-RankurBot
+ProWebWalker
+Rankivabot
+RebelMouse
 RepoMonkey
-Rogerbot
-Semrush
-SeoSiteCheckup
-SeobilityBot
-Sitevigil
-Snapbot
-SpankBot
-Sqlworm
-Telesphorep
-TinyTestBot
-Titan
-Toweyabot
-Trendictionbot
-Typhoeus
-Voil
-Voltron
-WEBDAV
-WISENutbot
-Web Auto
-Web Collage
-Web Enhancer
-WebEnhancer
-WebReaper
-WebSucker
-Webalta
+Screaming
+Searchestate
+SiteExplorer
+SiteSucker
+SocialRankIOBot
+Spanner
+Spider_Bot
+Sqworm
+Steeler
+SuperBot
+TeleportPro
+True_Robot
+UnisterBot
+VoidEYE
+Wallpapers/3.0
+WebFetch
+Wprecon
+Xaldon WebSpider
+Xenu
+YaK
 Zeus
-arquivo.pt
+ZmEu
+clark-crawler
 crawl.sogou.com
-demandbase-bot
-evc-batch
-facebookscraper
-heritrix
-iubenda-radar
-omgili
-openai
-page scorer
-serpstatbot
-sysscan
-zauba.io
+linkdexbot
+probe-image-size
+satoristudio.net
+siteripz
+spyfu
+tAkeOut
+ubermetrics-technologies.com
+voyagerx.com
+webprosbot