V4.2024.02.4291 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
 01h4x.com
-404checker
-404enemy
 ALittle Client
 AdsTxtCrawlerTP
-Aipbot
-AllSubmitter
-Anthill
-BackStreet
-BatchFTP
-Bitacle
-Bolt
-Brandwatch
-BuiltBotTough
-Bullseye
+Alligator
+Asterias
+Attach
+AwarioRssBot
+BBBike
+BackWeb
+Black Hole
+Buck
+BuiltWith
 CCBot
-ChinaClaw
-Chlooe
-Claritybot
-Cliqzbot
+CensysInspect
 Collector
+Copier
+Copyscape
 CrunchBot
-DnBCrawler-Analytics
-DnyzBot
-DomainStatsBot
-Drip
+Custo
+DittoSpyder
+Dragonfly
+Ecxi
 Exabot
-Extreme Picture Finder
+FDM
+FacebookBot
+FlashGet
 Freeuploader
-GermCrawler
-GetWeb
-Getintent
 Google-Extended
-HMView
-Havij
-IlseBot
+GrabNet
 Image Fetch
-Intelliseek
-Internet Ninja
-IstellaBot
-JetCar
-Leap
-Linkbot
-LinqiaMetadataDownloaderBot
-Lmspider
-Mag-Net
-MauiBot
-MegaIndex.ru
-Mister PiX
-Msrabot
-Musobot
-Navroad
-NearSite
-NetMechanic
-OnCrawl
-Page Analyzer
-Pi-Monster
-Picsearch
-Pockey
-ProWebWalker
+Iria
+Jbrofuzz
+LNSpiderguy
+Larbin
+LeechFTP
+MIDown tool
+MSFrontPage
+MSIECrawler
+MTRobot
+Mb2345Browser
+Mojeek
+Name Intelligence
+Netcraft
+Nutch
+Offline Explorer
+OrangeBot
+Picscout
+PictureFinder
+Pixray
 Proximic
-Pump
-Rankivabot
-ReGet
-Recorder
-SEOprofiler
-ScanAlert
-ScoutJet
+Rainbot
+Re-re
+RebelMouse
+RocketCrawler
+SEOlyticsCrawler
+SalesIntelligent
 Scrapy
-SearchmetricsBot
-Seekport
+Screaming
+SenutoBot
+Site Sucker
 SiteExplorer
-SiteSnagger
-Sitebeam
-SlySearch
-Sogou web spider
-Spammen
-Spbot
+SiteSucker
+SpankBot
+Sqworm
+Stripper
+SuperBot
 Swiftbot
+TeleportPro
+TightTwatBot
+Tracemyfile
+Trendiction
+Twice
 URLy.Warning
-Virusdie
-WEBDAV
-WISENutbot
-WallpapersHD
-WebFetch
-WebLeacher
-WebSucker
-WebWhacker
-Webalta
-Wprecon
-Zitebot
-backlink-check
-cah.io.community
-com.plumanalytics
-linkdexbot
-probe-image-size
-s1z.ru
-scalaj-http
-sysscan
-trendiction.com
+VidibleScraper
+VoidEYE
+Voil
+Voltron
+WPScan
+WWW-Collector-E
+WWWOFFLE
+Wallpapers
+Web Fetch
+Web Fuck
+WebGo IS
+WebsiteQuester
+WiseGuys Robot
+Xenu
+anthropic-ai
+clark-crawler
+cohere-ai
+demandbase-bot
+isitwp.com
+mediawords
+seobility
+seocompany.store
+seostar
+siteripz
+webpros.com
+x22Mozilla
+zauba.io