V4.2024.10.4830 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-ADmantX
-Acunetix
-AdsTxtCrawlerTP
-Ankit
-BBBike
-BacklinksExtendedBot
-Barkrowler
-BetaBot
-Bigfoot
-BlowFish
-CheeseBot
-Cogentbot
-CrazyWebCrawler
+Abonti
+AiHitBot
+Backlink-Ceck
+Bandit
+BatchFTP
+Boardreader
+Buddy
+Bullseye
+BunnySlippers
+CheTeam
+Cliqzbot
+Cloud mapping
+Cocolyzebot
+Copyscape
 CrunchBot
-DBLBot
-DIIbot
-DSearch
+Custo
+DigitalPebble
 Dirbuster
-EMail Siphon
-EMail Wolf
-Express WebPictures
-FlashGet
-Getintent
+DnBCrawler-Analytics
+DomainSigmaCrawler
+Dragonfly
+EyeNetIE
+Ezooms
+FacebookBot
+Genieo
 Gigabot
-GrabNet
-HTTP::Lite
-Heritrix
-Iblog
+HTTrack
+Haansoft
+Harvest
 Image Fetch
-InfoNaviRobot
-Intelliseek
-JOC Web Spider
-Kozmosbot
-LinkWalker
+IndeedBot
+InternetSeer
+Jyxobot
+LWP::Simple
+LinqiaScrapeBot
+Litemage_walker
+Lmspider
+MSIECrawler
 MTRobot
-Majestic SEO
-Microsoft Data Access
-Microsoft URL Control
-Morfeus Fucking Scanner
-NimbleCrawler
-Nutch
-Offline Explorer
+Majestic12
+Mass Downloader
+Masscan
+MauiBot
+Mediatoolkitbot
+Musobot
+Navroad
+Nmap
+Nuclei
 OpenLinkProfiler
-PHPCrawl
-PageGrabber
-PageScorer
-Pandalytics
-Petalbot
-Picsearch
-Piepmatz
-Rainbot
-Rankivabot
+PageThing.com
+Picscout
+QueryN Metasearch
+Re-re
+RocketCrawler
 Rogerbot
-SEOprofiler
-SEOstats
-Scrapy
-ScrepyBot
-Shodan
-SiteExplorer
-Sitevigil
+SEOlyticsCrawler
+SalesIntelligent
+ScoutJet
+Semrush
+SemrushBot
+SiteLockSpider
+SiteSucker
 SmartDownload
-Snoopy
-SpaceBison
-SpankBot
-Spbot
-SurveyBot
-Suzuran
-T0PHackTeam
-TightTwatBot
-Trendiction
-Twice
-UnisterBot
-Upflow
-WPScan
+Spammen
+Spider_Bot/3.0
+Telesphorep
+Titan
+TwengaBot
+VelenPublicWebCrawler
+WWWOFFLE
 WallpapersHD
-Web Auto
 Web Collage
-Web Pix
-WebAuto
-WebFuck
-WebPix
-Webster
-Whatweb
-Xenu
-ZoominfoBot
-ZumBot
-clark-crawler
-gopher
-heritrix
-linkdexbot
-magpie-crawler
-meanpathbot
-ripz
-scalaj-http
-serpstatbot
-sexsearcher
-sitechecker.pro
-sogouspider
-tAkeOut
-trendiction.de
+WebBandit
+WebEnhancer
+WebGo IS
+WebImageCollector
+WebSauger
+Webshag
+Xaldon_WebSpider
+YaK
+Zade
+ZmEu
+archive.org_bot
+arquivo-web-crawler
+arquivo.pt
+awario.com
+demandbase-bot
+imagesift.com
+internetVista monitor
+lwp-trivial
+mediawords
+page scorer
+probe-image-size
+s1z.ru
+siteripz
+spyfu
+webgains-bot
+webpros.com
+x22Mozilla
+zgrab