V4.2024.09.4743 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-Aipbot
-Alligator
-Anarchy
-Attach
-BlowFish
-BunnySlippers
-Calculon
-Citoid
+01h4x.com
+360Spider
+404checker
+Alexibot
+BLEXBot
+BackWeb
+Bandit
+Blackboard
+Brandwatch
 Claritybot
-Copier
-Crawling at Home Project
-CrazyWebCrawler
+Cliqzbot
+Collector
 CyotekWebCopy
-DigitalPebble
-DomainStatsBot
-Dragonfly
-Drip
-EroCrawler
-Exabot
-ExtractorPro
-FHscan
+Demon
+Devil
+Discobot
+DittoSpyder
+DnBCrawler-Analytics
+DnyzBot
+DomCopBot
+Dotbot
+EMail Wolf
+Ebingbong
+Ecxi
+Express WebPictures
+Ezooms
+Foobot
+G-i-g-a-b-o-t
 GermCrawler
-Go-Ahead-Got-It
-Gotit
-Haansoft
-Havij
-Heritrix
-IRLbot
-Image Sucker
-IndeedBot
-Iria
-Jorgee
-Keyword Density
-LibWeb
-Lightspeedsystems
-Lipperhey
-MIDown tool
-MIIxpc
-MSFrontPage
-Majestic12
-MarkMonitor
-Masscan
-MegaIndex.ru
-Mojolicious
-NICErsPRO
+GetRight
+Gigabot
+HaosouSpider
+Humanlinks
+HybridBot
+Iblog
+IlseBot
+JOC Web Spider
+JamesBOT
+Kozmosbot
+MSIECrawler
+MauiBot
+Microsoft URL Control
+Msrabot
+Navroad
+Nessus
 NetLyzer
-OpenVAS
-Page Analyzer
-Papa Foto
-Pi-Monster
+Netcraft
+Niki-bot
+Ninja
+Piepmatz
 PleaseCrawl
-ProPowerBot
-Pu_iN
-Rainbot
-RealDownload
-Reaper
-RebelMouse
-RepoMonkey
-SEOlyt
-SEOlyticsCrawler
-SEOprofiler
-SalesIntelligent
-ScreenerBot
-Searchestate
-Seekport
-SenutoBot
+Psbot
+PyCurl
+RankFlex
+ReGet
+SeekportBot
+Semrush
 SeoSiteCheckup
-SeobilityBot
-Site Sucker
-Siteimprove
-SocialRankIOBot
-SpaceBison
-Spammen
-Spbot
-Spider_Bot
-SuperHTTP
+SiteSucker
+SmartDownload
+Sosospider
+Sqlworm
+Surfbot
+Teleport
+TheNomad
+Turnitin
+TurnitinBot
+Typhoeus
+UnisterBot
 Vagabondo
-WWW-Collector-E
+VoidEYE
+WWW::Mechanize
 WWWOFFLE
 Wallpapers
-WebFuck
-WebGo IS
-WebWhacker
-Webalta
-WebsiteQuester
-Whacker
-WinHTTrack
+WebCollage
+WebImageCollector
+WebPix
+WebSucker
+Whatweb
 WiseGuys Robot
-ZyBorg
-autoemailspider
-com.plumanalytics
-crawl.sogou.com
+YaK
+Zermelo
+Zeus
+Zitebot
+ZmEu
+ZumBot
+arquivo-web-crawler
+awario.com
 crawler4j
-lwp-request
-mediawords
-openai.com
-pcBrowser
-probe-image-size
-seocompany.store
-webgains-bot
+instabid
+meanpathbot
+muhstik-scan
+page scorer
+polaris version
+seoscanners
+seostar
+serpstatbot
+spyfu
+xpymep1.exe
+zgrab