V4.2024.07.4655 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-360Spider
-ALittle Client
-Aboundex
-AhrefsBot
-Aipbot
-Backlink-Ceck
-CCBot
-CSHttp
-Calculon
-ClaudeBot
+80legs
+ADmantX
+Aspiegel
+Asterias
+BLEXBot
+BackWeb
+BetaBot
+Boardreader
+Brandprotect
+Buck
+Bytespider
 Copyscape
-Cosmos
-CrazyWebCrawler
-DSearch
-DTS Agent
-Digincore
-DigitalPebble
+Crawling at Home Project
+Disco
+Discoverybot
+DittoSpyder
 DomCopBot
-Domains Project
-Download Wonder
-EMail Siphon
-EasyDL
-Evil
+Dragonfly
+ECCP/1.0
+EirGrabber
+Extreme Picture Finder
+FacebookBot
+FemtosearchBot
 Flunky
-Genieo
-GetWeb
-Go!Zilla
-GoogleOther
-GrapeFX
-GridBot
-Humanlinks
-Image Fetch
-Internet Ninja
+Freeuploader
+Fuzz
+G-i-g-a-b-o-t
+Getintent
+Google-Extended
+IDBTE4M
+InfraSec Scanner
 Iria
-Kozmosbot
-LNSpiderguy
-Lftp
-Lightspeedsystems
-LinqiaMetadataDownloaderBot
+JOC Web Spider
+Jbrofuzz
+JikeSpider
+Jyxobot
+Kenjin Spider
+Likse
+LinkextractorPro
 Litemage_walker
-MFC_Tear_Sample
-Mail.RU_Bot
-Majestic12
+MIDown tool
+Mag-Net
+MarkMonitor
 MarkWatch
-Mata Hari
+Masscan
+MeanPath Bot
 Mediatoolkitbot
 MegaIndex.ru
-Moblie Safari
-Netcraft
-Netvibes
-Ninja
-Nmap
-OpenLinkProfiler
-Openfind
-OutclicksBot
-Page Analyzer
-Papa Foto
-Pavuk
-Pi-Monster
+Mister PiX
+Mojeek
+Name Intelligence
+Nameprotect
+Nikto
+PHPCrawl
+Pandalytics
 PictureFinder
-PleaseCrawl
-Pockey
-ProPowerBot
-ProWebWalker
-Rankivabot
-RebelMouse
-RepoMonkey
-Screaming
-Searchestate
+PyCurl
+RankingBot
+RankingBot2
+Reaper
+SMTBot
+ScanAlert
+Scrapy
+SemanticJuice
+Site Sucker
+SiteCheckerBotCrawler
 SiteExplorer
-SiteSucker
-SocialRankIOBot
+Siteimprove
+SlySearch
 Spanner
-Spider_Bot
-Sqworm
-Steeler
-SuperBot
-TeleportPro
-True_Robot
-UnisterBot
-VoidEYE
+Stripper
+Sucker
+SuperHTTP
+Teleport
+Telesoft
+TinyTestBot
+Trendiction
+URLy.Warning
+WWW-Collector-E
+WWWOFFLE
 Wallpapers/3.0
-WebFetch
-Wprecon
-Xaldon WebSpider
-Xenu
-YaK
+WallpapersHD
+Web Collage
+Web Sucker
+WebGo IS
+WebSauger
+Webalta
+Whacker
 Zeus
-ZmEu
-clark-crawler
-crawl.sogou.com
-linkdexbot
+cah.io.community
+crawler4j
+facebookscraper
+iubenda-radar
+netEstate NE Crawler
+polaris version
 probe-image-size
-satoristudio.net
+serpstatbot
 siteripz
-spyfu
-tAkeOut
-ubermetrics-technologies.com
-voyagerx.com
-webprosbot
+x09Mozilla
+zauba.io