V4.2024.05.4531 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-80legs
-Abonti
-Aipbot
-Alexibot
-BDCbot
-BackDoorBot
-BackStreet
-Bigfoot
-Bolt
-BotALot
-Brandprotect
-CherryPicker
+01h4x.com
+AiHitBot
+Anarchy99
+Ankit
+BLEXBot
+Barkrowler
+Blackboard
+Buddy
+BuzzSumo
 Cliqzbot
-Cloud mapping
-Copyscape
-Cosmos
 CrazyWebCrawler
-Custo
-Extreme Picture Finder
-FlashGet
-Freeuploader
-Getintent
-Grabber
-HMView
-Haansoft
-HonoluluBot
-Humanlinks
+Crescent
+CrunchBot
+Devil
+Discobot
+DittoSpyder
+DomainStatsBot
+Download Wonder
+EMail Siphon
+EroCrawler
+FacebookBot
+FemtosearchBot
+Flunky
+Fyrebot
+GrapeshotCrawler
+Heritrix
 HybridBot
-IlseBot
-Image Sucker
-InterGET
+IDBTE4M
+Id-search
+InternetMeasurement
+Jbrofuzz
 Jyxobot
-Kenjin Spider
-Lanshanbot
-Mag-Net
-Mail.RU_Bot
-Majestic-SEO
-Mata Hari
-Mister PiX
-Navroad
-NearSite
-Nessus
-NetAnts
+LeechGet
+Linkbot
+Lmspider
+Ltx71
+MFC_Tear_Sample
+MSIECrawler
+MTRobot
+Magnet
+MarkMonitor
+Masscan
+Microsoft URL Control
+MolokaiBot
+Mr.4x3
+Nameprotect
 NetZIP
 Ninja
+OrangeBot
 OrangeSpider
-PECL::HTTP
-PHPCrawl
-PageThing.com
-Petalbot
-Piepmatz
-Pixray
+PageScorer
+Panscient
+Pockey
 PyCurl
+Quick-Crawler
 Rainbot
-RealDownload
+RankActive
+Rankivabot
+ReGet
+Reaper
+Recorder
 RepoMonkey
-RocketCrawler
 SEOlyticsCrawler
-SalesIntelligent
-Searchestate
-SearchmetricsBot
-SlySearch
+SeekportBot
+Semrush
+Seomoz
+Siphon
+SiteLockSpider
 Sociscraper
-Sogou web spider
-SpaceBison
-Spinn3r
-Steeler
+Sottopop
+Spbot
 Stripper
-SurveyBot
-Teleport
-TheNomad
-Thumbor
-Titan
-Turnitin
+Sucker
+Szukacz
+T0PHackTeam
+True_Robot
+TurnitinBot
 Voltron
-WEBDAV
-WISENutbot
-WebFuck
-WebsiteQuester
-Webster
-Who.is Bot
-Wonderbot
-Wprecon
-Xenu
-YoudaoBot
+WBSearchBot
+Wallpapers
+WebBandit
+WebReaper
+WebmasterWorldForumBot
+Xaldon WebSpider
+Xaldon_WebSpider
+Zade
+Zermelo
 Zeus
-adscanner
-anthropic-ai
-cognitiveseo
-demandbase-bot
-domainsproject.org
-imagesift.com
-isitwp.com
-linkdexbot
-polaris version
+arquivo-web-crawler
+clark-crawler
+magpie-crawler
+oBot
+pcBrowser
+scan.lol
 seobility
-tAkeOut
-ubermetrics-technologies.com
-x22Mozilla
+serpstatbot
+sitechecker.pro
+x09Mozilla
 zauba.io
+zgrab