V4.2024.07.4602 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-360Spider
+01h4x.com
+404checker
+80legs
+ASPSeek
 Acunetix
-Aipbot
+AllSubmitter
 AwarioBot
-BBBike
-Bandit
+AwarioSmartBot
+BackDoorBot
+BetaBot
 Bitacle
 Brandwatch
-Calculon
-Craftbot
-DataCha0s
-DnBCrawler-Analytics
+BuiltBotTough
+Crawling at Home Project
+CrunchBot
+DatabaseDriverMysqli
+DigitalPebble
+DomCopBot
+DomainStatsBot
 Dotbot
-EMail Wolf
-EasyDL
-EyeNetIE
-FDM
-FyberSpider
-Gigabot
-Google-Extended
-Intelliseek
-InterGET
-Internet Ninja
-Iria
-JetCar
-Jyxobot
-Kinza
-Larbin
-Libwhisker
-LinkextractorPro
-LinqiaRSSBot
-MIIxpc
+Ebingbong
+Ecxi
+Extreme Picture Finder
+Foobot
+FrontPage
+GetRight
+Go-Ahead-Got-It
+HEADMasterSEO
+HTMLparser
+Image Fetch
+Kenjin Spider
+Keybot Translation-Search-Machine
+Leap
+Likse
+MSIECrawler
 MTRobot
 Mag-Net
 Mass Downloader
-MicroMessenger
-Microsoft URL Control
-NICErsPRO
-Navroad
-NetAnts
-OutclicksBot
-POE-Component-Client-HTTP
-PageGrabber
-Pandalytics
-Piepmatz
-PxBroker
-RSSingBot
-Rainbot
-RankurBot
+Mata Hari
+Mojeek
+Nameprotect
+Netvibes
+Nibbler
+Nikto
+Offline Navigator
+OnCrawl
+PECL::HTTP
+Page Analyzer
+Pi-Monster
+ProPowerBot
+Psbot
+PyCurl
+QueryN Metasearch
+Quick-Crawler
 Re-re
-Rogerbot
-SEOprofiler
-Scanbot
-ScoutJet
-Scrapy
+Reaper
+Recorder
+SEOkicks
+SEOkicks-Robot
+SEOlyticsCrawler
 SearchmetricsBot
-Semrush
-SentiBot
-Snapbot
-Sogou web spider
-Sottopop
+SemrushBot
+SenutoBot
+SiteExplorer
+SiteSucker
+Sqworm
 Stripper
-Sucuri
-Thumbor
-Toata
-Toweyabot
-Turingos
-TurnitinBot
-Typhoeus
-Voltron
-WWW::Mechanize
-Web Sucker
-WebBandit
-WebCollage
-WebGo IS
-WebPix
-WebReaper
-WebSucker
-Webshag
-WebsiteExtractor
+Surfbot
+SurveyBot
+Suzuran
+T0PHackTeam
+T8Abot
+TeleportPro
+Telesphoreo
+Telesphorep
+TheNomad
+VCI
+VelenPublicWebCrawler
+WASALive-Bot
+WWW-Mechanize
+Web Collage
+Web Fetch
+WebAuto
+WebFuck
+WebSauger
+Webster
+Whacker
 Whatweb
-WinHTTrack
-Wonderbot
-Woobot
-Xaldon_WebSpider
-YoudaoBot
+Wotbox
+YaK
 ZoomBot
-anthropic-ai
-check1.exe
-cohere-ai
-crawler4j
-dataforseobot
-domainsproject.org
-eCatch
-magpie-crawler
-s1z.ru
-seoscanners
-sitechecker.pro
-tAkeOut
-trendiction.com
+autoemailspider
+clark-crawler
+crawl.sogou.com
+crawler.feedback
+netEstate NE Crawler
+seostar
+voyagerx.com
+x09Mozilla
+zgrab