V4.2024.04.4423 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-404checker
-AhrefsBot
-AiHitBot
-Alexibot
-Anarchie
-Atomseobot
-BackStreet
-Barkrowler
+AdsTxtCrawlerTP
+AfD-Verbotsverfahren
+AwarioRssBot
+BackDoorBot
+Battleztar Bazinga
 Blow
-BotALot
-BuiltWith
-BunnySlippers
-Cocolyzebot
+Bullseye
+ChatGPT-User
+Chlooe
+Cloud mapping
+Collector
+CopyRightCheck
+Cosmos
+CrazyWebCrawler
+Crescent
+CrunchBot
+DBLBot
 Disco
-DnBCrawler-Analytics
-Ecxi
-ExtractorPro
+DomainStatsBot
+EirGrabber
+EroCrawler
+Exabot
 Fimap
-Foobot
+Firefox/7.0
+Flunky
 Fyrebot
-GPTBot
+GalaxyBot
 GetRight
 Gigabot
+GrabNet
 Grabber
-GrapeshotCrawler
 GridBot
-HTMLparser
 Harvest
-IDBot
+Image Fetch
 InterGET
-Jorgee
-JustView
-Jyxobot
-Kozmosbot
-Linkbot
-Ltx71
-Magnet
-Mail.RU_Bot
-MarkMonitor
-Masscan
-MauiBot
-Metauri
-Msrabot
-Nikto
+InternetSeer
+JOC Web Spider
+LeechGet
+Likse
+LinksManager
+Mag-Net
+Majestic-SEO
+Mata Hari
+MeanPath Bot
+MegaIndex.ru
+Minefield
+Nameprotect
+Ninja
+Nuclei
 Octopus
-OnCrawl
-OpenVAS
-OutclicksBot
-Picscout
-Pimonster
+OutfoxBot
+PageGrabber
+Pavuk
+Petalbot
+Pi-Monster
 Proximic
-Pump
+Psbot
+Quick-Crawler
+Re-re
 RealDownload
-RebelMouse
-RedesScrapy
+RocketCrawler
 SEOstats
-SISTRIX
-SalesIntelligent
-ScreenerBot
-Seekport
-Shodan
-SlySearch
-Sociscraper
+Siphon
 Sogou web spider
-SuperHTTP
-Surfbot
-SurveyBot
-True_Robot
-TurnitinBot
-TwengaBot
+Sottopop
+Spinn3r
+Sqworm
+Sucker
+Suzuran
+Swiftbot
+Teleport
+Turnitin
 Twice
-Upflow
-V-BOT
-VeriCiteCrawler
+Typhoeus
 Virusdie
-Voltron
-Wallpapers
-WebAuto
-WebSauger
-WebStripper
-WiseGuys Robot
-Wotbox
-YoudaoBot
-Zade
-Zitebot
-ZmEu
-ZoomBot
-anthropic-ai
-coccocbot
-domainsproject.org
-heritrix
-instabid
-magpie-crawler
-scalaj-http
-scan.lol
-seoscanners
-seostar
-sexsearcher
-sitechecker.pro
+Web Collage
+Web Fuck
+WebSucker
+WebWhacker
+WebmasterWorldForumBot
+Webster
+Woobot
+Xaldon WebSpider
+Xenu
+Zeus
+ZoominfoBot
+arquivo-web-crawler
+cohere-ai
+crawler.feedback
+ips-agent
+iubenda-radar
+muhstik-scan
+openai.com
+page scorer
+seocompany.store
 spyfu
+sysscan
+trendiction.de
+ubermetrics-technologies.com
+webmeup-crawler