V4.2023.05.3757 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-360Spider
-AhrefsBot
+ASPSeek
+Abonti
+AfD-Verbotsverfahren
 Ankit
-Anthill
-AwarioSmartBot
-Battleztar Bazinga
-Blackboard
-BuiltBotTough
-Bullseye
-BuzzSumo
-CCBot
-CSHttp
-Calculon
-Copier
-Curious
-Deusu
-Devil
-DigitalPebble
+Attach
+BDCbot
+Barkrowler
+Buck
+BunnySlippers
+CODE87
+CheeseBot
+ChinaClaw
+Claritybot
+Collector
+CopyRightCheck
+Digincore
 Discoverybot
-DomainAppender
+DnBCrawler-Analytics
+DomainCrawler
 Download Wonder
-Dragonfly
-EMail Wolf
+EroCrawler
 Extractor
-FlashGet
-FrontPage
-Grabber
-HTTP::Lite
+EyeNetIE
+Foobot
+Grafula
+GridBot
 Haansoft
 Harvest
-Indy Library
-InterGET
-Iskanie
-Keybot Translation-Search-Machine
-LinkScan
-LinqiaRSSBot
-Lmspider
+HybridBot
+IRLbot
+Intelliseek
+JennyBot
+Jorgee
+LieBaoFast
+LinkpadBot
+LinqiaScrapeBot
 MJ12bot
-MQQBrowser
-MeanPath Bot
-Mojeek
-Needle
-Netcraft
+MSFrontPage
+Majestic SEO
+Majestic-SEO
+Majestic12
+Mister PiX
+Moblie Safari
+Morfeus Fucking Scanner
+NPbot
+Net Vampire
+NetMechanic
 NextGenSearchBot
-NimbleCrawler
-OpenVAS
+Nuclei
+Nutch
 Openvas
-OrangeBot
-Page Analyzer
 Pandalytics
-Panscient
-Papa Foto
-Picsearch
-Pixray
+Pavuk
 Proximic
-Pump
-RankFlex
-RebelMouse
-Recorder
-RedesScrapy
-Ripper
-Screaming
+Pu_iN
+RSSingBot
+Re-re
+ReGet
+RocketCrawler
+ScrepyBot
 Searchestate
-Seekport
-SenutoBot
-SeobilityBot
-SiteSucker
-Stripper
-Sucuri
+SeekportBot
+Site Sucker
+SiteLockSpider
+Sitebeam
+SmartDownload
+Snake
+Sottopop
+SpaceBison
+SuperHTTP
 SurveyBot
-Suzuran
-T8Abot
 Teleport
-TeleportPro
-Telesphoreo
+TheNomad
+Thumbor
+Toata
+Tracemyfile
+Trendiction
+Trendictionbot
 True_Robot
-V-BOT
-VelenPublicWebCrawler
-Virusdie
-WWW-Mechanize
+TurnitinBot
+Twice
+URLy Warning
+WPScan
+WallpapersHD
+Web Enhancer
 Web Fetch
-WebCopier
-WebGo IS
-WebImageCollector
-WebStripper
-WebSucker
+Web Pix
 Whacker
-Whatweb
-Zade
-Zauba
-coccocbot
-iubenda-radar
-lwp-trivial
+Woobot
+Xaldon WebSpider
+ZmEu
+adscanner
+autoemailspider
+clark-crawler
+crawler4j
+heritrix
 meanpathbot
-mediawords
-muhstik-scan
-netEstate NE Crawler
-seocompany.store
-seostar
-sogouspider
+scalaj-http
+siteripz
+webgains-bot