V4.2023.12.4162 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,100 +1,100 @@
-AIBOT
-Aipbot
-Anarchie
-Asterias
-Attach
-BackDoorBot
-BacklinkCrawler
-Badass
-Bandit
-BlowFish
-BuiltBotTough
-CCBot
-CSHttp
-Claritybot
-Cloud mapping
-Copier
-Crawling at Home Project
-DIIbot
-Demon
-DittoSpyder
-EMail Wolf
-Exabot
+ALittle Client
+AiHitBot
+Aspiegel
+AwarioSmartBot
+BBBike
+BatchFTP
+Bullseye
+CherryPicker
+Cogentbot
+Copyscape
+Crescent
+CrunchBot
+DigitalPebble
+Discobot
+Domains Project
+ExtractorPro
 FemtosearchBot
-Foobot
-GetRight
-Havij
-Intelliseek
-IstellaBot
-JamesBOT
+FrontPage
+Gigabot
+GrabNet
+Grafula
+Haansoft
+Image Fetch
+Image Sucker
+JOC Web Spider
+JennyBot
+Joomla
+Jyxobot
 Keybot Translation-Search-Machine
-Keyword Density
-Larbin
-LinqiaRSSBot
-Lipperhey Spider
+Lanshanbot
+LinqiaMetadataDownloaderBot
 Lmspider
-Mail.RU_Bot
-MarkWatch
-Mass Downloader
-MeanPath Bot
+Metauri
+Minefield
 Morfeus Fucking Scanner
-Mozlila
-NICErsPRO
-Nameprotect
-Nettrack
-NimbleCrawler
+Name Intelligence
+Navroad
+Net Vampire
+NetAnts
+NetLyzer
 Ninja
-Offline Navigator
-OpenLinkProfiler
-Openvas
-OrangeBot
-OrangeSpider
-PageGrabber
-PageScorer
+OpenVAS
+Openfind
+Page Analyzer
 Pandalytics
-Papa Foto
-Pavuk
-QueryN Metasearch
-RankingBot
-Re-re
+PeoplePal
+Petalbot
+Picsearch
+Piepmatz
+ProPowerBot
+Pump
+PyCurl
+Quick-Crawler
+RedesScrapy
+Ripper
 SBIder
-SEOlyticsCrawler
 SentiBot
-SiteExplorer
-SiteSnagger
-SiteSucker
+Seomoz
+SiteCheckerBotCrawler
+Sitebeam
+Siteimprove
+Sociscraper
+Sogou web spider
 Spammen
-Spanner
+SputnikBot
 SuperHTTP
-T0PHackTeam
-Telesoft
-TinyTestBot
-Toata
-Toweyabot
-Trendictionbot
-True_Robot
-Turingos
-Upflow
-VeriCiteCrawler
-Virusdie
+TheNomad
+Turnitin
+URLy.Warning
 WBSearchBot
 WWW-Collector-E
+Wallpapers
 Web Enhancer
-WebCollage
-WebStripper
-Webshag
-Whacker
-Wonderbot
-Zitebot
-ZyBorg
-arquivo-web-crawler
-autoemailspider
-cognitiveseo
-crawl.sogou.com
-ips-agent
-muhstik-scan
-scalaj-http
+WebAuto
+WebReaper
+WebSucker
+WebWhacker
+Whack
+Widow
+Zade
+Zeus
+ZmEu
+ZoomBot
+adscanner
+crawler4j
+dataforseo.com
+gopher
+lwp-trivial
+netEstate NE Crawler
+openai
+pcBrowser
+plumanalytics
+polaris version
+seostar
 sp_auditbot
+spyfu
+ubermetrics-technologies.com
+voyagerx.com
 webgains-bot
 webmeup-crawler
-zgrab