V4.2021.12.2800 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 404checker
-80legs
 AIBOT
 ALittle Client
-Abonti
-Aboundex
-Acunetix
+ASPSeek
 AhrefsBot
+AllSubmitter
 Alligator
 AlphaBot
-Anarchy
 Anthill
-Apexoo
 Aspiegel
+Asterias
 Attach
-AwarioRssBot
-BLEXBot
-BackStreet
-Badass
+BackDoorBot
+Bandit
 Barkrowler
+BatchFTP
 Battleztar Bazinga
+BetaBot
 Bigfoot
-Blackboard
+Boardreader
 Bolt
-BotALot
-Brandwatch
-Buck
 Buddy
 BuiltWith
-Bullseye
 BunnySlippers
-Calculon
+BuzzSumo
+CCBot
+CSHttp
 CazoodleBot
-Cegbfeieh
-ChinaClaw
-Chlooe
+CheeseBot
+CherryPicker
 Claritybot
 Cloud mapping
-Copier
+Cocolyzebot
+Cogentbot
 Copyscape
+Cosmos
 Craftbot
-CrazyWebCrawler
 CyotekWebCopy
 DSearch
 DTS Agent
 DataCha0s
-Dirbuster
 Disco
+Discoverybot
 Dispatch
-DnBCrawler-Analytics
-DomainSigmaCrawler
-Dotbot
-ECCP/1.0
-EMail Wolf
+DittoSpyder
+DomCopBot
+Dragonfly
+Drip
 EasyDL
-Ebingbong
+Ecxi
 EirGrabber
 EroCrawler
-Express WebPictures
+ExtLinksBot
+FDM
+FHscan
+FemtosearchBot
 Fimap
-Firefox/7.0
 Flunky
 Freeuploader
 FrontPage
-Fuzz
-FyberSpider
-Fyrebot
+G-i-g-a-b-o-t
 GT::WWW
 GalaxyBot
-GetWeb
-Getintent
-Gigabot
-Go!Zilla
+GermCrawler
+GetRight
 Go-Ahead-Got-It
 GoZilla
-GrabNet
-GrapeshotCrawler
-HEADMasterSEO
+HTTP::Lite
 Haansoft
-HaosouSpider
-Harvest
+Hloader
 Humanlinks
 IDBot
-IRLbot
+Id-search
 IlseBot
+Image Fetch
 IndeedBot
-InfoNaviRobot
-InfoTekies
-InternetSeer
-Iria
+Intelliseek
+Internet Ninja
+Iskanie
+IstellaBot
+JOC Web Spider
 JennyBot
-JetCar
+Jetty
 JikeSpider
 Joomla
-Jorgee
+JustView
 Kenjin Spider
+Keyword Density
+Kinza
 Kozmosbot
-LeechFTP
-LeechGet
-LexiBot
-Libwhisker
-Linkdexbot
-LinkextractorPro
+LWP::Simple
+Lanshanbot
+LibWeb
+Lightspeedsystems
+LinkScan
+Linkbot
 LinksManager
-LinqiaRSSBot
-Lipperhey
-Lmspider
-Ltx71
-MFC_Tear_Sample
-MJ12bot
-MSIECrawler
-MTRobot
-Mag-Net
-Magnet
+MIIxpc
+Mail.RU_Bot
 Majestic SEO
-Majestic12
-MarkWatch
 Masscan
 Mata Hari
-MauiBot
-MeanPath Bot
-MicroMessenger
-Microsoft Data Access
-Microsoft URL Control
-Mister PiX
+MegaIndex.ru
+Minefield
 Mojeek
-Mojolicious
-Mr.4x3
-Msrabot
-NICErsPRO
+Musobot
 NPbot
-Nameprotect
 NearSite
 Needle
+Nessus
 NetLyzer
-NetMechanic
 NetSpider
-NetZIP
-NextGenSearchBot
-Nikto
+Nettrack
+Netvibes
+Niki-bot
 NimbleCrawler
 Nimbostratus
-Openfind
-OutclicksBot
+Offline Explorer
+Offline Navigator
+OpenLinkProfiler
+OpenVAS
+Openvas
+OrangeBot
+PECL::HTTP
+PHPCrawl
 POE-Component-Client-HTTP
 Page Analyzer
-Pi-Monster
-Picscout
+PageThing.com
 Picsearch
-PictureFinder
+Piepmatz
 Pimonster
 Pixray
 PleaseCrawl
-Pockey
-ProPowerBot
-Probethenet
-Psbot
+ProWebWalker
+Pu_iN
 PxBroker
+PyCurl
 QueryN Metasearch
-RankActive
-RankurBot
+Quick-Crawler
+RankActiveLinkBot
+RankFlex
 Re-re
-ReGet
+Reaper
 RepoMonkey
 Ripper
+RocketCrawler
 Rogerbot
-SEOlyticsCrawler
+SEOkicks
+SEOkicks-Robot
 SEOstats
-Scrapy
+SISTRIX
+SalesIntelligent
+ScanAlert
 Screaming
 ScreenerBot
-ScrepyBot
+SearchmetricsBot
 SemanticJuice
-Semrush
-Seomoz
-SiteLockSpider
+SentiBot
+SeoSiteCheckup
+SiteCheckerBotCrawler
+SiteSucker
+Sitebeam
 Siteimprove
-SmartDownload
-Snoopy
-Sociscraper
+Sitevigil
+SlySearch
+Snake
+Sogou web spider
 Sottopop
-Spanner
+SpaceBison
+Spammen
 Spbot
-SputnikBot
 Sqlmap
-SuperBot
-Surfbot
-SurveyBot
-Szukacz
+Sqlworm
+Sqworm
+Stripper
+SuperHTTP
+Suzuran
 Teleport
+Telesoft
+Telesphorep
 The Intraformant
 TightTwatBot
+Toweyabot
 Tracemyfile
-Trendiction
 Trendictionbot
 True_Robot
+Turingos
+Turnitin
 TurnitinBot
-TwengaBot
 Twice
-URLy Warning
-URLy.Warning
-Upflow
-V-BOT
-VB Project
+Typhoeus
+UnisterBot
+VCI
+Vacuum
+Vagabondo
 VelenPublicWebCrawler
 VidibleScraper
-Virusdie
+Voltron
 WBSearchBot
-WEBDAV
 WWW-Collector-E
 WWW-Mechanize
-WWW::Mechanize
 WWWOFFLE
-Wallpapers
-WeSEE
-Web Auto
-Web Sauger
-WebImageCollector
-WebPix
+Wallpapers/3.0
+Web Collage
+Web Sucker
+WebBandit
+WebCopier
+WebEnhancer
 WebReaper
-WebSauger
-WebZIP
-Website Quester
-Widow
+Webalta
+WebmasterWorldForumBot
 WinHTTrack
-Wotbox
-Xaldon_WebSpider
+Wonderbot
 YoudaoBot
-Zade
-Zeus
+Zermelo
 ZmEu
-ZoomBot
 ZumBot
+ZyBorg
+adscanner
+arquivo-web-crawler
 autoemailspider
 backlink-check
-com.plumanalytics
+cah.io.community
+clark-crawler
+crawl.sogou.com
+crawler.feedback
+crawler4j
 dataforseo.com
-eCatch
+demandbase-bot
 evc-batch
-magpie-crawler
+instabid
+internetVista monitor
+isitwp.com
+lwp-request
+lwp-trivial
+mediawords
 muhstik-scan
-netEstate NE Crawler
+page scorer
 plumanalytics
-scan.lol
+scalaj-http
+seobility
 seocompany.store
+seoscanners
+serpstatbot
+sexsearcher
+siteripz
+sogouspider
 spyfu
-tAkeOut
 ubermetrics-technologies.com
-webgains-bot
-webmeup-crawler
+x09Mozilla