V4.2021.12.2771 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 80legs
-AIBOT
-ALittle Client
-Aboundex
-Acunetix
+ASPSeek
+Abonti
+Aboundexbot
+AhrefsBot
 AiHitBot
-Alexibot
+Alligator
+AlphaBot
 Anarchie
-Anthill
+Ankit
 Aspiegel
-Asterias
 Attach
-AwarioRssBot
 AwarioSmartBot
 BDCbot
-BDFetch
-Backlink-Ceck
-BlackWidow
-Blow
+BLEXBot
+Bandit
+BatchFTP
+Black Hole
 Boardreader
 BotALot
-Brandprotect
-Buck
-BuzzSumo
+Brandwatch
+Buddy
+BuiltWith
+Bullseye
 CCBot
 CODE87
-CSHttp
-Calculon
-CazoodleBot
-Cegbfeieh
-CheTeam
 CheeseBot
 ChinaClaw
-Chlooe
 Citoid
-Cliqzbot
+Claritybot
 Cocolyzebot
 Copier
+CopyRightCheck
+Copyscape
+Cosmos
+Crescent
 CrunchBot
-DIIbot
+CyotekWebCopy
+DSearch
 DTS Agent
-DataCha0s
+DatabaseDriverMysqli
 Demon
 Deusu
+Devil
+Digincore
+DigitalPebble
 Dirbuster
-Discobot
-Dispatch
-DnBCrawler-Analytics
-Domains Project
-Dotbot
-Download Wonder
-ECCP/1.0
+DomainAppender
+DomainCrawler
+DomainSigmaCrawler
+Dragonfly
+Drip
+EMail Siphon
 EMail Wolf
-Ebingbong
-Exabot
-Express WebPictures
-ExtLinksBot
+EasyDL
+EroCrawler
+Evil
 Extractor
+Extreme Picture Finder
+Ezooms
 FHscan
 FemtosearchBot
-Fimap
 Flunky
 FrontPage
 Fuzz
 FyberSpider
-G-i-g-a-b-o-t
+GT::WWW
 GalaxyBot
 GermCrawler
-GetRight
-Go-Ahead-Got-It
-GoZilla
-GrabNet
-GrapeshotCrawler
-HEADMasterSEO
-HTTP::Lite
-Haansoft
+Getintent
+Gigabot
+Grafula
+GridBot
+HTMLparser
+HTTrack
 Harvest
 Heritrix
+Hloader
 Humanlinks
 IDBot
-Iblog
-Id-search
-Indy Library
-InfoNaviRobot
-InterGET
-Internet Ninja
-Iria
-Iskanie
-JOC Web Spider
-JamesBOT
+IRLbot
+IlseBot
+Image Fetch
+Image Sucker
+IndeedBot
+Intelliseek
+IstellaBot
 Jbrofuzz
-JetCar
-Jyxobot
-Keyword Density
-Larbin
-LexiBot
+Jetty
+Jorgee
+Kinza
+Kozmosbot
+Lanshanbot
+LeechFTP
+Libwhisker
 LinkScan
+LinkWalker
 Linkbot
 Linkdexbot
-LinkpadBot
+LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
 Lipperhey Spider
 Ltx71
+MFC_Tear_Sample
+MIIxpc
+MJ12bot
 MSFrontPage
-Mag-Net
-Majestic12
+Mail.RU_Bot
+Majestic SEO
+Majestic-SEO
 MarkMonitor
+MarkWatch
 Mass Downloader
-Masscan
-Mata Hari
-Mb2345Browser
-MeanPath Bot
-Mediatoolkitbot
+Meanpathbot
 MegaIndex.ru
-Metauri
-Microsoft Data Access
 Microsoft URL Control
-Moblie Safari
+Minefield
+Mister PiX
 Mojolicious
 Morfeus Fucking Scanner
 Mr.4x3
-NICErsPRO
-Navroad
-Net Vampire
-NetMechanic
-Netcraft
-Niki-bot
+Msrabot
+Musobot
+Name Intelligence
+Nameprotect
+NearSite
+Needle
+Nessus
+NetZIP
+Netvibes
+NextGenSearchBot
 NimbleCrawler
-Nimbostratus
 Nmap
-Nuclei
-Octopus
-OnCrawl
-OpenLinkProfiler
+Not
+Nutch
 OpenVAS
 Openfind
-OrangeSpider
-POE-Component-Client-HTTP
-Page Analyzer
-PageGrabber
-PageThing.com
-Pandalytics
+Openvas
+OutclicksBot
+OutfoxBot
+PageAnalyzer
+PageScorer
+Panscient
 PeoplePal
-Picscout
-PictureFinder
-Piepmatz
-Pockey
+Picsearch
+ProPowerBot
 ProWebWalker
-PxBroker
-PyCurl
+Pu_iN
 QueryN Metasearch
-RankActiveLinkBot
+Quick-Crawler
 RankFlex
 RankingBot
 Rankivabot
-RankurBot
 Reaper
-RebelMouse
-RedesScrapy
-Ripper
-Rogerbot
+Recorder
+RepoMonkey
+RocketCrawler
 SEOkicks
-SEOkicks-Robot
 SEOlyticsCrawler
-SEOstats
+SEOprofiler
 SISTRIX
-ScanAlert
-ScoutJet
-Scrapy
-ScreenerBot
-Searchestate
-SearchmetricsBot
-Semrush
-SeobilityBot
-Shodan
+SMTBot
+Scanbot
+ScrepyBot
+SemrushBot
+SentiBot
+SeoSiteCheckup
 Siphon
 Site Sucker
-SiteExplorer
-SiteSucker
+SiteLockSpider
 Sitebeam
+Siteimprove
 Sitevigil
-SocialRankIOBot
-Sociscraper
+SlySearch
+Sogou web spider
 Sottopop
-SpaceBison
-Spanner
+Spammen
+SpankBot
 Spinn3r
+Sqworm
 Steeler
-SurveyBot
-Suzuran
+Sucker
+SuperBot
 Swiftbot
-The Intraformant
-Titan
-TwengaBot
-Typhoeus
-Upflow
+Telesphorep
+TheNomad
+Thumbor
+Trendictionbot
+Turingos
+URLy Warning
+URLy.Warning
+V-BOT
 VB Project
-VidibleScraper
+VCI
+Vacuum
+Vagabondo
+VeriCiteCrawler
 Virusdie
 Voil
-WASALive-Bot
+Voltron
 WEBDAV
-WallpapersHD
+WWWOFFLE
+Wallpapers/3.0
 WeSEE
+Web Enhancer
 Web Fuck
-WebEnhancer
-WebLeacher
+Web Pix
+Web Sucker
+WebGo IS
 WebReaper
 WebSauger
-WebStripper
-WebWhacker
-WebZIP
-Webalta
+WebSucker
 Webshag
+Website Quester
 WebsiteQuester
-Webster
 Whatweb
-WiseGuys Robot
+Who.is Bot
+WinHTTrack
 Wotbox
 YoudaoBot
-Zauba
-Zermelo
-Zeus
-Zitebot
-ZyBorg
-arquivo.pt
-clark-crawler
-com.plumanalytics
-dataforseo.com
-eCatch
-gopher
-heritrix
-ips-agent
+ZoomBot
+ZumBot
+adscanner
+autoemailspider
+backlink-check
+cah.io.community
+coccocbot-web
+cognitiveseo
+crawl.sogou.com
+crawler.feedback
+domainsproject.org
+evc-batch
+facebookscraper
+instabid
+lwp-request
+magpie-crawler
 meanpathbot
-muhstik-scan
-pcBrowser
+oBot
+page scorer
 plumanalytics
-polaris version
-satoristudio.net
-serpstatbot
-trendiction.com
-trendiction.de
-ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
-webprosbot
-x09Mozilla
+s1z.ru
+seobility
+seoscanners
+seostar
+sexsearcher
+sogouspider
+sp_auditbot
+tAkeOut
+webgains-bot
 zgrab