V4.2021.09.2572 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
-404checker
-ALittle Client
-ASPSeek
+360Spider
+404enemy
+80legs
+Aboundex
+Aboundexbot
 Acunetix
-AiHitBot
-AllSubmitter
+AfD-Verbotsverfahren
 AlphaBot
-Anarchy
-Anthill
+Ankit
+Apexoo
+Asterias
+Attach
+BBBike
+BDFetch
 BackDoorBot
-BackStreet
-BacklinkCrawler
-Bandit
-Barkrowler
-BlackWidow
-Blow
-Bolt
-Brandwatch
-Buddy
+Badass
+Blackboard
+BotALot
+Brandprotect
+Buck
+BuiltBotTough
+BuiltWith
 BunnySlippers
-Calculon
+BuzzSumo
+CATExplorador
 CazoodleBot
-Cegbfeieh
-CensysInspect
 CherryPicker
-Chlooe
+ChinaClaw
 Claritybot
-Cliqzbot
 Cloud mapping
-Collector
-Copyscape
+Cocolyzebot
+Cogentbot
+Copier
+CopyRightCheck
+Craftbot
+Crawling at Home Project
+CrazyWebCrawler
+Crescent
 CrunchBot
-Curious
-DBLBot
+DTS Agent
+Devil
 Digincore
+DigitalPebble
+Dirbuster
+Discoverybot
 Dispatch
-DittoSpyder
 DnyzBot
 DomCopBot
 DomainAppender
-DomainStatsBot
-Domains Project
-Dotbot
+DomainCrawler
+DomainSigmaCrawler
 Download Wonder
-EMail Wolf
+Dragonfly
 EasyDL
-EirGrabber
-Evil
-Exabot
 ExtLinksBot
-ExtractorPro
-EyeNetIE
-FHscan
 FemtosearchBot
+Fimap
+Firefox/7.0
 FlashGet
 Flunky
 Foobot
-Freeuploader
 FrontPage
-Fyrebot
 G-i-g-a-b-o-t
-GT::WWW
-Genieo
 GetRight
-Getintent
-Go-Ahead-Got-It
-Gotit
-GrabNet
-Grabber
-GridBot
-HEADMasterSEO
-HTTrack
-HaosouSpider
+Gigabot
+GoZilla
+Grafula
+HMView
+HTMLparser
+HTTP::Lite
 Harvest
+Heritrix
 Humanlinks
-IDBTE4M
+HybridBot
 IDBot
 IRLbot
 Iblog
-IlseBot
-Image Fetch
-Indy Library
-Intelliseek
-Internet Ninja
+Id-search
+IndeedBot
+InfoNaviRobot
+InfoTekies
+InterGET
 InternetSeer
-Iskanie
-JOC Web Spider
+Iria
+IstellaBot
 JamesBOT
+JetCar
+Jorgee
 JustView
-Kenjin Spider
-Keyword Density
-Kozmosbot
+Jyxobot
+Kinza
 LNSpiderguy
-Larbin
-LibWeb
+LWP::Simple
+Leap
 Libwhisker
-LieBaoFast
-LinkScan
+Likse
 LinkWalker
-Linkbot
-LinkpadBot
-LinqiaScrapeBot
-Lipperhey
-Litemage_walker
+LinksManager
+LinqiaMetadataDownloaderBot
+Lmspider
 Ltx71
 MFC_Tear_Sample
-Mag-Net
-Majestic SEO
+MIIxpc
+MJ12bot
+MQQBrowser
+MTRobot
+Mail.RU_Bot
 Majestic12
-Masscan
+MarkWatch
+Mass Downloader
+MauiBot
+Mb2345Browser
 MeanPath Bot
-MegaIndex.ru
-Moblie Safari
+Meanpathbot
+MicroMessenger
+Microsoft Data Access
 Mojolicious
+Morfeus Fucking Scanner
 Mozlila
-Msrabot
-Musobot
-NICErsPRO
+NPbot
+Navroad
 NearSite
-Needle
 Nessus
 NetMechanic
-Netcraft
-Netvibes
+NetZIP
+NextGenSearchBot
+Nibbler
 Niki-bot
+Nikto
 NimbleCrawler
 Nimbostratus
-Nuclei
-Octopus
-Offline Explorer
-Offline Navigator
-OpenVAS
-Openvas
-OrangeBot
-OrangeSpider
-PECL::HTTP
+Ninja
+Nmap
+Nutch
+OutclicksBot
 PHPCrawl
-POE-Component-Client-HTTP
-PageAnalyzer
+Page Analyzer
 PageGrabber
 PageScorer
-PageThing.com
-Pandalytics
 Panscient
-Pavuk
+PeoplePal
 Pi-Monster
 Picsearch
+Pixray
 PleaseCrawl
 Pockey
-ProWebWalker
-Probethenet
+ProPowerBot
 Psbot
+Pu_iN
 Pump
-PyCurl
+PxBroker
 Quick-Crawler
-RSSingBot
+RankActiveLinkBot
 RankingBot2
-Reaper
-Recorder
-RocketCrawler
+Rankivabot
+RankurBot
+Re-re
+RedesScrapy
+SBIder
 SEOkicks
+SEOkicks-Robot
 SISTRIX
 SMTBot
-SalesIntelligent
+ScanAlert
 Scrapy
 SearchmetricsBot
-Semrush
 SemrushBot
 Seomoz
-Site Sucker
-Sitebeam
+SiteCheckerBotCrawler
+SiteExplorer
+SiteLockSpider
 Siteimprove
-Sociscraper
+SlySearch
+Snoopy
 Sogou web spider
 Sosospider
-Spammen
+Sottopop
 SpankBot
-Spbot
-SputnikBot
-Sqworm
+Sqlworm
 Steeler
 SurveyBot
 Swiftbot
-Teleport
+Szukacz
+TeleportPro
 Telesphoreo
-TheNomad
 Thumbor
-Toata
-Tracemyfile
-True_Robot
+Trendiction
 Turnitin
-TurnitinBot
-Typhoeus
-URLy.Warning
+Twice
+Upflow
+VB Project
+VCI
 Vacuum
-Vagabondo
-VelenPublicWebCrawler
+VeriCiteCrawler
 VidibleScraper
-Voil
-WASALive-Bot
-WEBDAV
+Voltron
+WISENutbot
 WPScan
 WWW-Collector-E
+WWWOFFLE
+Wallpapers
+Wallpapers/3.0
+WallpapersHD
+Web Auto
 Web Enhancer
-Web Sucker
-WebGo IS
-WebReaper
-WebSucker
-WebWhacker
+Web Fetch
+WebAuto
+WebCollage
+WebFetch
+WebFuck
+WebSauger
+WebStripper
+Webalta
+Webster
+Whack
 Whacker
-Who.is Bot
-Widow
 Wonderbot
-Zitebot
+Wotbox
+Wprecon
+Xaldon WebSpider
+Xenu
+Zauba
 ZmEu
 ZoominfoBot
-ZumBot
-ZyBorg
 arquivo.pt
-autoemailspider
+backlink-check
 cah.io.community
-check1.exe
-clark-crawler
 coccocbot-web
-crawler4j
-dataforseo.com
-demandbase-bot
-eCatch
-ips-agent
-lwp-request
+crawl.sogou.com
+facebookscraper
 lwp-trivial
-magpie-crawler
-meanpathbot
-muhstik-scan
+mediawords
+netEstate NE Crawler
 page scorer
-pcBrowser
-plumanalytics
-polaris version
-ripz
+probe-image-size
 satoristudio.net
-siteripz
-spyfu
+scan.lol
+seobility
+seoscanners
+seostar
+sitechecker.pro
 sysscan
+trendiction.com
 trendiction.de
+ubermetrics-technologies.com
 voyagerx.com
+webmeup-crawler
 webpros.com
-webprosbot
 x09Mozilla
 zgrab