V4.2021.10.2651 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
+404checker
 80legs
-ADmantX
-ALittle Client
+AIBOT
 ASPSeek
-Abonti
 Aboundex
-Acunetix
-AiHitBot
-AllSubmitter
+Aipbot
+Alexibot
 Alligator
+AlphaBot
+Anarchie
+Anarchy99
 Ankit
-Anthill
-Attach
+Apexoo
+AwarioSmartBot
 BDCbot
-BackWeb
-Backlink-Ceck
-Bigfoot
+Black Hole
 BlackWidow
-Blackboard
-Boardreader
+Blow
+BotALot
 Brandprotect
-Buddy
+Brandwatch
+Buck
+BuiltBotTough
 BuiltWith
-Bullseye
-BunnySlippers
-Calculon
+CODE87
 Cegbfeieh
 CensysInspect
-CheTeam
-CherryPicker
 Chlooe
-Claritybot
-Cliqzbot
-Cloud mapping
 Cocolyzebot
-Collector
-Copier
-CopyRightCheck
-Crawling at Home Project
-CrazyWebCrawler
+Cogentbot
 Crescent
-CrunchBot
-Curious
-Custo
+DIIbot
 DSearch
 DataCha0s
-Devil
-Dirbuster
-Disco
+DatabaseDriverMysqli
+Deusu
+Digincore
+DigitalPebble
+Discobot
+Discoverybot
 Dispatch
-DittoSpyder
-DomCopBot
-DomainAppender
-Domains Project
+DomainCrawler
 Dotbot
-Dragonfly
+Download Wonder
 ECCP/1.0
-EirGrabber
-Evil
-Extractor
+EMail Wolf
+EasyDL
+Ebingbong
+EroCrawler
 EyeNetIE
-Ezooms
-Fimap
-Flunky
-Fuzz
+FDM
+FemtosearchBot
+FrontPage
+FyberSpider
+Fyrebot
+G-i-g-a-b-o-t
 GalaxyBot
-GetWeb
-Gotit
+GermCrawler
+GetRight
+Getintent
+GrabNet
 Grabber
-Grafula
 GrapeFX
-GridBot
+HMView
+HTMLparser
 HTTP::Lite
-HaosouSpider
-Heritrix
+HTTrack
+Haansoft
 Hloader
-Humanlinks
-HybridBot
-Iblog
-Image Fetch
-Image Sucker
-Indy Library
-InfoNaviRobot
+IDBot
+Id-search
+IndeedBot
 InfoTekies
+InterGET
 Internet Ninja
+InternetSeer
 Iria
-Iskanie
-JOC Web Spider
-JamesBOT
-JetCar
-Jetty
-JustView
-Kinza
+JikeSpider
+Keyword Density
 LNSpiderguy
-LWP::Simple
 Leap
-LeechGet
 Lftp
-LibWeb
-LieBaoFast
+Libwhisker
 Lightspeedsystems
-Linkdexbot
+LinkScan
 LinkextractorPro
 LinkpadBot
-Ltx71
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
+Lipperhey
+Litemage_walker
+Lmspider
 MFC_Tear_Sample
-MIIxpc
+MIDown tool
+MJ12bot
+MQQBrowser
 MSFrontPage
 Mag-Net
+Magnet
 Mail.RU_Bot
 Majestic12
 MarkWatch
-MauiBot
+Mass Downloader
+Masscan
+Mb2345Browser
 MeanPath Bot
-MegaIndex.ru
 MicroMessenger
-Microsoft URL Control
 Mister PiX
-Moblie Safari
 Mojeek
-Morfeus Fucking Scanner
+Mojolicious
+Mozlila
 Mr.4x3
-Musobot
 NICErsPRO
-Needle
-Nessus
+Navroad
+Net Vampire
 NetAnts
 NetMechanic
 Nettrack
+Netvibes
+NextGenSearchBot
+Niki-bot
 Nikto
-Nutch
-Offline Explorer
-Offline Navigator
-Openvas
+NimbleCrawler
+Nimbostratus
+Nmap
+OnCrawl
+OpenLinkProfiler
+OpenVAS
+OutclicksBot
 OutfoxBot
-Page Analyzer
-PageAnalyzer
-PageThing.com
-Pandalytics
+POE-Component-Client-HTTP
 Papa Foto
+Pavuk
+PeoplePal
 Picsearch
-Pimonster
-Pockey
-Pu_iN
+Piepmatz
+PleaseCrawl
+Psbot
 Pump
+PxBroker
+PyCurl
 QueryN Metasearch
+Quick-Crawler
 RankActiveLinkBot
+RankFlex
 ReGet
 RealDownload
-RepoMonkey
-Ripper
-RocketCrawler
-SBIder
+RebelMouse
+Recorder
 SEOkicks
+SEOlyticsCrawler
 SISTRIX
-SalesIntelligent
 ScanAlert
+Scanbot
 ScoutJet
-Scrapy
-Screaming
-ScrepyBot
-Searchestate
+ScreenerBot
 SemrushBot
-SeobilityBot
+Seomoz
 Shodan
-Siphon
-SiteSnagger
+Site Sucker
+SiteExplorer
 SiteSucker
 Sitebeam
-Sitevigil
-SmartDownload
-Snapbot
+SlySearch
+Snake
+SocialRankIOBot
+Sottopop
+Spammen
+Spanner
+Sqlmap
 Sqworm
-Sucker
-SurveyBot
-T0PHackTeam
+SuperBot
+Szukacz
+TeleportPro
 Telesphoreo
+Telesphorep
+TheNomad
 TightTwatBot
-Toata
 Tracemyfile
 Trendiction
 Trendictionbot
-TurnitinBot
-TwengaBot
+True_Robot
+Turingos
 Twice
 Typhoeus
-URLy Warning
-URLy.Warning
-VCI
+V-BOT
+VB Project
 Vacuum
-Vagabondo
 VeriCiteCrawler
-VidibleScraper
-VoidEYE
-Voil
 Voltron
 WBSearchBot
-WPScan
-WWW-Collector-E
-Wallpapers/3.0
+Wallpapers
 WallpapersHD
-Web Fetch
-Web Sucker
-WebAuto
-WebCollage
-WebFuck
-WebPix
+Web Auto
+Web Fuck
+WebEnhancer
+WebFetch
+WebImageCollector
 WebReaper
+WebSauger
 WebSucker
-WebZIP
-WebmasterWorldForumBot
-Webshag
 WebsiteExtractor
 WebsiteQuester
-Whack
-Who.is Bot
-WiseGuys Robot
-Wprecon
+Webster
+Whacker
+Whatweb
+WinHTTrack
+Wonderbot
+Woobot
+Wotbox
+Xaldon WebSpider
 Xaldon_WebSpider
-YoudaoBot
-Zade
-Zeus
+Zauba
+Zitebot
+ZmEu
+ZoomBot
 ZumBot
 ZyBorg
-arquivo.pt
+archive.org_bot
 autoemailspider
+cah.io.community
+check1.exe
+clark-crawler
+coccocbot-web
+cognitiveseo
 crawl.sogou.com
-demandbase-bot
+crawler.feedback
+dataforseo.com
 domainsproject.org
 eCatch
+evc-batch
+facebookscraper
 gopher
-ips-agent
-lwp-request
-lwp-trivial
+heritrix
+internetVista monitor
+isitwp.com
 mediawords
 plumanalytics
-polaris version
-probe-image-size
-ripz
-seoscanners
-serpstatbot
-sp_auditbot
+s1z.ru
+scalaj-http
+seobility
+siteripz
+spyfu
 sysscan
 tAkeOut
-trendiction.de
-ubermetrics-technologies.com
-webmeup-crawler
-x22Mozilla
+voyagerx.com
+webprosbot
+xpymep1.exe
+zgrab