V4.2021.11.2694 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-360Spider
-404checker
-404enemy
+80legs
 AIBOT
-ALittle Client
+ASPSeek
 Abonti
-AfD-Verbotsverfahren
-AiHitBot
+Aboundexbot
 Alexibot
-Anarchy
-Ankit
-Anthill
+Anarchy99
+Apexoo
 Aspiegel
+Asterias
 Attach
-AwarioRssBot
 BBBike
 BDCbot
-BLEXBot
-Battleztar Bazinga
-BetaBot
-Bigfoot
-Black Hole
-BlackWidow
-Blackboard
+BDFetch
+BackDoorBot
+BackStreet
+BackWeb
+Backlink-Ceck
+BatchFTP
 Blow
 BlowFish
 Boardreader
+Brandprotect
 Buck
+BunnySlippers
 CATExplorador
 CCBot
-CensysInspect
-CheTeam
-CherryPicker
-Cocolyzebot
+CheeseBot
+ChinaClaw
+Cliqzbot
 Collector
-Copyscape
-CrazyWebCrawler
+CopyRightCheck
+Cosmos
+Craftbot
+Crawling at Home Project
 CrunchBot
-CyotekWebCopy
+Custo
 DBLBot
-DIIbot
-DSearch
-DTS Agent
-Deusu
-Discoverybot
-DittoSpyder
-DomainSigmaCrawler
-Domains Project
-Dotbot
-Download Wonder
+Devil
+Discobot
+DomainAppender
+DomainCrawler
+Dragonfly
+Drip
 EMail Siphon
-Ecxi
-EroCrawler
-Extractor
-ExtractorPro
+EMail Wolf
+EasyDL
+Ebingbong
+Evil
+Exabot
 Extreme Picture Finder
-FemtosearchBot
-Fimap
-FrontPage
+EyeNetIE
+Ezooms
+FDM
+Firefox/7.0
+Flunky
+FyberSpider
+Fyrebot
 GT::WWW
 GalaxyBot
+GermCrawler
+GetRight
 GetWeb
+Getintent
 Go-Ahead-Got-It
 GoZilla
-Grabber
-GrapeFX
+GrabNet
+Grafula
 GrapeshotCrawler
-GridBot
-HEADMasterSEO
-HMView
-HTMLparser
-HaosouSpider
-Harvest
-Havij
-Humanlinks
-Iblog
+HTTrack
+Heritrix
+IDBTE4M
+IRLbot
 Id-search
-Image Fetch
-Indy Library
-JOC Web Spider
-JamesBOT
-JetCar
+InfoNaviRobot
+InfoTekies
+Intelliseek
+Iria
+Iskanie
 Jetty
+JikeSpider
+JustView
 Kenjin Spider
+Kozmosbot
 LWP::Simple
 Lanshanbot
 LeechFTP
-LibWeb
-LieBaoFast
-Lightspeedsystems
-LinkScan
+LexiBot
+Lftp
+Libwhisker
 LinkextractorPro
 LinksManager
-LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 LinqiaScrapeBot
-Lipperhey
-Litemage_walker
-MFC_Tear_Sample
-MJ12bot
+Lmspider
+Ltx71
+MIDown tool
 MQQBrowser
-MSIECrawler
-MTRobot
-Mag-Net
+MSFrontPage
+Magnet
+Majestic-SEO
+Majestic12
 MarkMonitor
-MarkWatch
-Mb2345Browser
+Masscan
+Mata Hari
+MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
 Microsoft URL Control
 Mister PiX
-Mojolicious
-Morfeus Fucking Scanner
-Mozlila
-NPbot
-Navroad
-NearSite
-Needle
-Net Vampire
-NetMechanic
-Netcraft
-NextGenSearchBot
+Mojeek
+NICErsPRO
+NetAnts
+Nettrack
+Nibbler
 Niki-bot
 NimbleCrawler
 Ninja
-Nmap
-Nuclei
+Nutch
 Octopus
 OpenLinkProfiler
-OpenVAS
+Openfind
 OrangeBot
-OrangeSpider
 OutclicksBot
-OutfoxBot
-PECL::HTTP
 PHPCrawl
+POE-Component-Client-HTTP
+Page Analyzer
+PageScorer
 Panscient
+Papa Foto
+Pavuk
 Pi-Monster
 Picsearch
 PictureFinder
-Piepmatz
 Pimonster
-ProWebWalker
-Probethenet
-Pu_iN
-PyCurl
-RSSingBot
+Pixray
+PleaseCrawl
+ProPowerBot
+Pump
+QueryN Metasearch
+Quick-Crawler
 RankActive
-RankActiveLinkBot
-Rankivabot
-Re-re
-RealDownload
-RepoMonkey
-SEOlyticsCrawler
-SEOstats
-SalesIntelligent
-Scanbot
-ScoutJet
-Screaming
-Semrush
+RankFlex
+RankingBot
+RankingBot2
+ReGet
+Reaper
+RebelMouse
+RedesScrapy
+RocketCrawler
+SEOprofiler
+Scrapy
+ScreenerBot
+Searchestate
+SearchmetricsBot
 SeoSiteCheckup
 Seomoz
-Siphon
-Siteimprove
-Sitevigil
-Snake
-Snapbot
+Site Sucker
+SiteCheckerBotCrawler
+SiteLockSpider
+Sitebeam
+SlySearch
 Snoopy
+Sosospider
 SpaceBison
-SpankBot
+Spammen
+Spbot
+SputnikBot
 Sqlmap
 Sqlworm
 Stripper
 SuperBot
-Surfbot
-Suzuran
-T0PHackTeam
+SuperHTTP
+Swiftbot
+Szukacz
 TeleportPro
+Telesoft
+Telesphoreo
 Telesphorep
-Thumbor
-Toata
-Toweyabot
 Tracemyfile
-True_Robot
-Turingos
-Turnitin
+TurnitinBot
 TwengaBot
-URLy Warning
+Twice
 URLy.Warning
 UnisterBot
-V-BOT
+Upflow
 VB Project
 Vagabondo
-VelenPublicWebCrawler
-VidibleScraper
-Voltron
-WPScan
+WASALive-Bot
+WWW-Collector-E
+WWW::Mechanize
+WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WeSEE
-Web Fetch
+Web Auto
+Web Collage
+Web Fuck
 Web Pix
-Web Sauger
 WebAuto
+WebBandit
 WebCollage
-WebEnhancer
+WebFetch
+WebFuck
 WebGo IS
-WebImageCollector
-WebLeacher
+WebReaper
 WebSauger
 WebSucker
-Webalta
 Webshag
-Website Quester
-Whack
+WebsiteExtractor
+WebsiteQuester
 Whatweb
-WinHTTrack
-WiseGuys Robot
+Wonderbot
+Woobot
 Wotbox
-Xaldon WebSpider
-Xaldon_WebSpider
-YoudaoBot
+Zade
 Zauba
-Zitebot
-ZmEu
+Zeus
 ZoomBot
-archive.org_bot
+ZoominfoBot
+ZyBorg
 autoemailspider
-cah.io.community
+check1.exe
+clark-crawler
 coccocbot-web
-cognitiveseo
+crawl.sogou.com
 crawler4j
 dataforseo.com
-internetVista monitor
-magpie-crawler
+eCatch
+evc-batch
+isitwp.com
+lwp-trivial
 meanpathbot
-netEstate NE Crawler
-plumanalytics
-ripz
+muhstik-scan
+oBot
+pcBrowser
+s1z.ru
 satoristudio.net
-scalaj-http
 seobility
-seoscanners
 seostar
-sitechecker.pro
-trendiction.de
+sexsearcher
+sogouspider
+spyfu
+sysscan
+tAkeOut
 ubermetrics-technologies.com
+webmeup-crawler
 webprosbot
-x09Mozilla
-x22Mozilla
 xpymep1.exe
-zgrab
+zauba.io