V4.2022.02.2901 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,90 +1,86 @@
 01h4x.com
 360Spider
-404checker
-80legs
-AIBOT
+ADmantX
+ALittle Client
 ASPSeek
 Aboundex
+Acunetix
 Aipbot
-Alexibot
-AlphaBot
-Anarchie
-Anarchy99
+Ankit
+Apexoo
 Asterias
 Attach
-BBBike
 BDCbot
-BackWeb
-BacklinkCrawler
-Bitacle
+BackDoorBot
+BackStreet
+Backlink-Ceck
+Badass
 Blackboard
 BlowFish
-Bolt
-BotALot
 Brandprotect
 Buddy
-Bullseye
-CCBot
+BuiltBotTough
+BuiltWith
 CODE87
-CSHttp
+CazoodleBot
 CensysInspect
-ChinaClaw
+CheTeam
+CherryPicker
+Chlooe
+Citoid
 Claritybot
-Cloud mapping
 Cocolyzebot
-Cogentbot
-Crawling at Home Project
+Collector
+Copyscape
+CrazyWebCrawler
 CrunchBot
+CyotekWebCopy
+DIIbot
 DSearch
-DataCha0s
+DTS Agent
 Devil
-DigitalPebble
+Digincore
+Disco
 Discoverybot
-DnBCrawler-Analytics
-DomCopBot
 DomainAppender
-DomainCrawler
-DomainSigmaCrawler
 DomainStatsBot
-Domains Project
-Dotbot
 Download Wonder
-Dragonfly
-ECCP/1.0
-EasyDL
-EroCrawler
+Drip
+EMail Siphon
+Ecxi
 Evil
-Exabot
 Express WebPictures
 ExtLinksBot
-Extractor
-Extreme Picture Finder
+Ezooms
+FDM
 FHscan
-Foobot
 Freeuploader
+FrontPage
 FyberSpider
 GT::WWW
 GalaxyBot
-GermCrawler
-GetRight
 GetWeb
+Getintent
 Gigabot
+Go!Zilla
 Go-Ahead-Got-It
-GridBot
-HTTP::Lite
+GrabNet
+HEADMasterSEO
+HMView
 HTTrack
+Harvest
 Havij
-HonoluluBot
-Id-search
-Image Fetch
+HybridBot
+IRLbot
 Image Sucker
-IndeedBot
-Indy Library
-InfoNaviRobot
-Intelliseek
+InterGET
+InternetSeer
 IstellaBot
+JOC Web Spider
 JetCar
+Jetty
 JustView
+Keyword Density
 Kozmosbot
 LWP::Simple
 Lanshanbot
@@ -93,158 +89,162 @@ Leap
 LeechFTP
 Lftp
 LibWeb
-Likse
+Libwhisker
+LinkScan
+LinkWalker
+LinkpadBot
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-MIDown tool
+LinqiaRSSBot
+Lipperhey Spider
+Lmspider
+Ltx71
 MIIxpc
-MJ12bot
-MSIECrawler
+MQQBrowser
+MSFrontPage
 MTRobot
 Magnet
-Mail.RU_Bot
-Majestic12
-MarkWatch
-MeanPath Bot
-Meanpathbot
+Mass Downloader
+Masscan
 MegaIndex.ru
-Metauri
 Microsoft URL Control
-Moblie Safari
+Minefield
 Mojeek
 Mojolicious
 MolokaiBot
 Morfeus Fucking Scanner
-Mozlila
+NICErsPRO
 NPbot
 Name Intelligence
+Nameprotect
 Navroad
 Needle
-Nessus
+NetAnts
 NetZIP
 Netcraft
-Nimbostratus
-Nutch
-OnCrawl
-OpenLinkProfiler
-OutclicksBot
+Netvibes
+NextGenSearchBot
+Ninja
+Not
+OpenVAS
+OrangeBot
+PECL::HTTP
+PHPCrawl
 POE-Component-Client-HTTP
 Page Analyzer
-PageAnalyzer
 PageGrabber
 PageScorer
-Papa Foto
-PeoplePal
+PageThing.com
+Petalbot
+Pi-Monster
 Picsearch
+PictureFinder
 Piepmatz
-Pimonster
-Pixray
+PleaseCrawl
 Pockey
 ProPowerBot
-Psbot
-Pump
-Quick-Crawler
-RankFlex
-Rankivabot
+PyCurl
+QueryN Metasearch
+RSSingBot
+RankActive
+RankActiveLinkBot
 Re-re
-Reaper
+ReGet
 Recorder
-Rogerbot
-SEOlyticsCrawler
-SISTRIX
-SMTBot
-SalesIntelligent
+RedesScrapy
+RocketCrawler
+SBIder
+SEOstats
 ScanAlert
-Semrush
-SemrushBot
+ScoutJet
 SeoSiteCheckup
-SeobilityBot
-Seomoz
-SiteLockSpider
-SiteSnagger
+Shodan
+SiteExplorer
+SiteSucker
 Sitebeam
-Siteimprove
-SmartDownload
-Snake
+Sitevigil
 Snapbot
 SocialRankIOBot
-Sociscraper
-Sogou web spider
-SpaceBison
+Spammen
 Spbot
+Spinn3r
+SputnikBot
 Sqlworm
-Sqworm
+Steeler
+Sucuri
+SuperBot
 SuperHTTP
-Surfbot
+Suzuran
+Swiftbot
+Szukacz
 Teleport
-Thumbor
+Telesphorep
+The Intraformant
+TheNomad
 TightTwatBot
 Toata
 Toweyabot
 Trendiction
 True_Robot
 Turingos
-Typhoeus
-URLy Warning
+VCI
+Vacuum
 VelenPublicWebCrawler
-VoidEYE
-Voltron
+VeriCiteCrawler
+VidibleScraper
+Virusdie
+Voil
 WASALive-Bot
+WBSearchBot
 WEBDAV
-Web Fuck
-Web Sauger
-WebAuto
-WebBandit
+WPScan
+WWW::Mechanize
+WWWOFFLE
+WeSEE
+Web Auto
+Web Fetch
+WebCopier
 WebEnhancer
 WebGo IS
-WebLeacher
 WebPix
-WebStripper
-WebSucker
+WebReaper
 WebZIP
 Webalta
-WebmasterWorldForumBot
+Webshag
 Website Quester
-WebsiteExtractor
-Whack
-Whacker
+Webster
 Whatweb
-Who.is Bot
+Widow
+WinHTTrack
 Wonderbot
-Wprecon
+Woobot
 Xaldon WebSpider
-YoudaoBot
+Xaldon_WebSpider
+Xenu
 Zade
-Zauba
 Zermelo
-Zeus
+ZmEu
 adscanner
-archive.org_bot
-backlink-check
+arquivo.pt
+autoemailspider
+cah.io.community
 clark-crawler
 cognitiveseo
-com.plumanalytics
-crawler.feedback
-crawler4j
+crawl.sogou.com
 domainsproject.org
-facebookscraper
-instabid
 ips-agent
-lwp-request
-mediawords
+isitwp.com
 muhstik-scan
-netEstate NE Crawler
-page scorer
-plumanalytics
-probe-image-size
+pcBrowser
+polaris version
 ripz
-seoscanners
-sogouspider
+s1z.ru
+satoristudio.net
+seobility
+seostar
+siteripz
 spyfu
-trendiction.de
-ubermetrics-technologies.com
-webgains-bot
+tAkeOut
+voyagerx.com
 webmeup-crawler
-webpros.com
-x09Mozilla
 xpymep1.exe
+zauba.io