V4.2021.10.2679 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,178 +1,171 @@
+01h4x.com
+404checker
 AIBOT
-ASPSeek
-Abonti
-Aboundexbot
-AiHitBot
+Aipbot
 Alexibot
 AllSubmitter
-Anarchy
-Ankit
-Apexoo
+Anthill
+Asterias
 Attach
-AwarioRssBot
-AwarioSmartBot
-BackStreet
-Backlink-Ceck
+BDFetch
+BLEXBot
 BacklinkCrawler
-Badass
-Bandit
+Battleztar Bazinga
 Black Hole
-BlackWidow
-Blackboard
 Blow
-Boardreader
-Brandprotect
+BotALot
 Brandwatch
-Buck
+BuiltWith
+BunnySlippers
+BuzzSumo
 CATExplorador
 CCBot
-Calculon
 CazoodleBot
-Cegbfeieh
-CensysInspect
-CheeseBot
-ChinaClaw
+CheTeam
 Chlooe
-Claritybot
-Cocolyzebot
+Cliqzbot
+Cloud mapping
 Cogentbot
 Collector
+Copier
+CopyRightCheck
 Copyscape
-Cosmos
 Craftbot
-Crawling at Home Project
-CrazyWebCrawler
-Crescent
+Custo
+CyotekWebCopy
 DBLBot
-DIIbot
-DTS Agent
-DatabaseDriverMysqli
-Demon
-Devil
-DittoSpyder
-DomainAppender
-Dotbot
+Deusu
+Dirbuster
+Disco
+DomCopBot
+DomainStatsBot
+Domains Project
 Dragonfly
 Drip
-ECCP/1.0
-EMail Siphon
-Ebingbong
+EasyDL
+EirGrabber
 EroCrawler
-Express WebPictures
-ExtLinksBot
+Extractor
+ExtractorPro
 Extreme Picture Finder
-Fimap
+FDM
 Firefox/7.0
-Foobot
-GT::WWW
-GetWeb
+Freeuploader
+Fuzz
+FyberSpider
+Genieo
 Gigabot
+Go!Zilla
+Go-Ahead-Got-It
 GoZilla
 Grabber
-Grafula
-HTMLparser
+GrapeFX
+GrapeshotCrawler
+GridBot
+HMView
 HTTrack
-Haansoft
+Harvest
+Havij
 Heritrix
-IDBot
+Hloader
 IRLbot
-Iblog
-Id-search
-Image Fetch
-Image Sucker
-Intelliseek
-InterGET
+IlseBot
+InfoNaviRobot
 Internet Ninja
 InternetSeer
 IstellaBot
 JOC Web Spider
-JamesBOT
+JetCar
 Jetty
 JikeSpider
-JustView
-Jyxobot
-Kenjin Spider
-Kinza
-LNSpiderguy
+Jorgee
+LWP::Simple
 Lanshanbot
-LeechFTP
-Lftp
-Libwhisker
+LeechGet
+LibWeb
 LieBaoFast
-Linkdexbot
-LinkextractorPro
+Likse
+LinkWalker
 LinkpadBot
-LinksManager
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-MFC_Tear_Sample
-MIIxpc
-Majestic12
-Masscan
-MegaIndex.ru
-MicroMessenger
+Lipperhey
+Lipperhey Spider
+Litemage_walker
+MIDown tool
+MQQBrowser
+Mag-Net
+Mail.RU_Bot
+MarkMonitor
+MarkWatch
+Mediatoolkitbot
+Microsoft URL Control
 Moblie Safari
 Mojolicious
 Morfeus Fucking Scanner
-NICErsPRO
+Mr.4x3
+NPbot
+Name Intelligence
+NearSite
 Nessus
-Netcraft
-Netvibes
-NextGenSearchBot
+NetMechanic
+Niki-bot
+NimbleCrawler
 Nimbostratus
-Nmap
 Nutch
 Octopus
 OnCrawl
-OpenLinkProfiler
-OpenVAS
 Openfind
-OutclicksBot
+OrangeBot
 OutfoxBot
-PHPCrawl
+PECL::HTTP
+POE-Component-Client-HTTP
 PageAnalyzer
-PageScorer
-Pavuk
+PageGrabber
+PageThing.com
+Pandalytics
 PeoplePal
 Petalbot
-Pi-Monster
-Picscout
-PictureFinder
 Pimonster
-Pockey
+ProPowerBot
+ProWebWalker
+Psbot
 Pu_iN
-Pump
+PxBroker
 PyCurl
-RankActive
-RankActiveLinkBot
+RSSingBot
 RankFlex
-ReGet
-Reaper
-RedesScrapy
-RepoMonkey
-Ripper
+RankingBot
+Rankivabot
+Re-re
+RealDownload
+RebelMouse
+Recorder
+RocketCrawler
 Rogerbot
+SEOkicks
+SEOkicks-Robot
 SEOstats
-SISTRIX
-SalesIntelligent
-ScanAlert
+SMTBot
 ScoutJet
-Scrapy
-ScreenerBot
-ScrepyBot
-Searchestate
+Screaming
 SearchmetricsBot
 SemrushBot
-SeoSiteCheckup
-SiteSucker
+SeobilityBot
+Seomoz
+Shodan
+SiteLockSpider
+Sitebeam
 Siteimprove
 Sitevigil
-Snoopy
-SocialRankIOBot
+Snake
+Snapbot
 Sociscraper
-Sogou web spider
-SpaceBison
-SpankBot
+Sottopop
+Spammen
 Spbot
+SputnikBot
+Sqlmap
 Sqlworm
 Steeler
 Stripper
@@ -180,71 +173,78 @@ Sucuri
 SuperBot
 SuperHTTP
 Surfbot
+Szukacz
 T0PHackTeam
-Teleport
 TeleportPro
+Telesphoreo
 The Intraformant
 TheNomad
-TightTwatBot
-Tracemyfile
+Titan
+Toata
 Trendiction
-True_Robot
 Turingos
 Turnitin
-Twice
+TurnitinBot
 Typhoeus
-V-BOT
-Vacuum
-VidibleScraper
-WEBDAV
-WWW-Collector-E
+URLy.Warning
+VB Project
+VCI
+VeriCiteCrawler
+Virusdie
+VoidEYE
+Voltron
+WASALive-Bot
+WPScan
 WWW-Mechanize
-WWW::Mechanize
-WWWOFFLE
 Wallpapers
 Wallpapers/3.0
-WallpapersHD
-WeSEE
-Web Auto
 Web Collage
-Web Fuck
-Web Sauger
-WebFetch
-WebLeacher
+Web Fetch
+WebAuto
+WebImageCollector
+WebStripper
+WebSucker
 WebWhacker
-Webalta
+WebZIP
+WebmasterWorldForumBot
+Webshag
+WebsiteQuester
 Webster
-Whack
-Whatweb
 WinHTTrack
-Wonderbot
-Wotbox
-Zade
-Zeus
+Xaldon_WebSpider
+Xenu
+YoudaoBot
 ZoomBot
-ZoominfoBot
-ZumBot
+ZyBorg
 archive.org_bot
 arquivo.pt
-autoemailspider
-cah.io.community
+clark-crawler
+cognitiveseo
+com.plumanalytics
 dataforseo.com
 demandbase-bot
-eCatch
 evc-batch
-instabid
-isitwp.com
-meanpathbot
+gopher
+internetVista monitor
+ips-agent
+lwp-request
+magpie-crawler
 mediawords
-netEstate NE Crawler
-s1z.ru
-seostar
-siteripz
+muhstik-scan
+page scorer
+polaris version
+probe-image-size
+ripz
+satoristudio.net
+scalaj-http
+scan.lol
+seobility
+sogouspider
 spyfu
 tAkeOut
-trendiction.de
 ubermetrics-technologies.com
-webmeup-crawler
+voyagerx.com
 webprosbot
-x09Mozilla
 xpymep1.exe
+zauba.io
+zgrab