V4.2021.11.2713 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,24 +1,17 @@
-01h4x.com
-404enemy
-80legs
-AIBOT
-ALittle Client
-Aboundexbot
-Alexibot
-AllSubmitter
-Alligator
-Anarchy99
+360Spider
+ASPSeek
+Acunetix
+AiHitBot
+AlphaBot
+Anarchie
 Apexoo
-Asterias
-Attach
+Aspiegel
 AwarioRssBot
 AwarioSmartBot
-BBBike
 BDFetch
 BLEXBot
-BackStreet
-BackWeb
-Backlink-Ceck
+BackDoorBot
+BacklinkCrawler
 Badass
 Bandit
 Barkrowler
@@ -26,225 +19,232 @@ Bigfoot
 Bitacle
 Black Hole
 Blackboard
-BlowFish
 Boardreader
 BotALot
 Brandprotect
-Brandwatch
-BuiltBotTough
-BuiltWith
-CCBot
-Calculon
-CazoodleBot
-CensysInspect
-CheeseBot
+BuzzSumo
+CATExplorador
+CheTeam
 CherryPicker
-Copier
-Cosmos
-Craftbot
-Crawling at Home Project
-Crescent
-CrunchBot
-CyotekWebCopy
-DBLBot
-DIIbot
-Digincore
-Disco
+ChinaClaw
+Claritybot
+Cliqzbot
+Collector
+CopyRightCheck
+Curious
+DTS Agent
+Discoverybot
 Dispatch
-DnyzBot
+DittoSpyder
 DomCopBot
+DomainAppender
+DomainCrawler
 DomainSigmaCrawler
 DomainStatsBot
-Dotbot
+Download Wonder
+Dragonfly
 Drip
+EMail Siphon
+EMail Wolf
 EasyDL
-Ecxi
+EirGrabber
 Evil
-ExtLinksBot
+ExtractorPro
 EyeNetIE
 Ezooms
 FDM
 FHscan
+Fimap
 FlashGet
-Flunky
-Foobot
-Fyrebot
+Fuzz
+FyberSpider
 G-i-g-a-b-o-t
-GalaxyBot
-Genieo
-GetRight
+GT::WWW
 GetWeb
-Getintent
-Gigabot
-GoZilla
+Go-Ahead-Got-It
 GrabNet
-GrapeshotCrawler
-HTMLparser
-HTTP::Lite
-HTTrack
-HaosouSpider
-Humanlinks
-HybridBot
-IDBTE4M
-IDBot
-Iblog
-Image Sucker
+Grabber
+GrapeFX
+HEADMasterSEO
+HMView
+Havij
+Heritrix
+Hloader
+IlseBot
+Image Fetch
 IndeedBot
-InfoNaviRobot
+InfoTekies
 Intelliseek
+InterGET
+Internet Ninja
 InternetSeer
-Iria
+IstellaBot
 JOC Web Spider
 Jbrofuzz
-JennyBot
-Joomla
 Jorgee
-Kozmosbot
-LWP::Simple
+JustView
+Jyxobot
+Keyword Density
 Lanshanbot
-Larbin
 LeechFTP
-LeechGet
-Libwhisker
-LinkWalker
-Linkbot
+LinkScan
+LinkextractorPro
 LinksManager
-LinqiaMetadataDownloaderBot
-Lipperhey Spider
+LinqiaScrapeBot
+MFC_Tear_Sample
 MIIxpc
-MQQBrowser
-MTRobot
-Majestic12
+MSFrontPage
+Mag-Net
+Magnet
+Mass Downloader
+Mata Hari
 MauiBot
-MeanPath Bot
+Meanpathbot
+Mediatoolkitbot
 Metauri
+MicroMessenger
 Microsoft Data Access
+Microsoft URL Control
 Moblie Safari
-Morfeus Fucking Scanner
-Mozlila
-Mr.4x3
-Musobot
+Mojolicious
+Msrabot
 NPbot
-Navroad
+Name Intelligence
 NearSite
-Nessus
-Net Vampire
+Needle
 NetAnts
-NetSpider
+NetLyzer
 NetZIP
+Netcraft
 Netvibes
 Niki-bot
-Nikto
-NimbleCrawler
+Nimbostratus
 Nmap
+Octopus
 Offline Explorer
 Offline Navigator
-OnCrawl
 Openfind
-OrangeSpider
-OutfoxBot
+Openvas
+OrangeBot
 PECL::HTTP
-Page Analyzer
-PageAnalyzer
-PageScorer
+POE-Component-Client-HTTP
 Pandalytics
-Papa Foto
-Pavuk
 PeoplePal
 Petalbot
-Picscout
-PictureFinder
-ProWebWalker
+Picsearch
+Piepmatz
+Pimonster
+Probethenet
+Pump
+PxBroker
+QueryN Metasearch
 Quick-Crawler
+RankActiveLinkBot
 RankingBot2
-ReGet
-RealDownload
-Reaper
-Recorder
+Rankivabot
+RankurBot
+Re-re
+RocketCrawler
+SEOkicks
 SEOprofiler
 SEOstats
+SISTRIX
+SMTBot
 SalesIntelligent
-ScoutJet
+ScanAlert
+Scanbot
+Scrapy
 Searchestate
 SearchmetricsBot
-Semrush
+SemrushBot
 SeobilityBot
-Shodan
-SiteCheckerBotCrawler
-SiteExplorer
-SiteSnagger
+Siteimprove
+SlySearch
+SmartDownload
+Snake
+SocialRankIOBot
 Sociscraper
-Sosospider
 Sottopop
 SpaceBison
+Spammen
 Spanner
-Spinn3r
-Sqlworm
+Spbot
+Sqlmap
 Sqworm
-Sucuri
 SuperHTTP
 SurveyBot
-TeleportPro
-Telesoft
-TheNomad
-Toata
-Trendictionbot
-Turingos
-Typhoeus
+Swiftbot
+Szukacz
+T8Abot
+Telesphoreo
+Thumbor
+TightTwatBot
+Trendiction
+TurnitinBot
+Twice
+UnisterBot
 Upflow
+V-BOT
 VB Project
 VCI
-VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
-Virusdie
-VoidEYE
+Vacuum
 Voil
-WASALive-Bot
-WISENutbot
-WWWOFFLE
+Voltron
+WWW::Mechanize
+Wallpapers
+WallpapersHD
+Web Enhancer
 Web Fetch
-Web Fuck
-WebAuto
+Web Pix
+Web Sucker
+WebBandit
+WebCollage
 WebEnhancer
-WebLeacher
-WebsiteQuester
-Webster
-Whack
+WebFetch
+WebGo IS
+WebReaper
+WebSauger
+WebSucker
+WebWhacker
+WebZIP
+Whacker
 Whatweb
-Widow
-WiseGuys Robot
+Who.is Bot
+WinHTTrack
 Wonderbot
+Woobot
+Wprecon
+YoudaoBot
 Zauba
-Zeus
 Zitebot
-ZyBorg
+ZmEu
+ZoominfoBot
 archive.org_bot
 arquivo-web-crawler
-autoemailspider
-backlink-check
-cah.io.community
 check1.exe
-cognitiveseo
-crawl.sogou.com
+clark-crawler
+com.plumanalytics
 crawler.feedback
+crawler4j
+dataforseo.com
 domainsproject.org
+eCatch
 facebookscraper
-gopher
-internetVista monitor
-magpie-crawler
-netEstate NE Crawler
-pcBrowser
-probe-image-size
-ripz
-s1z.ru
+heritrix
+isitwp.com
+lwp-request
+lwp-trivial
+mediawords
+polaris version
 scalaj-http
+seocompany.store
 siteripz
-sysscan
-trendiction.com
+spyfu
+tAkeOut
+trendiction.de
 ubermetrics-technologies.com
+voyagerx.com
 webmeup-crawler
 x09Mozilla
-xpymep1.exe
+x22Mozilla
 zauba.io
-zgrab