V4.2021.11.2741 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
+01h4x.com
 404checker
+80legs
 AIBOT
-ALittle Client
-ASPSeek
-Aboundex
-AhrefsBot
-AiHitBot
+Aboundexbot
+Acunetix
+AfD-Verbotsverfahren
+Aipbot
 Alexibot
-AllSubmitter
-AlphaBot
 Anarchy
-Anarchy99
-Asterias
-Attach
+Ankit
+Anthill
 AwarioSmartBot
-BLEXBot
+BDCbot
 BackDoorBot
-BackWeb
+BackStreet
+BacklinkCrawler
+Badass
 Bandit
 Barkrowler
-BlackWidow
-Blackboard
+BetaBot
+Bigfoot
+Bitacle
 BlowFish
-Boardreader
-Bolt
-Brandwatch
+Brandprotect
 Buddy
-BuiltBotTough
-BuiltWith
 Bullseye
+BuzzSumo
+CCBot
+CODE87
 CazoodleBot
-Cegbfeieh
-CheTeam
-Claritybot
+CherryPicker
 Cliqzbot
 Cloud mapping
+Collector
 CopyRightCheck
-CrazyWebCrawler
-Curious
+Cosmos
+Crawling at Home Project
+CrunchBot
+Custo
+DBLBot
 DIIbot
 DSearch
-DTS Agent
-Demon
-Disco
 Discobot
-DnBCrawler-Analytics
+Discoverybot
+Dispatch
+DomainAppender
 DomainStatsBot
 Dragonfly
-EMail Wolf
-Ecxi
+Drip
+EMail Siphon
+EasyDL
 EirGrabber
+EroCrawler
 Evil
+ExtLinksBot
 ExtractorPro
 Extreme Picture Finder
-FDM
-FrontPage
-FyberSpider
+Fimap
+G-i-g-a-b-o-t
+GalaxyBot
 Genieo
-GermCrawler
-Getintent
+Gigabot
 Go!Zilla
+Go-Ahead-Got-It
 GoZilla
 GrabNet
+GrapeFX
 GrapeshotCrawler
-GridBot
 HEADMasterSEO
 HTTP::Lite
 HTTrack
-Havij
-Heritrix
 Hloader
 IDBTE4M
-IRLbot
-Id-search
+IDBot
+Iblog
+IlseBot
+IndeedBot
 InfoNaviRobot
-InfoTekies
 Intelliseek
 Iria
-IstellaBot
+Iskanie
+JamesBOT
+JennyBot
 JetCar
-Joomla
 Jorgee
 JustView
-Keyword Density
+Kozmosbot
 LNSpiderguy
-LWP::Simple
 Lanshanbot
+Larbin
 LeechGet
+LexiBot
 Lftp
-Libwhisker
-LieBaoFast
-Lightspeedsystems
-Likse
-LinkWalker
-Linkdexbot
-LinkpadBot
-Lipperhey
+LibWeb
+LinkextractorPro
+LinqiaRSSBot
 Ltx71
-MFC_Tear_Sample
-MIIxpc
 MJ12bot
 MQQBrowser
+MSFrontPage
 MSIECrawler
-MTRobot
+Mag-Net
 Magnet
 Mail.RU_Bot
 Majestic-SEO
-MarkWatch
+MarkMonitor
 Mata Hari
-MauiBot
-Meanpathbot
-MegaIndex.ru
-Metauri
-MicroMessenger
-Microsoft Data Access
-Mister PiX
-Mojeek
+Microsoft URL Control
+Moblie Safari
 Mojolicious
-Mr.4x3
+Msrabot
 Musobot
-Nameprotect
-Nessus
-Net Vampire
+NICErsPRO
+Name Intelligence
+Navroad
+Needle
 NetLyzer
 NetMechanic
-NetZIP
-Netvibes
+NetSpider
+Nettrack
+NextGenSearchBot
+Nibbler
 Nimbostratus
-Nutch
+Ninja
+Nmap
+Nuclei
 Octopus
-Offline Explorer
-OnCrawl
+Offline Navigator
+OpenLinkProfiler
+OpenVAS
 Page Analyzer
 PageAnalyzer
-PageThing.com
+PageScorer
+Pandalytics
 Panscient
-Papa Foto
-PeoplePal
-PictureFinder
+Petalbot
+Picsearch
 Piepmatz
-Pixray
-Pockey
+PleaseCrawl
 ProPowerBot
+ProWebWalker
+Probethenet
 PxBroker
-RankingBot2
-RebelMouse
+RankActive
+RankFlex
+Re-re
+ReGet
+RocketCrawler
 Rogerbot
-SBIder
-SEOkicks
+SISTRIX
 SMTBot
-SalesIntelligent
 ScanAlert
+Scanbot
+ScoutJet
 Scrapy
-ScreenerBot
+Screaming
+Searchestate
 Semrush
-SemrushBot
-SentiBot
-Seomoz
+SeoSiteCheckup
+Shodan
+SiteExplorer
+SiteLockSpider
+SiteSnagger
 SiteSucker
-Sitebeam
-Siteimprove
 Sitevigil
 SlySearch
+SmartDownload
 Snake
-Snapbot
-Sociscraper
+SocialRankIOBot
 Sosospider
+SpaceBison
+Spammen
 SpankBot
 Spbot
 Sqlmap
-Steeler
-Stripper
-Sucker
-SuperBot
-SuperHTTP
-SurveyBot
+Sqlworm
+Sqworm
 Suzuran
-Szukacz
-TeleportPro
+Telesoft
+The Intraformant
 Thumbor
-Trendictionbot
-Turnitin
 TurnitinBot
-TwengaBot
-URLy.Warning
-V-BOT
-VB Project
-Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
+Twice
+Typhoeus
+URLy Warning
+Upflow
 VidibleScraper
 Virusdie
-Voil
-WASALive-Bot
 WBSearchBot
 WEBDAV
-WPScan
+Wallpapers
 WallpapersHD
-WeSEE
+Web Auto
+Web Enhancer
+Web Fetch
 Web Fuck
 Web Pix
-Web Sauger
-WebAuto
+Web Sucker
+WebCollage
 WebCopier
-WebFetch
+WebEnhancer
 WebGo IS
 WebImageCollector
 WebLeacher
 WebReaper
 WebStripper
 WebSucker
+WebWhacker
+WebmasterWorldForumBot
+Webshag
 Website Quester
-WebsiteQuester
-Who.is Bot
-Widow
-Wonderbot
-Wotbox
+WebsiteExtractor
+Webster
+Whacker
+Woobot
 Wprecon
-Xaldon WebSpider
-Xaldon_WebSpider
-Zermelo
-Zeus
+Xenu
+YoudaoBot
 Zitebot
-ZoominfoBot
+ZmEu
+ZoomBot
 adscanner
-backlink-check
+arquivo-web-crawler
 cah.io.community
 check1.exe
 clark-crawler
 coccocbot-web
 com.plumanalytics
-crawler.feedback
-domainsproject.org
-gopher
-heritrix
+crawler4j
 internetVista monitor
 ips-agent
-lwp-trivial
-magpie-crawler
-scalaj-http
-serpstatbot
-sitechecker.pro
-spyfu
+lwp-request
+oBot
+pcBrowser
+probe-image-size
+ripz
+s1z.ru
+satoristudio.net
+seobility
+seocompany.store
+sogouspider
+sp_auditbot
 sysscan
-tAkeOut
 trendiction.com
-voyagerx.com
-webprosbot
-zgrab
+webgains-bot
+x09Mozilla
+x22Mozilla
+xpymep1.exe
+zauba.io