V4.2021.07.2458 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
 360Spider
+404checker
 ADmantX
-Aboundexbot
-Acunetix
-AfD-Verbotsverfahren
+Aboundex
+AhrefsBot
 AiHitBot
 Alexibot
-Anarchy
-Anthill
+AllSubmitter
+Alligator
 Apexoo
 Aspiegel
 Attach
-AwarioSmartBot
-Backlink-Ceck
-Bandit
-Barkrowler
-BatchFTP
-Bigfoot
-Bitacle
-BlackWidow
-Blow
-Boardreader
+BDFetch
+BLEXBot
+BacklinkCrawler
+Badass
+BetaBot
+BlowFish
 Bolt
+BotALot
+Brandprotect
 Buck
 Buddy
+BuiltBotTough
 BuiltWith
-BunnySlippers
+BuzzSumo
+CATExplorador
+CCBot
+CODE87
+CSHttp
 Calculon
-CheTeam
 ChinaClaw
-Chlooe
-Claritybot
-Cogentbot
-Collector
-Copier
+Copyscape
 Craftbot
-CrunchBot
-DIIbot
+CrazyWebCrawler
+Crescent
+Curious
+CyotekWebCopy
+DSearch
 DTS Agent
-Digincore
-Disco
+DataCha0s
+DigitalPebble
+Discobot
 DittoSpyder
-DnyzBot
-DomCopBot
 DomainCrawler
-DomainStatsBot
+DomainSigmaCrawler
 Domains Project
-Dotbot
-EMail Wolf
-EirGrabber
-EroCrawler
-Exabot
+Download Wonder
+Evil
 Express WebPictures
+ExtLinksBot
 Extractor
 ExtractorPro
+Extreme Picture Finder
 EyeNetIE
 Ezooms
 FemtosearchBot
-Fimap
 Firefox/7.0
-Fuzz
+Foobot
+Freeuploader
+G-i-g-a-b-o-t
 GalaxyBot
-Getintent
+GermCrawler
+GetRight
 Gigabot
-GrabNet
-HEADMasterSEO
-HTMLparser
+Go!Zilla
+Go-Ahead-Got-It
+Grabber
+HMView
 HTTP::Lite
 HTTrack
-HaosouSpider
-Harvest
+Haansoft
 Humanlinks
-IDBTE4M
+HybridBot
 IDBot
+IlseBot
+Image Fetch
 Image Sucker
 Intelliseek
-IstellaBot
+InterGET
 JOC Web Spider
-JamesBOT
-JennyBot
+Jbrofuzz
+JikeSpider
 Jorgee
+Keyword Density
+Kinza
 Kozmosbot
-LNSpiderguy
-LWP::Simple
-LeechFTP
+Lanshanbot
+Larbin
 LeechGet
 Lftp
-LibWeb
+Libwhisker
 LieBaoFast
-Likse
-LinkScan
-Linkbot
+Lightspeedsystems
+Linkdexbot
 LinkextractorPro
 LinksManager
-LinqiaRSSBot
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
 Lipperhey
 Lipperhey Spider
 Litemage_walker
 Lmspider
-MFC_Tear_Sample
 MIIxpc
+MJ12bot
+MQQBrowser
 Mag-Net
-Mail.RU_Bot
-Majestic SEO
+Magnet
 Majestic-SEO
 MarkMonitor
-Masscan
 Mata Hari
-MeanPath Bot
-Microsoft Data Access
-Microsoft URL Control
-Mister PiX
+Mb2345Browser
+Meanpathbot
+Mediatoolkitbot
+Metauri
 Moblie Safari
-Msrabot
-Name Intelligence
-Nameprotect
-Needle
+Mojeek
+Mojolicious
+NICErsPRO
+Navroad
+Nessus
 NetAnts
-NetMechanic
-NetZIP
-Nettrack
+NetLyzer
 Netvibes
-Nikto
-Nimbostratus
-Offline Navigator
+Nibbler
+Ninja
+Nutch
 OpenLinkProfiler
+Openvas
 OrangeBot
 OrangeSpider
 OutclicksBot
 PECL::HTTP
-PageAnalyzer
 PageGrabber
 PageScorer
-PageThing.com
-Pandalytics
-Papa Foto
+Panscient
 Pavuk
-Picscout
-Piepmatz
-Pixray
-PleaseCrawl
+Pi-Monster
+Pimonster
+Pockey
 ProPowerBot
-ProWebWalker
-Probethenet
-Psbot
+Pu_iN
 Pump
-PxBroker
+Quick-Crawler
+RSSingBot
 RankActiveLinkBot
-Rankivabot
+RankFlex
+RankingBot
 RankurBot
-Re-re
-Reaper
-RebelMouse
+RealDownload
 Recorder
-RedesScrapy
-SEOkicks
-SEOlyticsCrawler
-SEOprofiler
-SEOstats
-SMTBot
+Ripper
+Rogerbot
+SEOkicks-Robot
+SISTRIX
+ScanAlert
+Scanbot
+Scrapy
 Screaming
-ScreenerBot
-Searchestate
-SearchmetricsBot
-Semrush
-SentiBot
-Shodan
+SeoSiteCheckup
 Site Sucker
 SiteExplorer
-Siteimprove
-Sitevigil
+SiteSnagger
+SiteSucker
+Sitebeam
 SmartDownload
 Snoopy
-SocialRankIOBot
 Sociscraper
+Sogou web spider
 Sottopop
-Spammen
-SpankBot
-Sqworm
-Sucker
-SurveyBot
+Sqlmap
+Stripper
+SuperBot
+Surfbot
+Swiftbot
 T0PHackTeam
-Teleport
+T8Abot
 Telesoft
-Telesphorep
 The Intraformant
-TheNomad
-Toata
+TightTwatBot
 Tracemyfile
 Trendictionbot
-Turingos
-Turnitin
+TurnitinBot
+Twice
+UnisterBot
+Upflow
+V-BOT
 VCI
+Vacuum
 Vagabondo
-VelenPublicWebCrawler
-Virusdie
+VeriCiteCrawler
+VoidEYE
 WBSearchBot
 WEBDAV
-WPScan
-WallpapersHD
+WISENutbot
+WWW-Collector-E
+WWW-Mechanize
+WWW::Mechanize
+Wallpapers/3.0
 WeSEE
 Web Enhancer
 Web Pix
-Web Sauger
-Web Sucker
-WebAuto
-WebEnhancer
-WebFuck
-WebLeacher
+WebCollage
+WebFetch
+WebImageCollector
 WebPix
-WebReaper
+WebSauger
 WebStripper
-WebmasterWorldForumBot
+WebSucker
+WebsiteExtractor
+WebsiteQuester
+Webster
+Whack
 Whacker
+Whatweb
 Who.is Bot
-Widow
 Wonderbot
-Woobot
 Wotbox
-Xaldon_WebSpider
-Zauba
+Xaldon WebSpider
+Xenu
+YoudaoBot
+Zade
 Zeus
-ZoomBot
-arquivo.pt
+Zitebot
+ZumBot
+ZyBorg
+autoemailspider
 backlink-check
+coccocbot-web
+crawler.feedback
+crawler4j
 demandbase-bot
+domainsproject.org
 eCatch
-evc-batch
+facebookscraper
 gopher
-internetVista monitor
+instabid
+ips-agent
 lwp-trivial
-magpie-crawler
-muhstik-scan
-netEstate NE Crawler
-plumanalytics
+page scorer
+pcBrowser
 polaris version
-probe-image-size
-s1z.ru
 satoristudio.net
-scalaj-http
 sexsearcher
-sogouspider
+sitechecker.pro
 spyfu
-tAkeOut
+sysscan
 trendiction.com
 x09Mozilla
-xpymep1.exe
-zauba.io
+x22Mozilla