V4.2021.09.2594 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-ADmantX
-AIBOT
-ASPSeek
+360Spider
+404checker
+404enemy
+80legs
+ALittle Client
+Abonti
+AfD-Verbotsverfahren
 Aipbot
+Alexibot
+Alligator
 Anarchy
-Ankit
-Apexoo
-Aspiegel
+Anthill
 Asterias
-Attach
 AwarioRssBot
+BDCbot
+BDFetch
 BLEXBot
-BackDoorBot
-Backlink-Ceck
+Badass
+BatchFTP
+Battleztar Bazinga
 BetaBot
-Bigfoot
-BlackWidow
+Black Hole
 Blackboard
+Blow
 BlowFish
-BotALot
+Boardreader
 Brandprotect
-Buck
 BuiltBotTough
-Bullseye
-BuzzSumo
 CATExplorador
-CCBot
-CODE87
-CSHttp
 Calculon
-CazoodleBot
+CensysInspect
 CheTeam
 CheeseBot
 CherryPicker
 ChinaClaw
 Chlooe
-Claritybot
 Cloud mapping
+Cocolyzebot
 Cogentbot
-Collector
+Copier
 Copyscape
-Cosmos
-Craftbot
-Crescent
-Curious
+CrunchBot
+Custo
 CyotekWebCopy
-DBLBot
+DIIbot
+DSearch
+DTS Agent
+Demon
+Deusu
 Devil
+DigitalPebble
+Dirbuster
 Disco
 Discobot
-Dispatch
+Discoverybot
 DomainSigmaCrawler
 DomainStatsBot
+Dotbot
 Download Wonder
-Dragonfly
-EMail Siphon
-Ebingbong
+Drip
 Ecxi
 EirGrabber
-Exabot
-Extreme Picture Finder
-Ezooms
-FDM
+EyeNetIE
+Firefox/7.0
+Foobot
 Freeuploader
 FrontPage
+FyberSpider
+Fyrebot
 Genieo
+GermCrawler
+GetWeb
+Getintent
 Gigabot
-Gotit
-GrapeFX
-GrapeshotCrawler
-HEADMasterSEO
-HMView
-HTMLparser
+Go!Zilla
 HTTP::Lite
-Humanlinks
-HybridBot
-IDBot
-Iblog
-IlseBot
-IndeedBot
-Intelliseek
-Internet Ninja
-Iria
-JamesBOT
-JetCar
+Hloader
+Indy Library
+InfoNaviRobot
+InterGET
+IstellaBot
+JOC Web Spider
+JennyBot
+Jetty
+JikeSpider
+Jorgee
 JustView
 Keyword Density
-Larbin
-Lftp
-LibWeb
-LinkScan
-Linkdexbot
+Kozmosbot
+Lanshanbot
+Leap
+LinkextractorPro
+LinkpadBot
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-Lipperhey
+Lipperhey Spider
+Litemage_walker
 Ltx71
+MQQBrowser
+MSFrontPage
 MTRobot
 Magnet
-Majestic SEO
+Mail.RU_Bot
 Majestic-SEO
 MarkMonitor
-Mass Downloader
+Mata Hari
 MauiBot
-MeanPath Bot
-Meanpathbot
-MegaIndex.ru
-Moblie Safari
-Mr.4x3
+Mediatoolkitbot
+MicroMessenger
+Microsoft Data Access
+Microsoft URL Control
+Mister PiX
+Mojeek
+Mojolicious
+Msrabot
 Musobot
 NPbot
-Navroad
+Name Intelligence
+Nameprotect
+NearSite
 Nessus
-Net Vampire
 NetAnts
-Netcraft
 Nettrack
-Niki-bot
+Netvibes
+NextGenSearchBot
 Nimbostratus
+Ninja
 Nmap
-Nuclei
-Nutch
 Octopus
 Offline Explorer
-OnCrawl
-OrangeBot
+Offline Navigator
+OpenLinkProfiler
+Openvas
 OutclicksBot
-Page Analyzer
+PHPCrawl
+POE-Component-Client-HTTP
 PageGrabber
-PageScorer
-Pandalytics
+PageThing.com
+Panscient
+Papa Foto
 Pavuk
-Petalbot
-Pi-Monster
+PeoplePal
+Picscout
+Picsearch
 PictureFinder
+Pixray
 PleaseCrawl
 Pockey
 ProWebWalker
+Psbot
 QueryN Metasearch
 Quick-Crawler
-RankActiveLinkBot
-RankFlex
-RankingBot
+RankActive
 RankingBot2
-RankurBot
+Re-re
 ReGet
-RebelMouse
-Recorder
-RocketCrawler
-Rogerbot
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOstats
 SISTRIX
-SMTBot
 SalesIntelligent
 ScanAlert
-Scrapy
+ScoutJet
 Screaming
-ScreenerBot
 ScrepyBot
-SentiBot
-Seomoz
-Shodan
-Site Sucker
-SiteCheckerBotCrawler
+Searchestate
+Semrush
+SeoSiteCheckup
+SeobilityBot
+SiteExplorer
 SiteLockSpider
 SiteSnagger
-Sitebeam
+Siteimprove
+Sitevigil
 SmartDownload
-Snake
-Snoopy
-Sosospider
-Spammen
-Spbot
-Sqlmap
-Sqworm
-SuperBot
-Surfbot
+SpaceBison
+SpankBot
+Steeler
+Sucuri
+SuperHTTP
 SurveyBot
 Suzuran
-T0PHackTeam
-Telesoft
-Telesphoreo
+TeleportPro
+Thumbor
+Titan
 Toata
+Toweyabot
+Trendiction
 Turingos
 Turnitin
-Typhoeus
-UnisterBot
-Upflow
-VB Project
+URLy Warning
 VCI
-VeriCiteCrawler
-Voil
-WWW-Mechanize
-WWW::Mechanize
-WWWOFFLE
-WallpapersHD
-Web Auto
+Vacuum
+Vagabondo
+VelenPublicWebCrawler
+WEBDAV
+WWW-Collector-E
+Wallpapers
 Web Enhancer
 Web Sauger
-WebAuto
+Web Sucker
+WebBandit
 WebEnhancer
-WebFetch
 WebFuck
 WebGo IS
 WebLeacher
+WebPix
 WebReaper
 WebSauger
 WebStripper
-WebWhacker
-WebmasterWorldForumBot
-WebsiteExtractor
-WebsiteQuester
-Webster
+Webalta
+Webshag
+Whack
 Whacker
-Widow
-Wotbox
-Xaldon WebSpider
+Whatweb
+Xenu
 Zade
-Zauba
 Zermelo
+Zitebot
 ZmEu
 ZyBorg
+archive.org_bot
 arquivo-web-crawler
 arquivo.pt
 autoemailspider
 check1.exe
-crawler.feedback
-crawler4j
+crawl.sogou.com
 domainsproject.org
-eCatch
+gopher
+heritrix
 lwp-request
+lwp-trivial
 magpie-crawler
 meanpathbot
+mediawords
 muhstik-scan
-page scorer
 probe-image-size
 ripz
 s1z.ru
-serpstatbot
-sitechecker.pro
-siteripz
+scalaj-http
+scan.lol
+seobility
+seostar
 sogouspider
-sp_auditbot
-spyfu
-trendiction.com
-voyagerx.com
+tAkeOut
+trendiction.de
+ubermetrics-technologies.com
 webmeup-crawler
-webprosbot
-x22Mozilla
 xpymep1.exe
-zgrab