V4.2021.09.2622 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404enemy
+01h4x.com
+360Spider
+404checker
 80legs
-Abonti
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
+ADmantX
+ALittle Client
+ASPSeek
 Aipbot
 Alexibot
-Anarchie
-Anthill
-Apexoo
+Asterias
+Attach
 AwarioRssBot
 AwarioSmartBot
 BBBike
 BDFetch
-BackDoorBot
+BLEXBot
+BackStreet
 BackWeb
-Badass
-BatchFTP
-Black Hole
-BlowFish
-BotALot
-Buck
+Backlink-Ceck
+Bandit
+Barkrowler
+Bitacle
+Blackboard
+Brandprotect
+Buddy
+BuiltWith
+Bullseye
+BuzzSumo
 CATExplorador
-CSHttp
-Cegbfeieh
+CensysInspect
 CheTeam
-CherryPicker
-Chlooe
-Claritybot
-Cliqzbot
-Cloud mapping
-Collector
+Cogentbot
 Copier
-Crawling at Home Project
+Copyscape
+Cosmos
+CrazyWebCrawler
+Crescent
+Curious
 Custo
+DBLBot
 DSearch
+Deusu
 Digincore
-Dirbuster
-Disco
-Discobot
-DnyzBot
-DomainAppender
+DittoSpyder
 DomainCrawler
+Domains Project
+Dotbot
+Download Wonder
+Dragonfly
+Drip
 EirGrabber
-EroCrawler
 Evil
 Exabot
-ExtLinksBot
-FHscan
+ExtractorPro
 FemtosearchBot
 Fimap
 Firefox/7.0
 Flunky
-Foobot
 FrontPage
 Fuzz
-G-i-g-a-b-o-t
+FyberSpider
+GalaxyBot
 Genieo
-GermCrawler
-GoZilla
+Gigabot
+Go!Zilla
 GrabNet
-Grabber
-GrapeFX
-GridBot
-HTTP::Lite
-Haansoft
-HaosouSpider
+Grafula
+GrapeshotCrawler
+Havij
+Heritrix
 Hloader
 Humanlinks
 HybridBot
 IDBTE4M
-IlseBot
-IndeedBot
+IDBot
+IRLbot
+Image Fetch
 Indy Library
-InfoTekies
+Internet Ninja
+InternetSeer
 IstellaBot
-JOC Web Spider
-JetCar
+JamesBOT
+Jetty
 JikeSpider
-JustView
-Keyword Density
-Kozmosbot
-LWP::Simple
-Lanshanbot
+Jorgee
+Kenjin Spider
 Larbin
 LeechGet
-Libwhisker
+Lftp
 LieBaoFast
-Lightspeedsystems
-Likse
-LinkWalker
 Linkbot
-LinkextractorPro
-LinkpadBot
+Linkdexbot
 LinksManager
 LinqiaMetadataDownloaderBot
-Lipperhey
-Lipperhey Spider
-Ltx71
-MSFrontPage
-MSIECrawler
+LinqiaRSSBot
+Litemage_walker
+Lmspider
+MFC_Tear_Sample
+MIIxpc
+MJ12bot
+MQQBrowser
+MTRobot
 Mag-Net
-Mass Downloader
+Majestic SEO
+Majestic-SEO
+Majestic12
 Masscan
-Mata Hari
-MauiBot
+Mb2345Browser
 Meanpathbot
-Metauri
-MicroMessenger
+MegaIndex.ru
 Mister PiX
-Moblie Safari
-Morfeus Fucking Scanner
-Msrabot
+Mojeek
+Mojolicious
+Mozlila
 Musobot
-Name Intelligence
+Nameprotect
 Navroad
-Needle
-Net Vampire
+NetAnts
 NetMechanic
 NetSpider
+Nettrack
+Netvibes
 NextGenSearchBot
+Nibbler
 Niki-bot
 Nikto
-NimbleCrawler
-Nimbostratus
+Nuclei
+Nutch
 Offline Navigator
-OpenVAS
-Openfind
-Openvas
-OrangeBot
-OutfoxBot
+OpenLinkProfiler
+POE-Component-Client-HTTP
 PageAnalyzer
-PageGrabber
-Papa Foto
+PageScorer
+Panscient
 Pavuk
 PeoplePal
-Pi-Monster
-Picscout
-Picsearch
-Piepmatz
-Pimonster
-Pixray
-PleaseCrawl
-Pockey
-Pump
-Quick-Crawler
+ProPowerBot
+Probethenet
+Psbot
+QueryN Metasearch
 RankFlex
+Rankivabot
 RankurBot
+Re-re
+RealDownload
+RebelMouse
 Recorder
-RepoMonkey
-Ripper
+RocketCrawler
 Rogerbot
+SBIder
+SEOkicks
+SEOlyticsCrawler
+SEOprofiler
 SEOstats
-SISTRIX
 SMTBot
-ScanAlert
+SalesIntelligent
 ScoutJet
-Scrapy
 ScreenerBot
-Semrush
-SemrushBot
-SentiBot
+Searchestate
 SeobilityBot
 Shodan
-Site Sucker
-SiteExplorer
+Siphon
 SiteLockSpider
-SiteSnagger
 SiteSucker
 Siteimprove
-Snapbot
-Snoopy
-SocialRankIOBot
+SlySearch
+Snake
+Sociscraper
 Sosospider
-SpankBot
+Sottopop
 Spanner
-Sqlmap
-Sqlworm
-Sqworm
-Surfbot
-SurveyBot
+Sucker
+Sucuri
 Suzuran
+Swiftbot
+Szukacz
 T0PHackTeam
+T8Abot
 Teleport
 Telesoft
-Telesphoreo
 Telesphorep
-TheNomad
-Titan
-Toweyabot
+The Intraformant
+Toata
 Trendiction
 Trendictionbot
+True_Robot
+Twice
 Typhoeus
-Upflow
-Vacuum
+URLy Warning
+V-BOT
+VB Project
+VelenPublicWebCrawler
 VeriCiteCrawler
-VoidEYE
+Voltron
+WASALive-Bot
 WBSearchBot
-WISENutbot
+WEBDAV
 WWW-Collector-E
 WWW-Mechanize
-Wallpapers
-WeSEE
+WWW::Mechanize
+Wallpapers/3.0
 Web Enhancer
-Web Fuck
-WebBandit
+Web Sauger
+Web Sucker
 WebCollage
 WebFetch
-WebImageCollector
+WebFuck
+WebGo IS
 WebLeacher
+WebPix
+WebReaper
 WebSucker
 WebWhacker
-Webalta
-WebmasterWorldForumBot
-Website Quester
-WebsiteExtractor
-Webster
-Whatweb
-WinHTTrack
+WebZIP
+Webshag
+Widow
 Wonderbot
-Wprecon
 Xaldon WebSpider
-Xaldon_WebSpider
-YoudaoBot
-Zauba
+Zade
+Zitebot
 ZmEu
+ZoominfoBot
 ZumBot
 archive.org_bot
-autoemailspider
-backlink-check
-cah.io.community
-com.plumanalytics
-crawler4j
-domainsproject.org
-eCatch
+clark-crawler
+cognitiveseo
+crawler.feedback
+instabid
 internetVista monitor
+isitwp.com
+lwp-trivial
+magpie-crawler
 muhstik-scan
 oBot
+page scorer
+pcBrowser
+plumanalytics
+ripz
+s1z.ru
 satoristudio.net
-seoscanners
-seostar
+scalaj-http
+scan.lol
+serpstatbot
+sitechecker.pro
 siteripz
-sysscan
-trendiction.com
-ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
+sp_auditbot
+tAkeOut
+trendiction.de
 x09Mozilla
-x22Mozilla
+zauba.io