V4.2021.08.2528 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
 404enemy
-Aboundex
+80legs
+Abonti
 Aboundexbot
-AhrefsBot
+Acunetix
+AfD-Verbotsverfahren
+Aipbot
+Alexibot
 AllSubmitter
-Alligator
 AlphaBot
-Ankit
-Anthill
-Aspiegel
+Anarchie
+Anarchy99
 BBBike
-BLEXBot
-BackDoorBot
+BDCbot
 BackStreet
-BackWeb
 Backlink-Ceck
-Battleztar Bazinga
-BetaBot
-Bigfoot
-BlackWidow
+Barkrowler
+Black Hole
+Blow
+BotALot
 BuiltBotTough
-BuiltWith
+Bullseye
+BunnySlippers
 CATExplorador
+CCBot
 CODE87
-CazoodleBot
 Cegbfeieh
-CheTeam
+CensysInspect
 CheeseBot
-ChinaClaw
-Chlooe
+Cloud mapping
 Cocolyzebot
+Cogentbot
 Collector
+Copyscape
 Cosmos
-Craftbot
 CrunchBot
-DSearch
-DTS Agent
+Curious
+CyotekWebCopy
 DataCha0s
-Deusu
+DatabaseDriverMysqli
+Demon
 Digincore
-Dirbuster
-Discobot
-DomCopBot
-DomainCrawler
-DomainStatsBot
-Domains Project
-EMail Wolf
-EirGrabber
-Express WebPictures
+DomainAppender
+DomainSigmaCrawler
+Dotbot
+Download Wonder
+Dragonfly
+Drip
+ECCP/1.0
+EasyDL
+Exabot
+ExtLinksBot
+Extractor
 ExtractorPro
-Ezooms
+FDM
 FHscan
 FemtosearchBot
-FlashGet
+FrontPage
+Fuzz
 FyberSpider
-Fyrebot
-GT::WWW
-GetRight
+GalaxyBot
+GermCrawler
+GetWeb
 Getintent
 Go!Zilla
+Go-Ahead-Got-It
 GoZilla
-Gotit
-GrabNet
 Grabber
-Grafula
 HEADMasterSEO
-HMView
-HTMLparser
-HaosouSpider
-Havij
+HTTP::Lite
+Harvest
 Humanlinks
-HybridBot
-IRLbot
-Iblog
+IDBot
+Image Fetch
 IndeedBot
-InfoNaviRobot
-Internet Ninja
-Iskanie
-JOC Web Spider
-Jbrofuzz
-JennyBot
-JetCar
+Intelliseek
+InterGET
+Iria
+JikeSpider
 Joomla
 Jorgee
 JustView
+Keyword Density
 Kinza
+Kozmosbot
 LNSpiderguy
-LeechGet
+Lanshanbot
+Leap
+LeechFTP
 LexiBot
-Lftp
-LinkWalker
+LibWeb
+Likse
+LinkScan
 Linkbot
-LinkpadBot
+Linkdexbot
 LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaRSSBot
+Lmspider
 MJ12bot
-MSFrontPage
-Mag-Net
-Magnet
+MTRobot
+Mail.RU_Bot
+Majestic SEO
+Majestic12
+MarkMonitor
 MarkWatch
-MauiBot
 Mb2345Browser
 MeanPath Bot
+Metauri
 Microsoft Data Access
+Moblie Safari
 Mojeek
 Mojolicious
-Mozlila
 Mr.4x3
 Musobot
-Nessus
+Nameprotect
+NearSite
+Net Vampire
 NetAnts
-NetSpider
-Netcraft
-Netvibes
+NetLyzer
+NetZIP
+Nettrack
+NextGenSearchBot
 Nibbler
-Niki-bot
-Nikto
-NimbleCrawler
 Nimbostratus
-Ninja
 Nmap
-Nutch
+Octopus
 Offline Navigator
-OpenLinkProfiler
-Openfind
+OpenVAS
 OrangeBot
 OrangeSpider
-PECL::HTTP
 POE-Component-Client-HTTP
-PageAnalyzer
-PageScorer
-PageThing.com
+Page Analyzer
+Pandalytics
+Pavuk
+PeoplePal
+Petalbot
 Picsearch
+PictureFinder
 Pixray
-PleaseCrawl
 ProPowerBot
 ProWebWalker
-Psbot
-Pump
 QueryN Metasearch
-Quick-Crawler
+RankActive
 RankFlex
 RankingBot
-Rankivabot
-RankurBot
+ReGet
 RealDownload
+RebelMouse
+Recorder
 RedesScrapy
 Ripper
 Rogerbot
-SEOkicks-Robot
+SEOkicks
+SEOprofiler
 SEOstats
-SMTBot
+SalesIntelligent
 ScanAlert
+Scanbot
+ScoutJet
 Scrapy
-Screaming
+ScreenerBot
 ScrepyBot
+Searchestate
+SearchmetricsBot
+SemrushBot
 SentiBot
-SeoSiteCheckup
 SeobilityBot
-Shodan
-Site Sucker
+Siphon
 SiteCheckerBotCrawler
-Sitebeam
-SmartDownload
+Sitevigil
+Snake
+Snapbot
 Sogou web spider
-Sosospider
+Sottopop
+SpaceBison
 Spanner
 Spbot
-Sqlmap
-Sqlworm
-Sqworm
-Stripper
-SuperBot
+Spinn3r
+SputnikBot
+Steeler
+Sucker
+Sucuri
 Surfbot
 Swiftbot
 T0PHackTeam
-Telesoft
 Telesphoreo
-The Intraformant
-Thumbor
-TightTwatBot
-Tracemyfile
-Trendictionbot
-True_Robot
+Titan
+Toata
+Trendiction
 Turingos
-TurnitinBot
-TwengaBot
-URLy.Warning
-Upflow
-Vacuum
-VeriCiteCrawler
-VidibleScraper
-WASALive-Bot
+Twice
+VB Project
+VCI
+VoidEYE
+Voltron
+WISENutbot
 WWW-Collector-E
-WWW::Mechanize
+WWW-Mechanize
 WWWOFFLE
+Wallpapers
+Wallpapers/3.0
+WallpapersHD
 WeSEE
+Web Auto
 Web Collage
-Web Enhancer
 Web Fuck
-Web Pix
-Web Sauger
-Web Sucker
-WebCopier
+WebCollage
 WebFetch
 WebGo IS
+WebImageCollector
 WebStripper
 WebWhacker
+WebZIP
 Webalta
-WebmasterWorldForumBot
-WebsiteExtractor
-Whack
-Whacker
+Website Quester
+WebsiteQuester
+Webster
+Who.is Bot
+Widow
+WinHTTrack
 Wonderbot
-Xaldon WebSpider
-Zade
+Wotbox
+Xaldon_WebSpider
 Zermelo
 Zeus
-ZoomBot
-ZumBot
-ZyBorg
+Zitebot
+ZmEu
 arquivo.pt
 backlink-check
-demandbase-bot
-evc-batch
-gopher
+clark-crawler
+coccocbot-web
+com.plumanalytics
+crawl.sogou.com
+crawler.feedback
+domainsproject.org
 heritrix
-instabid
 internetVista monitor
-lwp-trivial
-meanpathbot
-mediawords
+magpie-crawler
 netEstate NE Crawler
-pcBrowser
-plumanalytics
-probe-image-size
+page scorer
+s1z.ru
 satoristudio.net
-scan.lol
-sexsearcher
+seobility
+siteripz
 sogouspider
-sp_auditbot
-sysscan
-tAkeOut
+trendiction.de
 ubermetrics-technologies.com
-voyagerx.com
-webmeup-crawler
-zauba.io
-zgrab
+webprosbot
+x22Mozilla