V4.2021.12.2765 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-404checker
-404enemy
-AIBOT
 ALittle Client
-Aboundex
-AhrefsBot
-Aipbot
+Aboundexbot
+Acunetix
+AfD-Verbotsverfahren
 Alexibot
 AllSubmitter
 Alligator
+AlphaBot
 Anarchie
-Apexoo
-Aspiegel
 Attach
-AwarioRssBot
 AwarioSmartBot
-BBBike
-BackDoorBot
-Bitacle
-BotALot
+BDCbot
+BDFetch
+BLEXBot
+BackWeb
+BacklinkCrawler
+BatchFTP
+Battleztar Bazinga
+Bigfoot
+BlackWidow
+BlowFish
+Bolt
+Brandprotect
 Brandwatch
-Buck
-Buddy
 BuiltWith
 BuzzSumo
 CATExplorador
 CCBot
-CensysInspect
-CheTeam
+CODE87
+CSHttp
+Calculon
 CheeseBot
-Citoid
-Cliqzbot
-Cogentbot
+ChinaClaw
+Chlooe
+Cloud mapping
 Collector
-CopyRightCheck
-Craftbot
-Crawling at Home Project
+Copier
+Cosmos
 Crescent
-CrunchBot
 Curious
-CyotekWebCopy
-DSearch
-DataCha0s
-Devil
-DigitalPebble
+Custo
+Deusu
+Digincore
 Dirbuster
-Discoverybot
-DnBCrawler-Analytics
-DomainCrawler
-Download Wonder
-Dragonfly
+Discobot
+Dispatch
+DnyzBot
+DomCopBot
+DomainAppender
+DomainSigmaCrawler
+DomainStatsBot
+EMail Siphon
 EMail Wolf
 Ebingbong
-Ecxi
 Evil
-Exabot
-Express WebPictures
 ExtLinksBot
+Extractor
 ExtractorPro
 Ezooms
-FDM
 FemtosearchBot
-Fimap
 Foobot
+Freeuploader
 FyberSpider
+Fyrebot
+G-i-g-a-b-o-t
+GalaxyBot
 Genieo
 GermCrawler
+GetRight
+GetWeb
 Getintent
 Go!Zilla
-HEADMasterSEO
-HTTP::Lite
-Haansoft
+Go-Ahead-Got-It
+GoZilla
+Grabber
+GrapeFX
+HMView
+HTTrack
 HaosouSpider
-Havij
 Hloader
 IDBTE4M
-IDBot
+IRLbot
 Id-search
 IndeedBot
-Indy Library
-InfoNaviRobot
-InternetSeer
+InfoTekies
+Intelliseek
+Internet Ninja
+Iskanie
 JOC Web Spider
 Jbrofuzz
+JetCar
+Jetty
+Joomla
 Jorgee
 Jyxobot
-Keyword Density
+Kinza
 Kozmosbot
-LNSpiderguy
-LWP::Simple
-Lanshanbot
-LeechGet
-LexiBot
-Lftp
-LibWeb
-LinkScan
-LinkWalker
-Linkbot
+Larbin
+Leap
+LieBaoFast
+Likse
+Linkdexbot
 LinkextractorPro
-LinkpadBot
-LinqiaMetadataDownloaderBot
+LinksManager
 LinqiaRSSBot
-Ltx71
+LinqiaScrapeBot
+Lipperhey Spider
 MIDown tool
+MIIxpc
+MJ12bot
+MQQBrowser
 MSFrontPage
 MTRobot
-Majestic12
-MarkMonitor
+Mag-Net
+Majestic SEO
 Mata Hari
 Mb2345Browser
-Meanpathbot
-Mediatoolkitbot
+MeanPath Bot
 MegaIndex.ru
 Metauri
 MicroMessenger
-Mister PiX
-Moblie Safari
-Mozlila
-Mr.4x3
-Msrabot
-Name Intelligence
+NICErsPRO
+Nameprotect
+Nessus
 Net Vampire
-NetAnts
-NetMechanic
 NetZIP
-Netcraft
 Nettrack
 Netvibes
-NextGenSearchBot
+Niki-bot
+NimbleCrawler
+Nimbostratus
 Nutch
-OnCrawl
-OpenVAS
+Offline Navigator
+Openfind
+Openvas
 OrangeBot
+OrangeSpider
 OutclicksBot
 OutfoxBot
-PECL::HTTP
 PHPCrawl
-PageThing.com
+Page Analyzer
 Papa Foto
-Picscout
-Picsearch
-Piepmatz
+Petalbot
 Pimonster
 Pixray
+Psbot
 Pu_iN
 QueryN Metasearch
-Quick-Crawler
 RSSingBot
-Rankivabot
-RankurBot
-RealDownload
+RankActiveLinkBot
+RankingBot
+Re-re
 Reaper
-Rogerbot
-SBIder
-SEOkicks-Robot
-Scanbot
+RepoMonkey
+SEOkicks
+SEOstats
+SISTRIX
+SMTBot
+Screaming
 ScrepyBot
+SearchmetricsBot
 SemanticJuice
-Semrush
-SeoSiteCheckup
-Siphon
-SiteExplorer
-SiteSucker
-Snake
-SocialRankIOBot
-Sociscraper
-Sottopop
-SpaceBison
-Spammen
+SemrushBot
+SeobilityBot
+Site Sucker
+SiteCheckerBotCrawler
+SiteLockSpider
+SlySearch
+Snoopy
+Sogou web spider
+SpankBot
 Spanner
-Spbot
 Spinn3r
 SputnikBot
-Stripper
-SuperBot
-Surfbot
+Sqworm
+Sucker
+SuperHTTP
+SurveyBot
+Suzuran
+Swiftbot
+T0PHackTeam
 T8Abot
 Teleport
+Telesoft
 Telesphoreo
+Telesphorep
 The Intraformant
-TheNomad
+TightTwatBot
+Toata
+Toweyabot
 Tracemyfile
+Trendictionbot
 True_Robot
-TwengaBot
-Twice
+TurnitinBot
+URLy Warning
+URLy.Warning
 V-BOT
 VB Project
-Virusdie
-Voil
+Vacuum
+VelenPublicWebCrawler
+VeriCiteCrawler
 Voltron
 WASALive-Bot
-WBSearchBot
+WEBDAV
+WISENutbot
+WPScan
 WWW-Collector-E
 WWWOFFLE
 Wallpapers/3.0
 WeSEE
+Web Auto
 Web Collage
-Web Fuck
-Web Sauger
+WebAuto
 WebCopier
-WebFetch
-WebLeacher
-WebSauger
-WebWhacker
-WebZIP
-Website Quester
-WebsiteExtractor
+WebEnhancer
+WebPix
+WebReaper
+Webshag
 Webster
-Whacker
-Whatweb
+Whack
 Who.is Bot
-WinHTTrack
 Wotbox
-Xaldon WebSpider
 Xaldon_WebSpider
-ZmEu
+YoudaoBot
+Zade
+Zeus
+Zitebot
 ZyBorg
-adscanner
-arquivo.pt
 autoemailspider
-backlink-check
-clark-crawler
-coccocbot-web
-cognitiveseo
+cah.io.community
+check1.exe
+crawl.sogou.com
 crawler.feedback
-crawler4j
-demandbase-bot
+dataforseo.com
 domainsproject.org
-eCatch
-evc-batch
-heritrix
+gopher
 lwp-request
 magpie-crawler
-meanpathbot
-mediawords
-oBot
-pcBrowser
-plumanalytics
-ripz
-s1z.ru
-scalaj-http
-seobility
-siteripz
+muhstik-scan
+scan.lol
+seoscanners
 sogouspider
-spyfu
+sp_auditbot
 sysscan
-tAkeOut
+trendiction.com
 ubermetrics-technologies.com
+voyagerx.com
 webgains-bot
-xpymep1.exe
+x22Mozilla
+zgrab