V4.2021.08.2549 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404enemy
-AIBOT
-Abonti
-Aboundex
-AfD-Verbotsverfahren
-AiHitBot
-Aipbot
+360Spider
+404checker
+80legs
+ADmantX
+ALittle Client
+ASPSeek
+Acunetix
+AhrefsBot
 Alexibot
-AllSubmitter
-Anarchy
-Anthill
+Alligator
+Ankit
 Asterias
-AwarioRssBot
-BBBike
-BackDoorBot
+Attach
+AwarioSmartBot
+BDCbot
 BackStreet
-BacklinkCrawler
 Barkrowler
+BatchFTP
+BetaBot
 Bigfoot
-BlackWidow
+Blow
+Bolt
+BotALot
 Brandprotect
+Brandwatch
 Buck
-BuiltBotTough
-BuzzSumo
-CSHttp
-CazoodleBot
-Cegbfeieh
-CensysInspect
+BuiltWith
+Bullseye
+CATExplorador
 CheTeam
 CherryPicker
 ChinaClaw
-Cloud mapping
-Collector
-Copier
-CrunchBot
+Chlooe
+Cliqzbot
+Cogentbot
+Crescent
 Curious
 CyotekWebCopy
-DBLBot
 DIIbot
-DSearch
-DatabaseDriverMysqli
+Demon
+Devil
 Digincore
 Discobot
-Discoverybot
-Dispatch
 DittoSpyder
 DomCopBot
+DomainCrawler
 DomainSigmaCrawler
-Domains Project
-Dragonfly
+DomainStatsBot
+Download Wonder
 EMail Siphon
-EMail Wolf
-EasyDL
 Ebingbong
-Ecxi
-EroCrawler
-Evil
-Exabot
-ExtLinksBot
-ExtractorPro
-EyeNetIE
-Ezooms
+Extractor
+Extreme Picture Finder
+FDM
 FemtosearchBot
-Firefox/7.0
-FlashGet
-FrontPage
+Flunky
+Foobot
 Fuzz
+FyberSpider
+GT::WWW
 GalaxyBot
-GetRight
+Genieo
+GermCrawler
 Getintent
-Gigabot
+Gigablast
 Gotit
 Grafula
+GrapeFX
 GrapeshotCrawler
 GridBot
-HEADMasterSEO
 HMView
 HTTP::Lite
-Harvest
-Havij
-Hloader
-Humanlinks
-IDBTE4M
-IRLbot
-Iblog
+HybridBot
+Id-search
 Image Fetch
-IndeedBot
+Image Sucker
 InfoTekies
-InterGET
+Intelliseek
 Internet Ninja
-InternetSeer
-Iria
+Iskanie
+JennyBot
 JetCar
-JikeSpider
 Joomla
 JustView
-Keyword Density
-Larbin
-Leap
-Libwhisker
+Kenjin Spider
+Kozmosbot
+LNSpiderguy
+LWP::Simple
+Lanshanbot
+LeechFTP
+Lftp
 Likse
+LinkScan
 LinkWalker
 Linkdexbot
 LinkextractorPro
-LinqiaMetadataDownloaderBot
-Lipperhey
 Lipperhey Spider
 Ltx71
+MFC_Tear_Sample
 MSIECrawler
-Majestic12
+Mag-Net
+Magnet
+Mail.RU_Bot
+Majestic SEO
+MarkMonitor
+MarkWatch
 Masscan
 Mata Hari
-Mb2345Browser
 MeanPath Bot
+Mediatoolkitbot
 Metauri
-Mister PiX
-Mojolicious
+MicroMessenger
+Microsoft URL Control
+Moblie Safari
 Morfeus Fucking Scanner
-Mozlila
-Name Intelligence
+Mr.4x3
+Musobot
+Nameprotect
+NearSite
+Needle
 Nessus
-Net Vampire
-NetSpider
-NetZIP
-Nettrack
-Netvibes
+NetAnts
+NextGenSearchBot
 Nibbler
-Niki-bot
 Nikto
+NimbleCrawler
 Nimbostratus
-Nuclei
-Octopus
+Ninja
+Offline Explorer
+Offline Navigator
 OnCrawl
-Openvas
+OpenLinkProfiler
+OrangeSpider
 OutclicksBot
-OutfoxBot
-PECL::HTTP
 PHPCrawl
-Page Analyzer
+PageAnalyzer
+PageScorer
+PageThing.com
 Panscient
-Papa Foto
-Pi-Monster
 Picscout
 Piepmatz
-Pimonster
-PleaseCrawl
-Pockey
+Pixray
 ProPowerBot
+Probethenet
 Psbot
 Pu_iN
-PxBroker
-QueryN Metasearch
-RankActive
 RankActiveLinkBot
-RankingBot
+RankFlex
 RankingBot2
+Rankivabot
+RankurBot
 Re-re
-RealDownload
-Recorder
+Reaper
 RedesScrapy
-RepoMonkey
-Ripper
+Rogerbot
 SBIder
-SEOkicks-Robot
-SISTRIX
-ScoutJet
+SEOlyticsCrawler
+SalesIntelligent
 Screaming
-ScreenerBot
 SearchmetricsBot
-SemrushBot
-Shodan
+SentiBot
+SiteCheckerBotCrawler
+SiteExplorer
+SiteSnagger
 Siteimprove
-Sociscraper
-Sogou web spider
-Sosospider
+SlySearch
+Snake
+Snapbot
+SpaceBison
 Spbot
-Spinn3r
-Sucuri
+SputnikBot
+Steeler
+Stripper
 SuperBot
-Swiftbot
-T0PHackTeam
-T8Abot
+Surfbot
+Suzuran
 Teleport
+TeleportPro
+Telesphoreo
+Thumbor
 Titan
-Toata
-Toweyabot
+Tracemyfile
+Trendiction
 True_Robot
-Turnitin
-TwengaBot
 URLy.Warning
 Upflow
-VeriCiteCrawler
-VidibleScraper
+V-BOT
+VB Project
+Voil
 Voltron
 WISENutbot
-WWW::Mechanize
+WWW-Collector-E
 WWWOFFLE
-Wallpapers
-Wallpapers/3.0
-WeSEE
 Web Collage
 Web Enhancer
 Web Fetch
-Web Fuck
-Web Pix
 Web Sauger
 WebBandit
-WebFuck
-WebImageCollector
+WebEnhancer
+WebFetch
+WebLeacher
+WebReaper
+WebSucker
+Webshag
 Website Quester
-Webster
+WebsiteExtractor
+WebsiteQuester
 Whack
-Whatweb
+Whacker
+Who.is Bot
 WiseGuys Robot
+Wonderbot
 Woobot
 Xaldon WebSpider
+Xaldon_WebSpider
 Xenu
-YoudaoBot
-Zade
-Zitebot
-ZoomBot
-ZumBot
+ZmEu
 archive.org_bot
 arquivo-web-crawler
-check1.exe
-clark-crawler
+autoemailspider
+backlink-check
 coccocbot-web
 cognitiveseo
 crawler4j
-demandbase-bot
-facebookscraper
+domainsproject.org
+evc-batch
 heritrix
-instabid
+internetVista monitor
+isitwp.com
 lwp-trivial
+meanpathbot
 mediawords
-page scorer
-probe-image-size
+oBot
+ripz
 satoristudio.net
 scalaj-http
 scan.lol
 seoscanners
 seostar
-serpstatbot
-siteripz
-sogouspider
+sitechecker.pro
 sysscan
 ubermetrics-technologies.com
+voyagerx.com
+webmeup-crawler
+webpros.com
 webprosbot
 x22Mozilla
+zauba.io
+zgrab