V4.2021.08.2498 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-ADmantX
+360Spider
+80legs
 AIBOT
+ALittle Client
 Abonti
 Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
-AhrefsBot
-AiHitBot
-Alexibot
+AlphaBot
+Anarchie
 Ankit
+Anthill
+Aspiegel
 Asterias
-AwarioSmartBot
-BDCbot
+Attach
+AwarioRssBot
+BBBike
 BDFetch
+BLEXBot
 BackDoorBot
-BacklinkCrawler
 Barkrowler
 BatchFTP
-Battleztar Bazinga
-BetaBot
-Blow
+Black Hole
+BlackWidow
 BlowFish
-Boardreader
-Bolt
-BotALot
-Brandwatch
-Buddy
-BunnySlippers
+BuiltBotTough
+BuiltWith
+BuzzSumo
 CATExplorador
+CCBot
 CODE87
 Calculon
-CazoodleBot
-CensysInspect
 CheTeam
-Chlooe
+CherryPicker
 Cloud mapping
 Cogentbot
-Copier
-CopyRightCheck
-Copyscape
-CrazyWebCrawler
-CyotekWebCopy
+Collector
+Cosmos
+Craftbot
+CrunchBot
+Curious
 DataCha0s
-Demon
-Deusu
-Devil
+DatabaseDriverMysqli
 Digincore
-DigitalPebble
 Dirbuster
 Discobot
-DomCopBot
+DittoSpyder
+DomainAppender
 DomainCrawler
-DomainStatsBot
+DomainSigmaCrawler
 Dragonfly
 ECCP/1.0
-EasyDL
-Ecxi
-Exabot
+EMail Wolf
+Ebingbong
+Express WebPictures
 Extractor
 ExtractorPro
+Extreme Picture Finder
 EyeNetIE
+Ezooms
+FDM
 FHscan
-Firefox/7.0
-Flunky
-FrontPage
+FlashGet
+Freeuploader
 Fuzz
-FyberSpider
 Fyrebot
-G-i-g-a-b-o-t
 GermCrawler
-GetRight
-GetWeb
 Getintent
+Gigabot
+GoZilla
 Gotit
 Grabber
+GrapeFX
+GridBot
+HEADMasterSEO
 HMView
-HTTP::Lite
-HTTrack
-Haansoft
-HaosouSpider
-Heritrix
-Hloader
-IDBTE4M
+Havij
+IRLbot
+Iblog
+Id-search
 Image Fetch
 Image Sucker
-Indy Library
 InfoNaviRobot
-Intelliseek
+Iskanie
 IstellaBot
-JetCar
-Jorgee
+Joomla
 Kenjin Spider
-Keyword Density
-Kozmosbot
+Kinza
+LWP::Simple
+Lanshanbot
+Larbin
 LeechFTP
 LeechGet
 LexiBot
-Lftp
-LibWeb
 Libwhisker
-LieBaoFast
-Likse
 Linkbot
+LinqiaMetadataDownloaderBot
 LinqiaScrapeBot
-Lipperhey Spider
+Lipperhey
 Litemage_walker
-Lmspider
-MFC_Tear_Sample
-MIDown tool
-MIIxpc
 MSFrontPage
-MTRobot
-Mag-Net
+Magnet
 Mail.RU_Bot
+MarkMonitor
 MarkWatch
-Masscan
 Mata Hari
-Microsoft URL Control
-Mister PiX
-Morfeus Fucking Scanner
+MauiBot
+Mb2345Browser
+MeanPath Bot
+Meanpathbot
+Metauri
+MicroMessenger
+Microsoft Data Access
+Mojolicious
+Mozlila
 Mr.4x3
-Musobot
+Msrabot
 NICErsPRO
+Name Intelligence
 NearSite
-Needle
 NetLyzer
+NetMechanic
+NetZIP
 Netcraft
-Nettrack
-Nimbostratus
+Netvibes
+Nibbler
 Ninja
+Nmap
 Nuclei
+Offline Explorer
 Offline Navigator
-Openfind
-PageScorer
-PageThing.com
+OrangeSpider
+OutclicksBot
+PHPCrawl
+Page Analyzer
 Pavuk
-Petalbot
-Picsearch
-Piepmatz
-PleaseCrawl
-Pockey
+Picscout
+PictureFinder
+Pimonster
+Pixray
 ProPowerBot
 ProWebWalker
-Probethenet
-Pump
 PxBroker
-RSSingBot
+QueryN Metasearch
+Quick-Crawler
 RankActiveLinkBot
-RankFlex
 RankingBot
 RankingBot2
-RankurBot
-Re-re
+Rankivabot
 ReGet
-Reaper
-RebelMouse
 RepoMonkey
+Ripper
+RocketCrawler
+Rogerbot
+SBIder
+SEOkicks
+SEOkicks-Robot
+SEOprofiler
 SMTBot
 ScanAlert
-Scanbot
-ScreenerBot
+ScoutJet
 Searchestate
 Semrush
-SeobilityBot
-Shodan
-Site Sucker
-SiteLockSpider
-Sitebeam
-Siteimprove
-SlySearch
-Snake
-Snapbot
+SemrushBot
+SentiBot
+SeoSiteCheckup
+Seomoz
+Siphon
+SiteSnagger
+SiteSucker
+SocialRankIOBot
+Sociscraper
 Sogou web spider
 Sosospider
-SpaceBison
 SpankBot
-Spanner
-Sqworm
+SputnikBot
 Stripper
-Swiftbot
-Szukacz
+SuperBot
+SuperHTTP
+Surfbot
+Suzuran
 Teleport
-The Intraformant
+Telesoft
+Telesphoreo
 TheNomad
-Thumbor
+TightTwatBot
 Toata
+Toweyabot
 Tracemyfile
-Trendiction
-Turnitin
-Typhoeus
-URLy.Warning
-UnisterBot
+Trendictionbot
+True_Robot
+TurnitinBot
+Upflow
 VB Project
-Vacuum
 Vagabondo
+VeriCiteCrawler
 VidibleScraper
 Voltron
-WASALive-Bot
-WBSearchBot
-WISENutbot
-WPScan
-Wallpapers
-Web Enhancer
-Web Fuck
-Web Pix
-Web Sucker
+WEBDAV
+WWW-Collector-E
+WWW::Mechanize
+WWWOFFLE
+Wallpapers/3.0
+WeSEE
+Web Collage
+Web Sauger
 WebAuto
-WebLeacher
-WebReaper
+WebBandit
+WebCollage
+WebCopier
+WebFetch
+WebPix
+WebSauger
 WebStripper
-Website Quester
+Webalta
 WebsiteExtractor
-WebsiteQuester
-Webster
-Who.is Bot
-Xaldon_WebSpider
-Xenu
+Whatweb
+WiseGuys Robot
+Woobot
+Wotbox
 YoudaoBot
 Zade
+Zeus
 Zitebot
-ZmEu
-ZoominfoBot
-ZyBorg
-autoemailspider
-coccocbot-web
-crawl.sogou.com
-crawler4j
-domainsproject.org
+arquivo.pt
+demandbase-bot
+facebookscraper
+gopher
+instabid
 internetVista monitor
 ips-agent
+isitwp.com
+lwp-request
 magpie-crawler
-meanpathbot
+mediawords
 muhstik-scan
-netEstate NE Crawler
-page scorer
-polaris version
+oBot
+pcBrowser
 probe-image-size
 satoristudio.net
-seoscanners
-sexsearcher
-sitechecker.pro
-siteripz
-sp_auditbot
+scalaj-http
+scan.lol
+seostar
+sogouspider
 spyfu
-tAkeOut
+sysscan
 trendiction.com
-trendiction.de
-ubermetrics-technologies.com
-voyagerx.com
+webmeup-crawler
+x09Mozilla
+xpymep1.exe