V4.2021.11.2721 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-ADmantX
+404enemy
+AIBOT
 ALittle Client
-Abonti
 Acunetix
 AfD-Verbotsverfahren
 AhrefsBot
-Alexibot
-AllSubmitter
-AlphaBot
+AiHitBot
+Alligator
 Anarchie
-Anarchy99
-Anthill
-Apexoo
 AwarioSmartBot
-BLEXBot
-BackDoorBot
 BackStreet
-BackWeb
 Backlink-Ceck
-BacklinkCrawler
+Bandit
 Barkrowler
 BatchFTP
 Battleztar Bazinga
-BetaBot
+Bigfoot
 Bitacle
+Black Hole
+BlackWidow
 Blackboard
-Blow
-Bolt
-BotALot
+Boardreader
 Brandprotect
-Buddy
-BuiltBotTough
-BuiltWith
-BuzzSumo
-CATExplorador
-CODE87
-CSHttp
+Brandwatch
+Buck
+Calculon
 ChinaClaw
 Chlooe
-Claritybot
-Craftbot
-Crawling at Home Project
+Cloud mapping
+Collector
+CopyRightCheck
+Copyscape
+Cosmos
 CrazyWebCrawler
-Curious
-Custo
-CyotekWebCopy
+Crescent
+CrunchBot
 DBLBot
 DatabaseDriverMysqli
+Demon
 Deusu
 Digincore
 Dirbuster
-Disco
 Discoverybot
-Dispatch
-DittoSpyder
-DnBCrawler-Analytics
-DnyzBot
-DomainCrawler
+DomainStatsBot
 Dotbot
-Dragonfly
+Download Wonder
+ECCP/1.0
+EMail Siphon
 EasyDL
+Ebingbong
 EirGrabber
-EroCrawler
-Exabot
-Extreme Picture Finder
-EyeNetIE
+Express WebPictures
+Extractor
+ExtractorPro
+Ezooms
 FemtosearchBot
-Firefox/7.0
-FlashGet
-FrontPage
-Fuzz
-Fyrebot
+Freeuploader
+G-i-g-a-b-o-t
 GT::WWW
-GalaxyBot
-Getintent
-GoZilla
+GermCrawler
+GetRight
+Gigabot
+Go-Ahead-Got-It
 Gotit
-Grabber
-Grafula
-GrapeFX
+GrabNet
+GrapeshotCrawler
 GridBot
-HEADMasterSEO
 HTMLparser
+HTTrack
+Haansoft
 HaosouSpider
-Harvest
-Heritrix
-HybridBot
-IDBTE4M
+Hloader
+Humanlinks
 IRLbot
+Iblog
+Image Sucker
 IndeedBot
+Indy Library
+InfoNaviRobot
+InfoTekies
 Intelliseek
-InternetSeer
-Jorgee
-Jyxobot
+InterGET
+Iria
+IstellaBot
+JOC Web Spider
+JamesBOT
+JetCar
+Keyword Density
 Kinza
-Kozmosbot
-LeechFTP
-Lftp
-LieBaoFast
-LinkScan
+LNSpiderguy
+LWP::Simple
+Lanshanbot
+LexiBot
+LibWeb
+Linkbot
 LinkextractorPro
-LinkpadBot
 LinqiaScrapeBot
 MFC_Tear_Sample
+MIDown tool
 MSFrontPage
-Mag-Net
-Magnet
-Majestic SEO
-Majestic-SEO
-Majestic12
-MarkWatch
+MSIECrawler
+MTRobot
 Masscan
-Mata Hari
 MauiBot
 MeanPath Bot
 Meanpathbot
-Mediatoolkitbot
-MicroMessenger
-Microsoft URL Control
-Mister PiX
-Mojolicious
+MegaIndex.ru
+Moblie Safari
 Morfeus Fucking Scanner
-Mozlila
 Msrabot
-NICErsPRO
+Musobot
 NPbot
 Name Intelligence
 Needle
+NetMechanic
 Nettrack
-Nimbostratus
+Nibbler
+Niki-bot
+Nikto
 Ninja
+Nmap
 Octopus
-Openfind
-OutfoxBot
+OnCrawl
+OpenLinkProfiler
+OrangeSpider
+PECL::HTTP
 PHPCrawl
+Page Analyzer
 PageAnalyzer
-PageScorer
+PageGrabber
 PageThing.com
-Pandalytics
+Panscient
 Papa Foto
-Petalbot
-PictureFinder
-Pixray
-ProWebWalker
+PeoplePal
+Pi-Monster
+Picscout
+Pimonster
+Psbot
+Pump
 PxBroker
+PyCurl
 QueryN Metasearch
-RankActive
-RankActiveLinkBot
 RankFlex
 RankingBot2
+Re-re
+RealDownload
+Reaper
+RebelMouse
+Recorder
 RedesScrapy
-RepoMonkey
-Ripper
-RocketCrawler
-SEOprofiler
+SEOlyticsCrawler
+SEOstats
+SMTBot
 ScanAlert
-Scanbot
 ScoutJet
-Scrapy
 Screaming
+ScreenerBot
 ScrepyBot
+SearchmetricsBot
 SemanticJuice
-SemrushBot
-SiteLockSpider
+Semrush
+SeobilityBot
+Seomoz
+Site Sucker
+SiteCheckerBotCrawler
+SiteSnagger
 SiteSucker
 Siteimprove
-Sitevigil
 SlySearch
-Snake
-SocialRankIOBot
 Sosospider
 SpaceBison
-Spanner
-Spbot
 SputnikBot
-Sqlworm
-Stripper
+Sqlmap
+Sqworm
+Sucker
+Sucuri
 SuperBot
 SuperHTTP
-Telesoft
+Surfbot
+T8Abot
+Teleport
+TeleportPro
 Telesphoreo
-Thumbor
-TightTwatBot
-Titan
+Telesphorep
+Toata
 Toweyabot
-Tracemyfile
-UnisterBot
+True_Robot
+TwengaBot
+Twice
+URLy.Warning
 Upflow
-VB Project
+V-BOT
 Vacuum
 VelenPublicWebCrawler
-VeriCiteCrawler
-VidibleScraper
-VoidEYE
-Voil
-Voltron
-WASALive-Bot
 WEBDAV
-WISENutbot
 WPScan
-WWW-Collector-E
-WWW-Mechanize
-Wallpapers
-Web Fuck
-Web Sauger
+WWWOFFLE
+Wallpapers/3.0
+WallpapersHD
+Web Collage
+Web Pix
 Web Sucker
+WebEnhancer
 WebGo IS
-WebImageCollector
 WebLeacher
-WebSauger
-WebStripper
+WebReaper
 WebSucker
 WebWhacker
 Webalta
+WebmasterWorldForumBot
+Website Quester
 WebsiteQuester
-Webster
+Whatweb
+Who.is Bot
 Widow
 WinHTTrack
-WiseGuys Robot
-Woobot
-Wprecon
-Xaldon WebSpider
-Xenu
-Zade
+YoudaoBot
 Zauba
 Zermelo
-Zitebot
-adscanner
-archive.org_bot
-arquivo-web-crawler
+Zeus
+ZmEu
+ZoomBot
+ZoominfoBot
 autoemailspider
-cognitiveseo
-com.plumanalytics
-dataforseo.com
-domainsproject.org
-eCatch
-evc-batch
-instabid
-isitwp.com
+check1.exe
+clark-crawler
+crawler.feedback
+ips-agent
 lwp-request
 magpie-crawler
-muhstik-scan
-polaris version
+netEstate NE Crawler
+plumanalytics
 probe-image-size
-satoristudio.net
-scalaj-http
+ripz
+s1z.ru
+scan.lol
 seoscanners
+seostar
+serpstatbot
+sitechecker.pro
+siteripz
+sogouspider
+tAkeOut
+ubermetrics-technologies.com
+voyagerx.com
 webpros.com
-x09Mozilla
+x22Mozilla
+xpymep1.exe
+zauba.io