V4.2021.12.2813 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-ADmantX
+404checker
+80legs
+ALittle Client
 Abonti
+Aboundex
 AhrefsBot
-AiHitBot
+Aipbot
 Alexibot
-Anarchie
+AllSubmitter
+Alligator
+AlphaBot
+Apexoo
 Asterias
 Attach
+AwarioRssBot
 BBBike
-BDFetch
-BackDoorBot
+BDCbot
 BackStreet
 Backlink-Ceck
 BacklinkCrawler
 Badass
-Battleztar Bazinga
+Bandit
 BetaBot
 Bitacle
-Black Hole
-BlackWidow
-Blackboard
-Blow
-BlowFish
-Boardreader
-Bolt
+BotALot
+Brandprotect
 Brandwatch
 Buck
-Buddy
-BuiltWith
+BunnySlippers
+BuzzSumo
 CATExplorador
-CODE87
+CCBot
+Calculon
+CazoodleBot
+Cegbfeieh
 CensysInspect
-CherryPicker
-ChinaClaw
-Cogentbot
+CheTeam
+Citoid
+Claritybot
+Cliqzbot
 CopyRightCheck
 Copyscape
 CrazyWebCrawler
-Crescent
-CrunchBot
+Curious
+CyotekWebCopy
 DBLBot
 DIIbot
-DSearch
-DTS Agent
-Devil
-DigitalPebble
-Dirbuster
-Disco
-Discoverybot
-DnBCrawler-Analytics
+DataCha0s
+Deusu
+DnyzBot
 DomCopBot
-DomainCrawler
-DomainStatsBot
+Domains Project
+Dotbot
 Download Wonder
 Dragonfly
 ECCP/1.0
 EasyDL
-Ecxi
 EirGrabber
-Evil
+EroCrawler
+Exabot
 Extractor
-Extreme Picture Finder
-EyeNetIE
-Firefox/7.0
+ExtractorPro
+FHscan
+Fimap
 FlashGet
-Foobot
+Flunky
+Freeuploader
+FrontPage
+Fuzz
+GalaxyBot
 Genieo
+GermCrawler
+GetRight
+GetWeb
+Gotit
 GrabNet
-Grabber
-HEADMasterSEO
 HTTP::Lite
-Haansoft
-HaosouSpider
-Humanlinks
+Havij
+Heritrix
+HybridBot
+IDBTE4M
+Iblog
 Id-search
-IlseBot
-Image Fetch
-Image Sucker
 IndeedBot
+InfoNaviRobot
+InfoTekies
+Intelliseek
 Iria
-JOC Web Spider
-Jbrofuzz
-Joomla
-LWP::Simple
-Lanshanbot
-Leap
-LeechFTP
+JamesBOT
+JetCar
+Kinza
+Kozmosbot
+Larbin
 LeechGet
+LexiBot
+Lftp
 Libwhisker
-LieBaoFast
+Lightspeedsystems
 LinkScan
-Linkbot
-Linkdexbot
 LinkextractorPro
-LinkpadBot
 LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaRSSBot
 Lipperhey
 Lipperhey Spider
-Litemage_walker
-Lmspider
-MJ12bot
+Ltx71
+MIDown tool
 MQQBrowser
 MSFrontPage
+MSIECrawler
 MTRobot
+Mag-Net
 Mail.RU_Bot
-Majestic SEO
-Mass Downloader
-Mata Hari
-MauiBot
+Majestic12
+MarkWatch
+Mb2345Browser
+MeanPath Bot
+Meanpathbot
 Mediatoolkitbot
+MegaIndex.ru
+Metauri
 MicroMessenger
 Microsoft URL Control
-Moblie Safari
-Morfeus Fucking Scanner
-Mr.4x3
-Musobot
-NICErsPRO
-NPbot
+Mister PiX
+Msrabot
 Name Intelligence
-Nameprotect
-NearSite
-Nessus
-Net Vampire
-Nettrack
-Nibbler
-Niki-bot
-Ninja
-Octopus
-Openfind
-Openvas
+Navroad
+NetZIP
+NextGenSearchBot
+Nikto
+Not
+Nuclei
+Offline Explorer
+Offline Navigator
+OnCrawl
+OpenLinkProfiler
+OpenVAS
+OrangeBot
 OrangeSpider
-OutfoxBot
-POE-Component-Client-HTTP
+OutclicksBot
+Page Analyzer
 PageThing.com
 Pandalytics
 Panscient
-Papa Foto
-Pavuk
 PeoplePal
 Pi-Monster
-Picsearch
+Picscout
+PictureFinder
 Pimonster
-PleaseCrawl
-Pockey
+ProPowerBot
+Probethenet
 Pu_iN
-PxBroker
+Pump
 QueryN Metasearch
 Quick-Crawler
 RSSingBot
-RankingBot2
-Reaper
-RepoMonkey
-RocketCrawler
+RankFlex
+RankingBot
+Rankivabot
+Re-re
+ReGet
+RealDownload
+Recorder
 SBIder
 SEOkicks
-SEOkicks-Robot
-SEOlyticsCrawler
-ScanAlert
-ScoutJet
-Scrapy
-Screaming
-Searchestate
+SISTRIX
+SMTBot
+SalesIntelligent
+SemanticJuice
 SemrushBot
-SentiBot
+SeoSiteCheckup
 SeobilityBot
-Site Sucker
-SiteExplorer
+Seomoz
+Shodan
+Siphon
+SiteLockSpider
 Snake
-SocialRankIOBot
-SputnikBot
-Sqlmap
-Sqworm
+Sogou web spider
+Sosospider
+Sottopop
+SpaceBison
+Spammen
+SpankBot
+Spbot
 Stripper
-Sucker
 Sucuri
-SuperHTTP
-Surfbot
-SurveyBot
 Suzuran
+Swiftbot
+T8Abot
+TeleportPro
 Telesphoreo
-TheNomad
-Thumbor
-TightTwatBot
-Titan
+Telesphorep
 Toweyabot
 Tracemyfile
-TwengaBot
+Turingos
 Twice
-URLy.Warning
-VCI
-Vagabondo
-VeriCiteCrawler
-VidibleScraper
-Virusdie
-Voil
-WASALive-Bot
-WWW-Mechanize
-WallpapersHD
+V-BOT
+VelenPublicWebCrawler
+VoidEYE
+Voltron
+WBSearchBot
+WEBDAV
+Wallpapers
 WeSEE
-Web Enhancer
-Web Fetch
-Web Sauger
-Web Sucker
+Web Fuck
+Web Pix
+WebAuto
 WebBandit
-WebEnhancer
-WebImageCollector
+WebCollage
 WebLeacher
-WebReaper
-WebWhacker
-WebZIP
-Webalta
-Webshag
+WebSauger
+WebStripper
 Website Quester
-WebsiteExtractor
-Whack
-Whacker
-Who.is Bot
-WiseGuys Robot
-Woobot
-Wotbox
 Wprecon
-YoudaoBot
+Zauba
+Zermelo
 ZoomBot
 ZumBot
-ZyBorg
-archive.org_bot
-arquivo.pt
-autoemailspider
+adscanner
+backlink-check
 check1.exe
-coccocbot-web
+clark-crawler
 cognitiveseo
 com.plumanalytics
-crawl.sogou.com
+crawler.feedback
 crawler4j
-domainsproject.org
+dataforseo.com
 evc-batch
 gopher
-heritrix
 internetVista monitor
+lwp-request
+lwp-trivial
+oBot
+pcBrowser
 plumanalytics
 ripz
-s1z.ru
-seocompany.store
+satoristudio.net
 seoscanners
+sexsearcher
 sogouspider
+sysscan
+tAkeOut
+trendiction.com
 ubermetrics-technologies.com
+voyagerx.com
+webprosbot
 x22Mozilla
-xpymep1.exe
+zauba.io
 zgrab