V4.2022.04.3071 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 360Spider
-AIBOT
+404checker
+ADmantX
+ASPSeek
 Abonti
-Aboundex
 Aboundexbot
-Acunetix
-AhrefsBot
+AfD-Verbotsverfahren
+Alexibot
 AllSubmitter
-AlphaBot
-Anarchie
+Alligator
+Anarchy
+Anarchy99
+Ankit
 Apexoo
-Aspiegel
-Atomseobot
 AwarioRssBot
 AwarioSmartBot
-BBBike
-BackStreet
-BackWeb
+BDFetch
+BLEXBot
+Backlink-Ceck
+Badass
+Barkrowler
 BatchFTP
-Battleztar Bazinga
-BetaBot
-Black Hole
+Bigfoot
+BlackWidow
+BlowFish
 Boardreader
+BotALot
 Brandwatch
-Buddy
 BuiltWith
 BunnySlippers
-CODE87
+CATExplorador
+CCBot
 Calculon
+CensysInspect
 CheeseBot
 CherryPicker
-Chlooe
-Claritybot
-Cloud mapping
-Cocolyzebot
-Cogentbot
+ChinaClaw
+Citoid
 Collector
-Copyscape
-Cosmos
-Craftbot
-Crawling at Home Project
+CopyRightCheck
 CrazyWebCrawler
 CrunchBot
-DatabaseDriverMysqli
-Deusu
+Curious
+DSearch
+DataCha0s
+Digincore
+Dirbuster
 Disco
-Discoverybot
+Discobot
 Dispatch
-DittoSpyder
+DnBCrawler-Analytics
+DomainAppender
 DomainCrawler
-Download Wonder
-Dragonfly
+Drip
 EMail Siphon
-EasyDL
-Ebingbong
-EirGrabber
+Ecxi
 Evil
-Express WebPictures
-ExtLinksBot
-Extractor
+Exabot
 ExtractorPro
-Ezooms
-FHscan
+EyeNetIE
 Fimap
-FyberSpider
-G-i-g-a-b-o-t
-GermCrawler
+FlashGet
+FrontPage
+GT::WWW
+GalaxyBot
+Genieo
+GetRight
 GetWeb
+Gigabot
+Go!Zilla
 GoZilla
 Gotit
+Grabber
 Grafula
-GridBot
-HEADMasterSEO
-HMView
+HTMLparser
 HTTP::Lite
-HTTrack
 HaosouSpider
 Harvest
-Heritrix
+Havij
 HonoluluBot
-IDBot
-Id-search
+HybridBot
+IlseBot
 IndeedBot
-Iria
-Iskanie
-JOC Web Spider
-JamesBOT
-Jetty
-JikeSpider
-Keybot Translation-Search-Machine
-LNSpiderguy
-Lanshanbot
-LeechGet
-LieBaoFast
-LinkScan
+Internet Ninja
+JennyBot
+JetCar
+Jorgee
+Kinza
+LexiBot
+LibWeb
+Linkbot
 LinkpadBot
 LinksManager
-LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
-MIDown tool
-MIIxpc
-MTRobot
-Mail.RU_Bot
+LinqiaRSSBot
+Lmspider
+Ltx71
+MQQBrowser
+Majestic SEO
+Majestic-SEO
 Majestic12
-Mass Downloader
+MarkWatch
+Masscan
 Mata Hari
-MegaIndex.ru
-Metauri
+MauiBot
+Mb2345Browser
+MicroMessenger
 Microsoft Data Access
+Minefield
 Moblie Safari
-Mojolicious
-Msrabot
-NPbot
-NearSite
+Mojeek
+Morfeus Fucking Scanner
+Mozlila
+Musobot
+Nessus
 NetLyzer
-NetSpider
-NetZIP
 Netcraft
-Nikto
-NimbleCrawler
+Nettrack
+Nimbostratus
 Not
-Nutch
-Octopus
+Nuclei
+Offline Navigator
+OnCrawl
 OpenVAS
+Openvas
 OrangeBot
 OrangeSpider
+OutclicksBot
 OutfoxBot
-PECL::HTTP
-PHPCrawl
+POE-Component-Client-HTTP
+Page Analyzer
 PageGrabber
+PageScorer
+PageThing.com
 Pandalytics
 Panscient
+Papa Foto
 Pavuk
-Pi-Monster
-Picscout
-Picsearch
-PictureFinder
-Pimonster
+Piepmatz
+Pixray
 Probethenet
-Psbot
-Pump
-Quick-Crawler
-Re-re
-Reaper
-RedesScrapy
+QueryN Metasearch
+RankActive
+RankingBot
+RankingBot2
+ReGet
+RealDownload
 RepoMonkey
-SEOkicks-Robot
-SEOprofiler
-SEOstats
-SISTRIX
-Scanbot
+SEOkicks
+SEOlyticsCrawler
+SMTBot
+SalesIntelligent
 Screaming
-ScrepyBot
-Searchestate
-Seekport
+ScreenerBot
 Semrush
-SentiBot
-SeobilityBot
-Shodan
-SiteExplorer
+Seomoz
+Site Sucker
 SiteLockSpider
-SiteSnagger
 Sitevigil
+SlySearch
+SmartDownload
 Snake
 Snapbot
 Snoopy
+Sociscraper
 Sogou web spider
 Spammen
 Spanner
 SputnikBot
-Sqworm
-Steeler
-SuperBot
+Sucker
 SuperHTTP
-Surfbot
-SurveyBot
 Suzuran
 T0PHackTeam
-Telesoft
-Telesphoreo
-Turingos
+The Intraformant
+Thumbor
+TightTwatBot
+Toweyabot
+Trendiction
+Trendictionbot
+Turnitin
 Typhoeus
-Upflow
-V-BOT
-VB Project
+URLy Warning
+URLy.Warning
+UnisterBot
+Vacuum
 Vagabondo
-VelenPublicWebCrawler
-VeriCiteCrawler
 VoidEYE
-Voil
-WASALive-Bot
-WBSearchBot
-WEBDAV
 WPScan
-WWW-Collector-E
+WWW::Mechanize
+WWWOFFLE
 Wallpapers
-WeSEE
-Web Collage
-Web Fuck
-Web Pix
+Web Fetch
+WebAuto
 WebBandit
-WebCollage
-WebEnhancer
+WebFetch
 WebFuck
-WebSauger
+WebGo IS
+WebImageCollector
+WebLeacher
+WebPix
 WebStripper
-Webshag
+WebSucker
+WebWhacker
+WebZIP
+Webalta
+WebsiteExtractor
 Webster
 Whacker
-Whatweb
 Widow
-WiseGuys Robot
+WinHTTrack
+Wonderbot
 Wotbox
-Xaldon WebSpider
-Zeus
+Xaldon_WebSpider
+Zade
 Zitebot
 ZmEu
-ZoominfoBot
+ZumBot
 ZyBorg
-archive.org_bot
-arquivo-web-crawler
-arquivo.pt
-com.plumanalytics
-crawler.feedback
-facebookscraper
-instabid
+autoemailspider
+cah.io.community
+clark-crawler
+crawl.sogou.com
+dataforseo.com
+demandbase-bot
+domainsproject.org
+eCatch
+evc-batch
+gopher
+heritrix
 internetVista monitor
-ips-agent
-isitwp.com
 linkdexbot
-magpie-crawler
-mediawords
+lwp-request
+meanpathbot
+muhstik-scan
+netEstate NE Crawler
 page scorer
 pcBrowser
-scalaj-http
-scan.lol
-seobility
-seoscanners
+s1z.ru
+satoristudio.net
+seostar
 serpstatbot
-sitechecker.pro
-siteripz
 sogouspider
-sysscan
-tAkeOut
+sp_auditbot
+trendiction.com
+trendiction.de
+ubermetrics-technologies.com
 voyagerx.com
+webgains-bot
+webmeup-crawler
 webpros.com
-x09Mozilla
 x22Mozilla
-zauba.io
-zgrab