V4.2022.04.3018 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-360Spider
-80legs
-AIBOT
-ASPSeek
+ADmantX
+Abonti
 Aboundex
-Acunetix
 AfD-Verbotsverfahren
-AiHitBot
-Aipbot
 Alexibot
+Alligator
 Anarchie
-Anarchy
 Anarchy99
 Ankit
+Anthill
 Aspiegel
+Asterias
+AwarioRssBot
 AwarioSmartBot
-BBBike
 BDCbot
-BDFetch
-Backlink-Ceck
-BatchFTP
-Bigfoot
-Blow
-BlowFish
+BackStreet
+BackWeb
+BacklinkCrawler
+Battleztar Bazinga
+Black Hole
+BlackWidow
 Boardreader
-Bolt
 Brandprotect
 Brandwatch
+Buck
+BuiltWith
+Bullseye
 BunnySlippers
 BuzzSumo
 CATExplorador
+CODE87
 CSHttp
-Chlooe
-Citoid
-Cliqzbot
-Cloud mapping
-Collector
+CheTeam
+CherryPicker
 Copier
+CopyRightCheck
 Copyscape
-Cosmos
-Custo
-CyotekWebCopy
+Crawling at Home Project
+CrazyWebCrawler
 DBLBot
-DSearch
-DTS Agent
-DatabaseDriverMysqli
-Discobot
-Dragonfly
+DIIbot
+Demon
+Devil
+Discoverybot
+DnyzBot
+DomCopBot
+DomainAppender
+Domains Project
+Dotbot
+Download Wonder
 ECCP/1.0
-Ebingbong
+EMail Wolf
+Ecxi
 EroCrawler
-Express WebPictures
-ExtractorPro
-Extreme Picture Finder
-EyeNetIE
-Ezooms
+Evil
+Exabot
+ExtLinksBot
+Extractor
 FHscan
 Flunky
-FrontPage
-FyberSpider
+Foobot
+Fyrebot
+G-i-g-a-b-o-t
+GT::WWW
 Genieo
+GetRight
 GetWeb
-Gotit
-GrabNet
+Getintent
+Gigabot
+Go-Ahead-Got-It
 Grabber
 GrapeFX
-GrapeshotCrawler
-HEADMasterSEO
-HMView
-HTMLparser
+HTTP::Lite
+Harvest
 Heritrix
-Hloader
-Humanlinks
 HybridBot
 IDBTE4M
 Iblog
+Id-search
 IndeedBot
-InterGET
+InfoTekies
 Internet Ninja
 InternetSeer
-Iria
 IstellaBot
+Jbrofuzz
+JennyBot
 JetCar
+JikeSpider
 Joomla
-JustView
-Kenjin Spider
-Keyword Density
-LNSpiderguy
-LWP::Simple
-Lanshanbot
-LeechGet
-Lftp
+Keybot Translation-Search-Machine
+Kozmosbot
+Leap
+LexiBot
 LibWeb
 Libwhisker
-Lightspeedsystems
 Likse
-LinkWalker
+LinkScan
 LinkpadBot
-Lipperhey
-Litemage_walker
+LinksManager
+LinqiaMetadataDownloaderBot
+LinqiaScrapeBot
+Lipperhey Spider
 Lmspider
 Ltx71
-MFC_Tear_Sample
-MJ12bot
+MIDown tool
+MIIxpc
 MQQBrowser
-MSFrontPage
+MSIECrawler
 MTRobot
-Mag-Net
 Magnet
-Mail.RU_Bot
-Majestic-SEO
-MarkMonitor
+Majestic SEO
+Majestic12
+MarkWatch
 Masscan
 MauiBot
-Mb2345Browser
-MeanPath Bot
+Meanpathbot
 Mediatoolkitbot
 MegaIndex.ru
-MicroMessenger
+Metauri
 Microsoft URL Control
+Minefield
 Moblie Safari
+Mojeek
 MolokaiBot
-Morfeus Fucking Scanner
 Mozlila
-Nameprotect
-Navroad
+NPbot
+Needle
 Nessus
-NetLyzer
 NetMechanic
+NetSpider
 Netcraft
-NextGenSearchBot
-Nikto
-Nimbostratus
+Nettrack
+Netvibes
+Nibbler
+Niki-bot
+NimbleCrawler
+Ninja
+Nmap
 Nutch
-Octopus
-Offline Explorer
+Offline Navigator
 OpenVAS
-Openfind
 OutfoxBot
 PHPCrawl
+POE-Component-Client-HTTP
 Page Analyzer
 PageGrabber
-PageScorer
 PageThing.com
-Panscient
 Papa Foto
+Pavuk
+PeoplePal
 Petalbot
+Pi-Monster
 Picscout
-Pockey
+Piepmatz
+Pimonster
+Pixray
+Probethenet
 Pu_iN
-Quick-Crawler
+PxBroker
 RankActive
+RankFlex
 RankingBot
-Re-re
-ReGet
-RealDownload
-Ripper
-Rogerbot
-SBIder
-SEOlyticsCrawler
-SEOstats
-SISTRIX
+Rankivabot
+SMTBot
 SalesIntelligent
-Scrapy
-ScreenerBot
+ScanAlert
 ScrepyBot
+Searchestate
+Semrush
 SemrushBot
+SeoSiteCheckup
 SeobilityBot
-Shodan
 Siphon
+Site Sucker
+SiteExplorer
 SiteLockSpider
-SlySearch
+Sitebeam
+Sitevigil
+Snapbot
 SocialRankIOBot
-Sosospider
-SpaceBison
-Spammen
-Spanner
-SputnikBot
-Sucker
-Sucuri
-Suzuran
+SpankBot
+Steeler
 Swiftbot
-T0PHackTeam
+Szukacz
 Telesoft
 Telesphoreo
-Thumbor
+Telesphorep
 TightTwatBot
 Titan
-Trendiction
-Trendictionbot
-TurnitinBot
-Typhoeus
-URLy.Warning
-Upflow
-VB Project
+Toata
+Turnitin
+Twice
+URLy Warning
+UnisterBot
+V-BOT
 VCI
-Vagabondo
-Virusdie
-VoidEYE
+VeriCiteCrawler
+Voltron
 WEBDAV
-WWW-Collector-E
-WWW::Mechanize
+WPScan
 WWWOFFLE
-Wallpapers
-WeSEE
-Web Enhancer
-Web Fetch
+Wallpapers/3.0
+WallpapersHD
+Web Auto
 Web Fuck
 Web Pix
-WebEnhancer
-WebFetch
-WebReaper
-WebmasterWorldForumBot
-Whack
-Widow
+WebBandit
+WebCopier
+WebGo IS
+WebLeacher
+WebSauger
+WebSucker
+WebWhacker
+WebsiteExtractor
+WebsiteQuester
+Whatweb
+WinHTTrack
 Wonderbot
-Wprecon
-Xenu
-ZoominfoBot
+Woobot
+Zade
+Zermelo
+Zeus
+Zitebot
 ZumBot
 adscanner
+archive.org_bot
 arquivo-web-crawler
 arquivo.pt
 autoemailspider
+backlink-check
+cah.io.community
 check1.exe
-clark-crawler
-coccocbot
-cognitiveseo
 crawl.sogou.com
-crawler.feedback
-eCatch
-evc-batch
+gopher
 instabid
-ips-agent
 iubenda-radar
-lwp-request
-magpie-crawler
-netEstate NE Crawler
-page scorer
-polaris version
-ripz
-satoristudio.net
-seoscanners
+lwp-trivial
+plumanalytics
+s1z.ru
+scalaj-http
+seobility
 seostar
 sexsearcher
-sitechecker.pro
-siteripz
-tAkeOut
-webprosbot
-xpymep1.exe
+spyfu
+sysscan
+webmeup-crawler
+webpros.com
+x09Mozilla
+x22Mozilla
+zauba.io