V4.2021.08.2539 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+01h4x.com
+80legs
 ADmantX
-AIBOT
+ALittle Client
 ASPSeek
-Aboundexbot
+Abonti
+Acunetix
+AfD-Verbotsverfahren
+AhrefsBot
+Aipbot
 Alexibot
-Anarchie
-Asterias
+Alligator
+Anarchy
+Anthill
+Aspiegel
+Attach
 AwarioRssBot
 AwarioSmartBot
+BBBike
 BDCbot
-BLEXBot
+BackStreet
 BackWeb
-Backlink-Ceck
+Badass
+Barkrowler
 BatchFTP
-Bitacle
-Blow
+BetaBot
 BlowFish
-Brandwatch
-Buddy
 Bullseye
 BuzzSumo
-CSHttp
-CheTeam
-ChinaClaw
-Cogentbot
-Copier
-CopyRightCheck
-Copyscape
-CrazyWebCrawler
-Curious
+CCBot
+CODE87
+Calculon
+CensysInspect
+CheeseBot
+Cliqzbot
 Custo
 DIIbot
 DSearch
 DataCha0s
 DatabaseDriverMysqli
 Deusu
-Devil
-Dirbuster
 Disco
+Discobot
+Discoverybot
+DittoSpyder
 DomainCrawler
 DomainStatsBot
+Domains Project
+Dotbot
 Download Wonder
-Drip
-ECCP/1.0
+Dragonfly
 EMail Siphon
-EMail Wolf
+EasyDL
+Ebingbong
 Ecxi
 EroCrawler
 Evil
 Extreme Picture Finder
-FDM
+Ezooms
+FHscan
 FemtosearchBot
-Fimap
 Firefox/7.0
-Foobot
-Fuzz
-Fyrebot
-GetRight
-GetWeb
+FrontPage
+FyberSpider
+G-i-g-a-b-o-t
 Getintent
 Gigablast
 Go!Zilla
-GrapeFX
-HTMLparser
-HTTP::Lite
-HTTrack
+Go-Ahead-Got-It
+Grabber
+HEADMasterSEO
+HMView
 Humanlinks
-HybridBot
-IDBTE4M
-IndeedBot
-Indy Library
+InfoNaviRobot
 InfoTekies
-Internet Ninja
 InternetSeer
-Iria
 Iskanie
-IstellaBot
-JamesBOT
 Jetty
-JikeSpider
-Jyxobot
-Kenjin Spider
+Jorgee
+Keyword Density
 Kinza
-Kozmosbot
-LWP::Simple
+Lanshanbot
+Leap
+LibWeb
 Libwhisker
 LieBaoFast
-LinkWalker
+Lightspeedsystems
+LinkScan
+Linkbot
+Linkdexbot
 LinkextractorPro
-Lipperhey Spider
+LinqiaRSSBot
+LinqiaScrapeBot
+Lipperhey
 Litemage_walker
-Lmspider
-MIIxpc
+Ltx71
+MFC_Tear_Sample
+MIDown tool
 MSIECrawler
-MTRobot
-Magnet
-Majestic SEO
-Majestic12
 MarkMonitor
-Mass Downloader
-MauiBot
+MarkWatch
+Masscan
+Mata Hari
+MeanPath Bot
+Meanpathbot
 MegaIndex.ru
+MicroMessenger
 Microsoft URL Control
-Mister PiX
-Musobot
-NICErsPRO
-NearSite
-NetLyzer
+Mojeek
+Mojolicious
+NPbot
+NetSpider
+NetZIP
 Netcraft
-Nettrack
+NextGenSearchBot
+Nibbler
+Niki-bot
 Nikto
 NimbleCrawler
 Nimbostratus
-Nuclei
-Octopus
+Ninja
+Nmap
+Nutch
 Offline Navigator
+Openfind
 Openvas
-OrangeSpider
-OutfoxBot
-PECL::HTTP
-POE-Component-Client-HTTP
-PageGrabber
+OrangeBot
+PHPCrawl
 PageScorer
-PageThing.com
 Pandalytics
 Panscient
-Papa Foto
-PeoplePal
 Petalbot
-Picscout
-Picsearch
+PictureFinder
+Piepmatz
 Pixray
-PleaseCrawl
 ProPowerBot
-ProWebWalker
 Psbot
-Pump
-QueryN Metasearch
-Quick-Crawler
-RankActive
+PyCurl
 RankFlex
 RankingBot2
-Rankivabot
-ReGet
+RankurBot
+Re-re
+RealDownload
 Reaper
+RebelMouse
 RocketCrawler
 Rogerbot
-SBIder
-SEOkicks-Robot
+SEOkicks
 SEOlyticsCrawler
 SEOprofiler
-SEOstats
-SalesIntelligent
-ScanAlert
-ScreenerBot
+SISTRIX
+SMTBot
+Scanbot
 SearchmetricsBot
 Semrush
-SentiBot
 SeoSiteCheckup
+SeobilityBot
 Seomoz
-Shodan
 Site Sucker
 SiteCheckerBotCrawler
+SiteExplorer
 SiteLockSpider
-SiteSnagger
+SiteSucker
+Sitebeam
 Sitevigil
-SmartDownload
-Snapbot
+Snake
+Snoopy
+SocialRankIOBot
 Sociscraper
-Sogou web spider
-Sottopop
+Sosospider
+Spammen
 SpankBot
-Spanner
 Spbot
-Steeler
+SputnikBot
+Sqlmap
+Sucker
 Sucuri
 SuperBot
-SuperHTTP
 Surfbot
 SurveyBot
-Suzuran
-Swiftbot
-Szukacz
-T0PHackTeam
-Teleport
-The Intraformant
-Titan
-Toweyabot
+T8Abot
+Telesoft
+Telesphoreo
+TheNomad
 Tracemyfile
 Trendiction
 Turingos
-Twice
-URLy.Warning
+Turnitin
+TurnitinBot
+UnisterBot
 Upflow
+VB Project
 VCI
-Vacuum
 Vagabondo
-VeriCiteCrawler
-Virusdie
-VoidEYE
+VelenPublicWebCrawler
+Voil
+Voltron
 WASALive-Bot
 WBSearchBot
 WISENutbot
-WPScan
-WWW::Mechanize
+WWW-Collector-E
 WWWOFFLE
-Web Pix
-Web Sauger
+WeSEE
 Web Sucker
-WebAuto
-WebImageCollector
-WebPix
+WebEnhancer
+WebFuck
+WebGo IS
+WebSauger
+WebStripper
 WebZIP
-WebmasterWorldForumBot
+WebsiteExtractor
+Webster
 Whack
-Whacker
+Whatweb
+Who.is Bot
 WiseGuys Robot
 Wonderbot
 Wotbox
-Xaldon_WebSpider
+Wprecon
 YoudaoBot
-Zade
-Zauba
 ZmEu
 ZoomBot
-arquivo.pt
+ZyBorg
+archive.org_bot
+arquivo-web-crawler
 autoemailspider
+cah.io.community
 check1.exe
+cognitiveseo
+crawl.sogou.com
+crawler.feedback
+crawler4j
 demandbase-bot
-gopher
-heritrix
+eCatch
+evc-batch
 ips-agent
+lwp-request
 lwp-trivial
 mediawords
-netEstate NE Crawler
-plumanalytics
+polaris version
 probe-image-size
 ripz
-s1z.ru
 satoristudio.net
-sexsearcher
-sogouspider
+scalaj-http
+seobility
 sp_auditbot
 spyfu
-trendiction.com
-trendiction.de
-voyagerx.com
-webprosbot
-xpymep1.exe
+tAkeOut
+x09Mozilla
 zgrab