V4.2021.09.2611 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
+01h4x.com
+360Spider
+AIBOT
 ALittle Client
-ASPSeek
-Abonti
-Acunetix
+Aboundex
+Aboundexbot
 AfD-Verbotsverfahren
-Aipbot
-Alexibot
-AllSubmitter
-Ankit
-Aspiegel
+AhrefsBot
+AiHitBot
+AlphaBot
+Anarchy99
 Asterias
-Attach
 AwarioRssBot
+AwarioSmartBot
+BDCbot
 BDFetch
-BLEXBot
-BackDoorBot
-Backlink-Ceck
+BackStreet
+BacklinkCrawler
 Badass
-Battleztar Bazinga
-Blackboard
-Boardreader
+Barkrowler
+BetaBot
+Black Hole
+BlackWidow
+Bolt
+Brandprotect
+Buck
 Bullseye
-BunnySlippers
-CATExplorador
-CensysInspect
-CheTeam
-CherryPicker
-Chlooe
+CheeseBot
+ChinaClaw
 Claritybot
-Cloud mapping
+Cliqzbot
 Cogentbot
-Copyscape
-Cosmos
-CrazyWebCrawler
+Copier
+Craftbot
+Crawling at Home Project
+Crescent
+CrunchBot
+Curious
 Custo
-DBLBot
-DataCha0s
-DatabaseDriverMysqli
-Deusu
-Devil
-Digincore
-Disco
-Dispatch
-DnyzBot
+DSearch
+DigitalPebble
+Discobot
 DomainAppender
 DomainCrawler
+DomainSigmaCrawler
+Domains Project
+Dotbot
 Download Wonder
 Dragonfly
-Drip
-ECCP/1.0
+EMail Siphon
+EMail Wolf
 EasyDL
 EirGrabber
-Exabot
+EroCrawler
 ExtLinksBot
 ExtractorPro
 Ezooms
-FHscan
 Firefox/7.0
-FlashGet
-Freeuploader
-FrontPage
-Fuzz
-GT::WWW
+Flunky
+Foobot
+FyberSpider
+Genieo
+GetWeb
 Getintent
-GoZilla
-Gotit
-Grafula
+GrabNet
+Grabber
 GrapeFX
 GrapeshotCrawler
-GridBot
-HMView
+HEADMasterSEO
 HTTP::Lite
-HaosouSpider
-Harvest
+HTTrack
+Havij
 Heritrix
-Humanlinks
-HybridBot
 IDBTE4M
-Iblog
-Id-search
-IndeedBot
-Intelliseek
+Image Fetch
+InfoTekies
 InternetSeer
 Iria
-Jbrofuzz
-Jorgee
+IstellaBot
+JetCar
+Joomla
 Jyxobot
+Keyword Density
+Kinza
+LNSpiderguy
+LWP::Simple
+Lanshanbot
 Larbin
 LeechFTP
-Lftp
-LibWeb
-Libwhisker
-LieBaoFast
-Likse
+LexiBot
+Lightspeedsystems
 LinkWalker
+Linkbot
 LinkextractorPro
+LinkpadBot
+LinksManager
 LinqiaMetadataDownloaderBot
-LinqiaScrapeBot
+LinqiaRSSBot
 Lipperhey
-Lipperhey Spider
 Lmspider
-MJ12bot
+Ltx71
+MIDown tool
+MSFrontPage
+MSIECrawler
 Mag-Net
-Magnet
-Majestic12
-Mass Downloader
+Mail.RU_Bot
+Majestic SEO
+MarkMonitor
 Mb2345Browser
-Mediatoolkitbot
 Metauri
 Microsoft Data Access
-Microsoft URL Control
-Mister PiX
-Mojeek
-Mr.4x3
-NearSite
+Mojolicious
+Morfeus Fucking Scanner
+Musobot
+NICErsPRO
+Nameprotect
 Needle
 Nessus
-Net Vampire
-NetAnts
 NetLyzer
 NetMechanic
 NetSpider
-NextGenSearchBot
 Nibbler
 Niki-bot
-NimbleCrawler
-Ninja
-Nutch
-Offline Explorer
-OpenLinkProfiler
+Nimbostratus
+Nmap
+Octopus
+Offline Navigator
 OpenVAS
-OrangeSpider
+Openvas
+OrangeBot
 OutclicksBot
+OutfoxBot
+PECL::HTTP
 PHPCrawl
 POE-Component-Client-HTTP
+Page Analyzer
+PageThing.com
 Pandalytics
-Panscient
 Papa Foto
-Pavuk
 PeoplePal
+Pi-Monster
 Picscout
-Piepmatz
-Pixray
+Picsearch
 PleaseCrawl
-ProPowerBot
 Psbot
-Pu_iN
+Pump
+PxBroker
 PyCurl
-RSSingBot
+QueryN Metasearch
 RankActive
 RankFlex
-RebelMouse
+RankingBot
+Reaper
+Recorder
 RepoMonkey
+RocketCrawler
+Rogerbot
 SBIder
-SEOkicks
+SEOkicks-Robot
 SEOlyticsCrawler
+SEOprofiler
 SEOstats
-Scanbot
-ScoutJet
+SalesIntelligent
+ScreenerBot
 ScrepyBot
-SemrushBot
-SeobilityBot
+Searchestate
+SeoSiteCheckup
 Seomoz
+Shodan
 Siphon
+Site Sucker
+SiteSnagger
 Sitebeam
+Siteimprove
 Sitevigil
-SlySearch
 SmartDownload
-Snake
-Snoopy
-Sociscraper
 Sogou web spider
-Sottopop
+Sosospider
 Spammen
 SpankBot
-Spbot
+Spanner
+Spinn3r
 SputnikBot
-Sqlmap
+Sqworm
+Steeler
 Stripper
 Sucker
-SuperBot
+SuperHTTP
 Surfbot
-SurveyBot
-Teleport
-Telesphorep
+T8Abot
+TeleportPro
+Telesoft
+Titan
 Toata
-Tracemyfile
-Trendictionbot
-Turingos
+Turnitin
 TurnitinBot
 Twice
-Typhoeus
-URLy Warning
-UnisterBot
-V-BOT
-VB Project
+URLy.Warning
+Upflow
 VelenPublicWebCrawler
-VeriCiteCrawler
-Virusdie
 VoidEYE
-WWW-Collector-E
+WASALive-Bot
+WBSearchBot
 WWW-Mechanize
-WWWOFFLE
-Wallpapers/3.0
-Web Auto
-WebCopier
-WebEnhancer
-WebGo IS
+Web Sucker
+WebAuto
+WebFetch
+WebLeacher
 WebReaper
 WebSauger
-WebWhacker
-WebmasterWorldForumBot
-Webster
-Who.is Bot
-WiseGuys Robot
-Wonderbot
-Xaldon_WebSpider
-Xenu
+WebSucker
+WebZIP
+Webshag
+WebsiteExtractor
+Widow
+Woobot
+Wprecon
+Xaldon WebSpider
+Zade
 Zauba
 Zermelo
 Zeus
 Zitebot
-ZyBorg
+ZmEu
+ZoomBot
 archive.org_bot
-autoemailspider
-cah.io.community
-check1.exe
+backlink-check
 cognitiveseo
 crawler4j
 demandbase-bot
 domainsproject.org
-eCatch
-facebookscraper
-instabid
-ips-agent
-lwp-request
-mediawords
-muhstik-scan
-pcBrowser
-plumanalytics
-polaris version
-scalaj-http
-seoscanners
-serpstatbot
-sitechecker.pro
+evc-batch
+heritrix
+internetVista monitor
+magpie-crawler
+meanpathbot
+ripz
+scan.lol
+seobility
+sexsearcher
+siteripz
 sogouspider
-spyfu
-voyagerx.com
+sysscan
+tAkeOut
+ubermetrics-technologies.com
 webmeup-crawler
+webpros.com
+webprosbot
+x22Mozilla
 zauba.io