V4.2021.12.2811 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
-01h4x.com
+80legs
+AIBOT
 ALittle Client
+ASPSeek
 Aboundex
-Acunetix
-AiHitBot
+AfD-Verbotsverfahren
+Aipbot
+Alexibot
 AllSubmitter
 Alligator
 Anarchy
-Anthill
-Aspiegel
-Asterias
+Anarchy99
+Ankit
+Apexoo
 AwarioRssBot
-BBBike
+BDFetch
 BackDoorBot
+BackWeb
+Backlink-Ceck
 BacklinkCrawler
 Badass
-Bandit
 Barkrowler
 BetaBot
 Bitacle
 BlackWidow
+Blackboard
 Blow
-Boardreader
 BotALot
 Brandprotect
-Brandwatch
+Buck
 Buddy
 BuiltWith
-BunnySlippers
-BuzzSumo
-CATExplorador
-CCBot
-CODE87
 Calculon
 CensysInspect
 CheTeam
-ChinaClaw
+CherryPicker
+Citoid
 Claritybot
+Cliqzbot
 Cloud mapping
-Cocolyzebot
-Copier
+Cogentbot
 Copyscape
 Cosmos
+Crawling at Home Project
+CrazyWebCrawler
+Custo
 CyotekWebCopy
-DTS Agent
-Deusu
-Devil
-Digincore
+DatabaseDriverMysqli
 DigitalPebble
 Dirbuster
 Dispatch
-DomainCrawler
+DittoSpyder
+DnBCrawler-Analytics
+DomainStatsBot
+Domains Project
+Dotbot
 Download Wonder
-ECCP/1.0
+EMail Siphon
+EMail Wolf
 EasyDL
-Ebingbong
-Exabot
 ExtLinksBot
-FemtosearchBot
 Firefox/7.0
 FlashGet
-Fuzz
-FyberSpider
+Fyrebot
+GalaxyBot
+Genieo
 Gigabot
-GoZilla
-Gotit
-Grafula
-GrapeshotCrawler
-Harvest
+Go!Zilla
+Go-Ahead-Got-It
+GrapeFX
+HEADMasterSEO
+HTMLparser
+HTTrack
+Haansoft
+HaosouSpider
 Havij
+Heritrix
+HybridBot
+IDBTE4M
 IRLbot
-Iblog
 Id-search
-Image Sucker
-InfoTekies
+IndeedBot
+InfoNaviRobot
 Intelliseek
 InternetSeer
+Iria
 Iskanie
-IstellaBot
-JOC Web Spider
-JamesBOT
 Jbrofuzz
-JennyBot
-JetCar
 Joomla
-Jorgee
 Jyxobot
+Kenjin Spider
 Kinza
-LNSpiderguy
 LWP::Simple
-Larbin
-Libwhisker
-LieBaoFast
+LexiBot
+LibWeb
 Lightspeedsystems
-Likse
-LinkScan
-Linkbot
+LinkextractorPro
+LinkpadBot
 LinqiaMetadataDownloaderBot
+Lipperhey
+Lmspider
 Ltx71
-MFC_Tear_Sample
-MIDown tool
-MIIxpc
-MQQBrowser
-Mag-Net
-Majestic12
+Magnet
+Mail.RU_Bot
+Majestic SEO
 MarkMonitor
-Masscan
-MauiBot
 Mb2345Browser
 Meanpathbot
 Mediatoolkitbot
-Microsoft Data Access
-Microsoft URL Control
-Minefield
+MicroMessenger
+Moblie Safari
 Mojeek
-Mojolicious
-Morfeus Fucking Scanner
-Musobot
-NPbot
-Needle
+Mozlila
+Mr.4x3
+NICErsPRO
+Name Intelligence
+Nameprotect
 Net Vampire
+NetAnts
+NetLyzer
+NetMechanic
 NetSpider
 Netcraft
-Nettrack
-Netvibes
-NextGenSearchBot
-Nibbler
+Nimbostratus
 Ninja
-Nuclei
-Nutch
-Offline Explorer
-Openvas
+Octopus
+Openfind
+OrangeBot
+OrangeSpider
 OutclicksBot
-PHPCrawl
-PageGrabber
-Papa Foto
-Pavuk
+OutfoxBot
+Page Analyzer
+PageAnalyzer
+PageThing.com
+Pandalytics
 Petalbot
 Pi-Monster
-Picscout
 PictureFinder
 Piepmatz
-Pixray
 PleaseCrawl
-Pockey
 ProPowerBot
+Probethenet
 Psbot
-Pump
-QueryN Metasearch
+PxBroker
+PyCurl
 Quick-Crawler
-RSSingBot
-RankingBot
+RankActive
+RankFlex
 Rankivabot
-RankurBot
 ReGet
 Reaper
-RebelMouse
+Recorder
 RepoMonkey
-Ripper
 Rogerbot
-SBIder
 SEOkicks
 SEOlyticsCrawler
-SEOprofiler
-SMTBot
-ScanAlert
-Scanbot
+SEOstats
+ScoutJet
 Scrapy
 Screaming
-SemrushBot
-Seomoz
-Siphon
-SiteCheckerBotCrawler
+Semrush
+SeobilityBot
+Shodan
 SiteExplorer
-Sitevigil
-SlySearch
-SmartDownload
-Snake
-Snoopy
-Sogou web spider
+SiteLockSpider
+SiteSnagger
+Sitebeam
+Snapbot
 SpankBot
+Spanner
 Spbot
+Spinn3r
 Sqlmap
+Sqlworm
 Sqworm
-Steeler
-Stripper
-Sucuri
-Surfbot
-Suzuran
-T0PHackTeam
+Sucker
+SuperBot
+SurveyBot
+Swiftbot
+Teleport
+TeleportPro
 Telesoft
+Telesphoreo
 TightTwatBot
+Titan
 Toata
+Toweyabot
 Tracemyfile
+True_Robot
+Turnitin
 TwengaBot
 Twice
-URLy Warning
-Upflow
+Typhoeus
+VB Project
 Vacuum
-VelenPublicWebCrawler
-VoidEYE
+Vagabondo
+VeriCiteCrawler
 WWW-Collector-E
+WWW-Mechanize
 WWW::Mechanize
 Wallpapers
+WeSEE
 Web Collage
-Web Fuck
-Web Pix
+Web Enhancer
 Web Sucker
-WebAuto
-WebBandit
-WebCollage
-WebCopier
-WebEnhancer
-WebFetch
-WebFuck
-WebGo IS
+WebLeacher
 WebReaper
+Webalta
+Website Quester
 WebsiteExtractor
 Whack
 Whacker
 Widow
 WiseGuys Robot
+Woobot
+Wprecon
 Xaldon WebSpider
-YoudaoBot
+Xaldon_WebSpider
 Zauba
+Zeus
 ZmEu
-adscanner
+arquivo-web-crawler
+arquivo.pt
+cah.io.community
+clark-crawler
 coccocbot-web
-crawl.sogou.com
-crawler.feedback
-demandbase-bot
-facebookscraper
-heritrix
-internetVista monitor
+cognitiveseo
+crawler4j
+domainsproject.org
+evc-batch
 isitwp.com
 lwp-request
+lwp-trivial
 magpie-crawler
-meanpathbot
+oBot
 pcBrowser
-polaris version
-scalaj-http
+plumanalytics
+probe-image-size
+ripz
+scan.lol
+seobility
 seocompany.store
-seostar
-siteripz
+sexsearcher
 sogouspider
+sp_auditbot
 trendiction.de
 voyagerx.com
 webgains-bot
-webprosbot
+x22Mozilla
 zgrab