V4.2021.07.2471 [ci skip]

dev-tools/test_units/random-bots-for-whitelist-test.list

@@ -1,250 +1,250 @@
 01h4x.com
-404checker
+360Spider
+80legs
 ADmantX
-Abonti
+AIBOT
+Aboundex
+Aboundexbot
 Acunetix
-AfD-Verbotsverfahren
 AiHitBot
-Aipbot
-Alexibot
 AllSubmitter
+Alligator
 AlphaBot
-Asterias
-AwarioRssBot
-AwarioSmartBot
-BDCbot
+Anarchy99
+Anthill
+BBBike
 BDFetch
 BLEXBot
-BackDoorBot
-BackStreet
-BackWeb
-BacklinkCrawler
+Badass
+Barkrowler
+Battleztar Bazinga
 Bigfoot
-Black Hole
-BlackWidow
-Blow
-Bolt
-BotALot
-Brandwatch
-Buck
-BuiltBotTough
-BuiltWith
-Bullseye
-BuzzSumo
-CATExplorador
-CSHttp
+BlowFish
+Buddy
+CODE87
 CazoodleBot
-CensysInspect
-CherryPicker
+CheTeam
+CheeseBot
 Chlooe
-Cloud mapping
-Cocolyzebot
-CopyRightCheck
+Claritybot
+Cliqzbot
+Copier
+Craftbot
 CrazyWebCrawler
-DTS Agent
+Crescent
+DBLBot
+DIIbot
+DSearch
 DataCha0s
 Deusu
 Digincore
-DigitalPebble
-DnyzBot
-DomainCrawler
-DomainSigmaCrawler
+Dirbuster
+Disco
+Discoverybot
+DittoSpyder
+DomainAppender
 DomainStatsBot
-Domains Project
-Download Wonder
-Drip
-EroCrawler
-ExtLinksBot
+Ebingbong
+Ecxi
+Exabot
 Extreme Picture Finder
+EyeNetIE
 FDM
-FHscan
 FemtosearchBot
+Foobot
 Freeuploader
 FrontPage
 Fuzz
-GalaxyBot
+FyberSpider
+G-i-g-a-b-o-t
+GetRight
 GetWeb
-Getintent
-Gigabot
-Gotit
+Gigablast
+Go-Ahead-Got-It
+GoZilla
 Grafula
-GrapeFX
-GridBot
-HMView
-Heritrix
+GrapeshotCrawler
+HTTP::Lite
+Harvest
+Havij
 Hloader
-Humanlinks
-IRLbot
 Iblog
+Id-search
+IlseBot
 Image Sucker
-IndeedBot
-Indy Library
 InfoNaviRobot
-Iskanie
+InterGET
+InternetSeer
+Iria
 IstellaBot
-JOC Web Spider
+JamesBOT
+Jbrofuzz
 JetCar
 Jetty
-JikeSpider
-Joomla
-Jorgee
 JustView
-Kenjin Spider
-Keyword Density
 Kozmosbot
-LNSpiderguy
-LWP::Simple
-Lanshanbot
 Larbin
+LeechGet
 Lftp
 LibWeb
 Libwhisker
 LieBaoFast
 Linkdexbot
 LinkextractorPro
+LinksManager
 LinqiaMetadataDownloaderBot
 LinqiaRSSBot
+Lipperhey Spider
 Ltx71
-MFC_Tear_Sample
-MJ12bot
+MIIxpc
+MQQBrowser
 MTRobot
-Mag-Net
-Mass Downloader
+Majestic-SEO
 Masscan
-MauiBot
-Mb2345Browser
-MeanPath Bot
-Microsoft Data Access
-Mojeek
-Morfeus Fucking Scanner
-Musobot
-Name Intelligence
-Needle
+Meanpathbot
+Mediatoolkitbot
+MicroMessenger
+Microsoft URL Control
+Mister PiX
+Mojolicious
+Mr.4x3
+Msrabot
+Nameprotect
+Navroad
+NearSite
 Nessus
 Net Vampire
-NetMechanic
-NextGenSearchBot
+NetZIP
+Netcraft
 Niki-bot
 Nuclei
-Nutch
-OpenLinkProfiler
+OPPO A33
+Offline Navigator
+OnCrawl
+OrangeBot
 OrangeSpider
-OutfoxBot
-PHPCrawl
-PageAnalyzer
+OutclicksBot
+PECL::HTTP
+POE-Component-Client-HTTP
+Page Analyzer
 PageScorer
-Pandalytics
-Panscient
-Pi-Monster
-Pimonster
+Papa Foto
+PeoplePal
+Pixray
 Pockey
+ProPowerBot
+Probethenet
 Pu_iN
-PyCurl
-QueryN Metasearch
-Quick-Crawler
-RSSingBot
-RankActive
-RankActiveLinkBot
-RankurBot
-Reaper
+RankingBot2
+Re-re
 RebelMouse
 Recorder
+RedesScrapy
+Ripper
 RocketCrawler
-Rogerbot
+SBIder
 SEOkicks
+SEOkicks-Robot
 SEOlyticsCrawler
-SEOprofiler
 SEOstats
+SISTRIX
 SalesIntelligent
-ScanAlert
-Scrapy
-ScreenerBot
-Searchestate
+ScrepyBot
 Semrush
-SemrushBot
-SentiBot
+SeobilityBot
 Seomoz
-Shodan
-Siphon
 Site Sucker
-SiteLockSpider
-SiteSucker
-Sitebeam
-SlySearch
-SmartDownload
-Snapbot
+SiteCheckerBotCrawler
+SiteExplorer
+Siteimprove
+Sitevigil
+Snake
 Sociscraper
+Sogou web spider
+Sosospider
 Sottopop
-Spammen
+SpaceBison
 Spanner
+Spbot
 Spinn3r
 SputnikBot
-Sqlmap
+Sqlworm
+Steeler
 Sucker
-SuperBot
+Sucuri
 SuperHTTP
-Surfbot
 Suzuran
-Swiftbot
 Szukacz
+T0PHackTeam
 T8Abot
-TeleportPro
 Telesoft
-Trendiction
-Trendictionbot
-True_Robot
+Telesphoreo
+The Intraformant
+TheNomad
+Thumbor
+Titan
+Tracemyfile
+Turingos
+TurnitinBot
 TwengaBot
 Twice
 UnisterBot
-VB Project
-Virusdie
-VoidEYE
-Voil
-WBSearchBot
-WISENutbot
-WPScan
+Upflow
+VCI
+Vacuum
+Vagabondo
+VidibleScraper
 WWW-Mechanize
-Web Enhancer
-Web Fetch
-Web Fuck
-WebAuto
+WWWOFFLE
+WeSEE
+Web Sucker
 WebBandit
-WebCopier
 WebFetch
-WebImageCollector
-WebSauger
-WebSucker
+WebGo IS
+WebLeacher
+WebPix
+WebStripper
+WebZIP
+Webalta
 Webshag
-Webster
+Website Quester
 Whatweb
+Who.is Bot
 Widow
-WinHTTrack
+Wonderbot
 Wotbox
-Wprecon
+Xenu
+YoudaoBot
 Zade
 ZmEu
 ZoomBot
 ZoominfoBot
 arquivo-web-crawler
-arquivo.pt
+backlink-check
 check1.exe
-coccocbot-web
-crawl.sogou.com
-crawler4j
+com.plumanalytics
+crawler.feedback
 eCatch
-evc-batch
-isitwp.com
-lwp-request
+gopher
+instabid
+internetVista monitor
+ips-agent
 magpie-crawler
 meanpathbot
+mediawords
+muhstik-scan
 netEstate NE Crawler
 oBot
 pcBrowser
+probe-image-size
 ripz
 satoristudio.net
+scalaj-http
 seobility
-serpstatbot
-sogouspider
-sp_auditbot
+seostar
+sitechecker.pro
+spyfu
+sysscan
+trendiction.com
 webmeup-crawler
-x22Mozilla
+zgrab