vrr/Pulse
2
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-07-09 16:00:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 7

Surface provider MSP backup readiness

Browse source
This commit is contained in:
rcourtman 2026-06-02 16:02:04 +01:00
parent 42fb8eed3f
commit e86143f07f
5 changed files with 199 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

129
cmd/pulse-control-plane/provider_msp_status.go
View file

@ -3,6 +3,8 @@ package main
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
@ -14,8 +16,9 @@ import (
)
type providerMSPStatusOptions struct {
AllowEnvPlan bool
PullImage bool
AllowEnvPlan bool
PullImage bool
RequireBackup bool
}
type providerMSPStatusReport struct {
@ -38,12 +41,30 @@ type providerMSPStatusReport struct {
StuckProvisioningTenants []string
CountsByState map[registry.TenantState]int
Preflight *providerMSPPreflightReport
Backup *providerMSPBackupStatus
BackupReadyForUpgrade bool
Failures []string
Warnings []string
}
type providerMSPBackupStatus struct {
Directory string
LatestPath string
LatestCreatedAt time.Time
LatestModifiedAt time.Time
LatestAge time.Duration
LatestBytes int64
Verified bool
LicenseIncluded bool
RegistryTenantCount int
RuntimeTenantCount int
Warning string
}
type providerMSPStatusDependencies struct {
OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error)
RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error)
CheckBackup func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error)
Now func() time.Time
}
@ -64,6 +85,7 @@ func newProviderMSPStatusCmd() *cobra.Command {
}
cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development")
cmd.Flags().BoolVar(&opts.PullImage, "pull-image", false, "Pull the tenant runtime image during status instead of inspecting only")
cmd.Flags().BoolVar(&opts.RequireBackup, "require-backup", false, "Fail status unless a verified provider MSP backup archive is available")
return cmd
}
@ -94,6 +116,17 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo
report.OK = false
report.Failures = append(report.Failures, fmt.Sprintf(format, args...))
}
addWarning := func(format string, args ...any) {
report.Warnings = append(report.Warnings, fmt.Sprintf(format, args...))
}
addBackupProblem := func(format string, args ...any) {
message := fmt.Sprintf(format, args...)
if opts.RequireBackup {
addFailure("backup: %s", message)
} else {
addWarning("backup: %s", message)
}
}
preflight, preflightErr := deps.RunPreflight(ctx, cfg, providerMSPPreflightOptions{
AllowEnvPlan: opts.AllowEnvPlan,
@ -179,6 +212,25 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo
}
}
backup, backupErr := deps.CheckBackup(ctx, cfg)
report.Backup = backup
if backupErr != nil {
addBackupProblem("%v", backupErr)
} else if backup == nil {
addBackupProblem("backup posture unavailable")
} else {
if !backup.LatestCreatedAt.IsZero() {
backup.LatestAge = deps.Now().Sub(backup.LatestCreatedAt)
if backup.LatestAge < 0 {
backup.LatestAge = 0
}
}
report.BackupReadyForUpgrade = backup.Verified
if backup.Warning != "" {
addBackupProblem("%s", backup.Warning)
}
}
return report, providerMSPStatusError(report)
}
@ -191,12 +243,65 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies)
if deps.RunPreflight == nil {
deps.RunPreflight = runProviderMSPPreflight
}
if deps.CheckBackup == nil {
deps.CheckBackup = checkProviderMSPBackupStatus
}
if deps.Now == nil {
deps.Now = func() time.Time { return time.Now().UTC() }
}
return deps
}
func checkProviderMSPBackupStatus(ctx context.Context, cfg *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
if cfg == nil {
return nil, fmt.Errorf("control plane config is required")
}
backupDir := filepath.Join(strings.TrimSpace(cfg.DataDir), "backups", "provider-msp")
status := &providerMSPBackupStatus{Directory: backupDir}
entries, err := os.ReadDir(backupDir)
if err != nil {
if os.IsNotExist(err) {
status.Warning = "no provider MSP backup directory found; run provider-msp backup create before upgrades or recovery drills"
return status, nil
}
return status, fmt.Errorf("read provider MSP backup directory: %w", err)
}
var latestInfo os.FileInfo
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tar.gz") {
continue
}
info, infoErr := entry.Info()
if infoErr != nil {
return status, fmt.Errorf("stat provider MSP backup archive %s: %w", entry.Name(), infoErr)
}
if latestInfo == nil || info.ModTime().After(latestInfo.ModTime()) {
latestInfo = info
status.LatestPath = filepath.Join(backupDir, entry.Name())
status.LatestModifiedAt = info.ModTime()
status.LatestBytes = info.Size()
}
}
if latestInfo == nil {
status.Warning = "no provider MSP backup archive found; run provider-msp backup create before upgrades or recovery drills"
return status, nil
}
verify, err := cloudcp.VerifyProviderMSPBackup(ctx, status.LatestPath)
if err != nil {
return status, fmt.Errorf("verify latest provider MSP backup %s: %w", status.LatestPath, err)
}
status.Verified = true
status.LatestCreatedAt = verify.Manifest.CreatedAt
status.LatestBytes = verify.VerifiedArchiveBytes
status.LicenseIncluded = verify.Manifest.LicenseIncluded
status.RegistryTenantCount = verify.Manifest.RegistryTenantCount
status.RuntimeTenantCount = verify.Manifest.RuntimeTenantCount
return status, nil
}
func providerMSPStatusError(report *providerMSPStatusReport) error {
if report == nil || report.OK {
return nil
@ -252,6 +357,26 @@ func printProviderMSPStatusReport(report *providerMSPStatusReport) {
fmt.Printf("storage_guardrails_enabled=%t\n", report.Preflight.Storage.Enabled)
fmt.Printf("storage_guardrails_ok=%t\n", report.Preflight.Storage.OK)
}
fmt.Printf("backup_ready_for_upgrade=%t\n", report.BackupReadyForUpgrade)
if report.Backup != nil {
fmt.Printf("backup_directory=%s\n", report.Backup.Directory)
fmt.Printf("latest_backup_path=%s\n", report.Backup.LatestPath)
if !report.Backup.LatestCreatedAt.IsZero() {
fmt.Printf("latest_backup_created_at=%s\n", report.Backup.LatestCreatedAt.UTC().Format(time.RFC3339))
}
if !report.Backup.LatestModifiedAt.IsZero() {
fmt.Printf("latest_backup_modified_at=%s\n", report.Backup.LatestModifiedAt.UTC().Format(time.RFC3339))
}
fmt.Printf("latest_backup_age=%s\n", report.Backup.LatestAge)
fmt.Printf("latest_backup_bytes=%d\n", report.Backup.LatestBytes)
fmt.Printf("latest_backup_verified=%t\n", report.Backup.Verified)
fmt.Printf("latest_backup_license_included=%t\n", report.Backup.LicenseIncluded)
fmt.Printf("latest_backup_registry_tenants=%d\n", report.Backup.RegistryTenantCount)
fmt.Printf("latest_backup_runtime_tenants=%d\n", report.Backup.RuntimeTenantCount)
}
for _, warning := range report.Warnings {
fmt.Printf("warning=%s\n", warning)
}
for _, failure := range report.Failures {
fmt.Printf("failure=%s\n", failure)
}

62
cmd/pulse-control-plane/provider_msp_status_test.go
View file

@ -37,6 +37,9 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
gotPreflight = opts
return healthyProviderMSPStatusPreflightReport(), nil
},
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
return healthyProviderMSPBackupStatus(now), nil
},
Now: func() time.Time { return now },
})
if err != nil {
@ -57,6 +60,12 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
if report.LicenseID != "lic_provider_msp_test" || report.LicenseEmail != "provider@example.com" {
t.Fatalf("license status = %q %q", report.LicenseID, report.LicenseEmail)
}
if !report.BackupReadyForUpgrade || report.Backup == nil || !report.Backup.Verified {
t.Fatalf("backup posture not ready for upgrade: %#v", report.Backup)
}
if report.Backup.LatestAge != 2*time.Hour {
t.Fatalf("backup age = %s, want 2h", report.Backup.LatestAge)
}
}
func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) {
@ -102,6 +111,45 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T)
}
}
func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T) {
cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile)
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
deps := providerMSPStatusDependencies{
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
return healthyProviderMSPStatusPreflightReport(), nil
},
CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) {
return &providerMSPBackupStatus{
Directory: "/data/backups/provider-msp",
Warning: "no provider MSP backup archive found; run provider-msp backup create before upgrades or recovery drills",
}, nil
},
Now: func() time.Time { return now },
}
report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, deps)
if err != nil {
t.Fatalf("status without required backup should warn, not fail: %v", err)
}
if !report.OK || report.BackupReadyForUpgrade {
t.Fatalf("optional backup status = ok %t ready %t", report.OK, report.BackupReadyForUpgrade)
}
if got := strings.Join(report.Warnings, "; "); !strings.Contains(got, "provider-msp backup create") {
t.Fatalf("warnings = %q, want backup create guidance", got)
}
requiredReport, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{RequireBackup: true}, deps)
if err == nil {
t.Fatal("expected required backup status to fail")
}
if requiredReport == nil || requiredReport.OK {
t.Fatalf("required backup report OK = %v, want false", requiredReport != nil && requiredReport.OK)
}
if got := strings.Join(requiredReport.Failures, "; "); !strings.Contains(got, "backup: no provider MSP backup archive found") {
t.Fatalf("failures = %q, want missing backup failure", got)
}
}
func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
return &providerMSPPreflightReport{
OK: true,
@ -129,6 +177,20 @@ func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
}
}
func healthyProviderMSPBackupStatus(now time.Time) *providerMSPBackupStatus {
return &providerMSPBackupStatus{
Directory: "/data/backups/provider-msp",
LatestPath: "/data/backups/provider-msp/provider-msp-backup-20260602T100000Z.tar.gz",
LatestCreatedAt: now.Add(-2 * time.Hour),
LatestModifiedAt: now.Add(-time.Hour),
LatestBytes: 4096,
Verified: true,
LicenseIncluded: true,
RegistryTenantCount: 1,
RuntimeTenantCount: 1,
}
}
func createProviderMSPStatusTenant(t *testing.T, cfg *cloudcp.CPConfig, tenant *registry.Tenant) {
t.Helper()
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())

4
deploy/provider-msp/.env.example
View file

@ -56,6 +56,10 @@ PULSE_EMAIL_REPLY_TO=support@example.com
# Non-mutating operational status:
# docker compose run --rm control-plane provider-msp status
#
# Pre-upgrade / pre-maintenance status gate. This fails unless a verified
# provider MSP backup archive is available:
# docker compose run --rm control-plane provider-msp status --require-backup
#
# Fresh-install provider proof. This bootstraps the owner account, checks install
# readiness, creates two proof client workspaces, verifies tenant-local agent
# ingest and portal handoff, creates and verifies a backup, dry-runs restore and

6
docs/release-control/v6/internal/subsystems/deployment-installability.md
View file

@ -166,7 +166,11 @@ surfaces.
companion to that proof: it must report registry readiness, tenant
state/health counts, stuck provisioning workspaces, Docker runtime
prerequisites, storage guardrails, and the same license-backed plan identity
without pulling tenant images unless the operator asks for it.
without pulling tenant images unless the operator asks for it. It must also
surface backup readiness for upgrades and recovery drills by identifying the
latest verified provider MSP backup archive when one exists, warning when no
backup is available yet, and offering a strict `--require-backup` status gate
for pre-upgrade or pre-maintenance checks.
5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary.
`pulse-control-plane provider-msp backup create`, `backup verify`, and
`backup restore` must

1
scripts/installtests/provider_msp_deploy_test.go
View file

@ -60,6 +60,7 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
"docker compose run --rm control-plane provider-msp bootstrap",
"docker compose run --rm control-plane provider-msp preflight",
"docker compose run --rm control-plane provider-msp status",
"docker compose run --rm control-plane provider-msp status --require-backup",
"./run-install-proof.sh",
"docker compose run --rm control-plane provider-msp install-proof",
"docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run",