diff --git a/cmd/pulse-control-plane/provider_msp_status.go b/cmd/pulse-control-plane/provider_msp_status.go index e54ee6de3..75ed1ec9b 100644 --- a/cmd/pulse-control-plane/provider_msp_status.go +++ b/cmd/pulse-control-plane/provider_msp_status.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "os" + "path/filepath" "sort" "strings" "time" @@ -14,8 +16,9 @@ import ( ) type providerMSPStatusOptions struct { - AllowEnvPlan bool - PullImage bool + AllowEnvPlan bool + PullImage bool + RequireBackup bool } type providerMSPStatusReport struct { @@ -38,12 +41,30 @@ type providerMSPStatusReport struct { StuckProvisioningTenants []string CountsByState map[registry.TenantState]int Preflight *providerMSPPreflightReport + Backup *providerMSPBackupStatus + BackupReadyForUpgrade bool Failures []string + Warnings []string +} + +type providerMSPBackupStatus struct { + Directory string + LatestPath string + LatestCreatedAt time.Time + LatestModifiedAt time.Time + LatestAge time.Duration + LatestBytes int64 + Verified bool + LicenseIncluded bool + RegistryTenantCount int + RuntimeTenantCount int + Warning string } type providerMSPStatusDependencies struct { OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error) RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) + CheckBackup func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) Now func() time.Time } @@ -64,6 +85,7 @@ func newProviderMSPStatusCmd() *cobra.Command { } cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development") cmd.Flags().BoolVar(&opts.PullImage, "pull-image", false, "Pull the tenant runtime image during status instead of inspecting only") + cmd.Flags().BoolVar(&opts.RequireBackup, "require-backup", false, "Fail status unless a verified provider MSP backup archive is available") return cmd } @@ -94,6 +116,17 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo report.OK = false report.Failures = append(report.Failures, fmt.Sprintf(format, args...)) } + addWarning := func(format string, args ...any) { + report.Warnings = append(report.Warnings, fmt.Sprintf(format, args...)) + } + addBackupProblem := func(format string, args ...any) { + message := fmt.Sprintf(format, args...) + if opts.RequireBackup { + addFailure("backup: %s", message) + } else { + addWarning("backup: %s", message) + } + } preflight, preflightErr := deps.RunPreflight(ctx, cfg, providerMSPPreflightOptions{ AllowEnvPlan: opts.AllowEnvPlan, @@ -179,6 +212,25 @@ func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPCo } } + backup, backupErr := deps.CheckBackup(ctx, cfg) + report.Backup = backup + if backupErr != nil { + addBackupProblem("%v", backupErr) + } else if backup == nil { + addBackupProblem("backup posture unavailable") + } else { + if !backup.LatestCreatedAt.IsZero() { + backup.LatestAge = deps.Now().Sub(backup.LatestCreatedAt) + if backup.LatestAge < 0 { + backup.LatestAge = 0 + } + } + report.BackupReadyForUpgrade = backup.Verified + if backup.Warning != "" { + addBackupProblem("%s", backup.Warning) + } + } + return report, providerMSPStatusError(report) } @@ -191,12 +243,65 @@ func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) if deps.RunPreflight == nil { deps.RunPreflight = runProviderMSPPreflight } + if deps.CheckBackup == nil { + deps.CheckBackup = checkProviderMSPBackupStatus + } if deps.Now == nil { deps.Now = func() time.Time { return time.Now().UTC() } } return deps } +func checkProviderMSPBackupStatus(ctx context.Context, cfg *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { + if cfg == nil { + return nil, fmt.Errorf("control plane config is required") + } + backupDir := filepath.Join(strings.TrimSpace(cfg.DataDir), "backups", "provider-msp") + status := &providerMSPBackupStatus{Directory: backupDir} + + entries, err := os.ReadDir(backupDir) + if err != nil { + if os.IsNotExist(err) { + status.Warning = "no provider MSP backup directory found; run provider-msp backup create before upgrades or recovery drills" + return status, nil + } + return status, fmt.Errorf("read provider MSP backup directory: %w", err) + } + + var latestInfo os.FileInfo + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tar.gz") { + continue + } + info, infoErr := entry.Info() + if infoErr != nil { + return status, fmt.Errorf("stat provider MSP backup archive %s: %w", entry.Name(), infoErr) + } + if latestInfo == nil || info.ModTime().After(latestInfo.ModTime()) { + latestInfo = info + status.LatestPath = filepath.Join(backupDir, entry.Name()) + status.LatestModifiedAt = info.ModTime() + status.LatestBytes = info.Size() + } + } + if latestInfo == nil { + status.Warning = "no provider MSP backup archive found; run provider-msp backup create before upgrades or recovery drills" + return status, nil + } + + verify, err := cloudcp.VerifyProviderMSPBackup(ctx, status.LatestPath) + if err != nil { + return status, fmt.Errorf("verify latest provider MSP backup %s: %w", status.LatestPath, err) + } + status.Verified = true + status.LatestCreatedAt = verify.Manifest.CreatedAt + status.LatestBytes = verify.VerifiedArchiveBytes + status.LicenseIncluded = verify.Manifest.LicenseIncluded + status.RegistryTenantCount = verify.Manifest.RegistryTenantCount + status.RuntimeTenantCount = verify.Manifest.RuntimeTenantCount + return status, nil +} + func providerMSPStatusError(report *providerMSPStatusReport) error { if report == nil || report.OK { return nil @@ -252,6 +357,26 @@ func printProviderMSPStatusReport(report *providerMSPStatusReport) { fmt.Printf("storage_guardrails_enabled=%t\n", report.Preflight.Storage.Enabled) fmt.Printf("storage_guardrails_ok=%t\n", report.Preflight.Storage.OK) } + fmt.Printf("backup_ready_for_upgrade=%t\n", report.BackupReadyForUpgrade) + if report.Backup != nil { + fmt.Printf("backup_directory=%s\n", report.Backup.Directory) + fmt.Printf("latest_backup_path=%s\n", report.Backup.LatestPath) + if !report.Backup.LatestCreatedAt.IsZero() { + fmt.Printf("latest_backup_created_at=%s\n", report.Backup.LatestCreatedAt.UTC().Format(time.RFC3339)) + } + if !report.Backup.LatestModifiedAt.IsZero() { + fmt.Printf("latest_backup_modified_at=%s\n", report.Backup.LatestModifiedAt.UTC().Format(time.RFC3339)) + } + fmt.Printf("latest_backup_age=%s\n", report.Backup.LatestAge) + fmt.Printf("latest_backup_bytes=%d\n", report.Backup.LatestBytes) + fmt.Printf("latest_backup_verified=%t\n", report.Backup.Verified) + fmt.Printf("latest_backup_license_included=%t\n", report.Backup.LicenseIncluded) + fmt.Printf("latest_backup_registry_tenants=%d\n", report.Backup.RegistryTenantCount) + fmt.Printf("latest_backup_runtime_tenants=%d\n", report.Backup.RuntimeTenantCount) + } + for _, warning := range report.Warnings { + fmt.Printf("warning=%s\n", warning) + } for _, failure := range report.Failures { fmt.Printf("failure=%s\n", failure) } diff --git a/cmd/pulse-control-plane/provider_msp_status_test.go b/cmd/pulse-control-plane/provider_msp_status_test.go index 74c299b1b..888a62a79 100644 --- a/cmd/pulse-control-plane/provider_msp_status_test.go +++ b/cmd/pulse-control-plane/provider_msp_status_test.go @@ -37,6 +37,9 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { gotPreflight = opts return healthyProviderMSPStatusPreflightReport(), nil }, + CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { + return healthyProviderMSPBackupStatus(now), nil + }, Now: func() time.Time { return now }, }) if err != nil { @@ -57,6 +60,12 @@ func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { if report.LicenseID != "lic_provider_msp_test" || report.LicenseEmail != "provider@example.com" { t.Fatalf("license status = %q %q", report.LicenseID, report.LicenseEmail) } + if !report.BackupReadyForUpgrade || report.Backup == nil || !report.Backup.Verified { + t.Fatalf("backup posture not ready for upgrade: %#v", report.Backup) + } + if report.Backup.LatestAge != 2*time.Hour { + t.Fatalf("backup age = %s, want 2h", report.Backup.LatestAge) + } } func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) { @@ -102,6 +111,45 @@ func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) } } +func TestProviderMSPStatusBackupWarningBecomesFailureWhenRequired(t *testing.T) { + cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile) + now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) + deps := providerMSPStatusDependencies{ + RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { + return healthyProviderMSPStatusPreflightReport(), nil + }, + CheckBackup: func(context.Context, *cloudcp.CPConfig) (*providerMSPBackupStatus, error) { + return &providerMSPBackupStatus{ + Directory: "/data/backups/provider-msp", + Warning: "no provider MSP backup archive found; run provider-msp backup create before upgrades or recovery drills", + }, nil + }, + Now: func() time.Time { return now }, + } + + report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, deps) + if err != nil { + t.Fatalf("status without required backup should warn, not fail: %v", err) + } + if !report.OK || report.BackupReadyForUpgrade { + t.Fatalf("optional backup status = ok %t ready %t", report.OK, report.BackupReadyForUpgrade) + } + if got := strings.Join(report.Warnings, "; "); !strings.Contains(got, "provider-msp backup create") { + t.Fatalf("warnings = %q, want backup create guidance", got) + } + + requiredReport, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{RequireBackup: true}, deps) + if err == nil { + t.Fatal("expected required backup status to fail") + } + if requiredReport == nil || requiredReport.OK { + t.Fatalf("required backup report OK = %v, want false", requiredReport != nil && requiredReport.OK) + } + if got := strings.Join(requiredReport.Failures, "; "); !strings.Contains(got, "backup: no provider MSP backup archive found") { + t.Fatalf("failures = %q, want missing backup failure", got) + } +} + func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport { return &providerMSPPreflightReport{ OK: true, @@ -129,6 +177,20 @@ func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport { } } +func healthyProviderMSPBackupStatus(now time.Time) *providerMSPBackupStatus { + return &providerMSPBackupStatus{ + Directory: "/data/backups/provider-msp", + LatestPath: "/data/backups/provider-msp/provider-msp-backup-20260602T100000Z.tar.gz", + LatestCreatedAt: now.Add(-2 * time.Hour), + LatestModifiedAt: now.Add(-time.Hour), + LatestBytes: 4096, + Verified: true, + LicenseIncluded: true, + RegistryTenantCount: 1, + RuntimeTenantCount: 1, + } +} + func createProviderMSPStatusTenant(t *testing.T, cfg *cloudcp.CPConfig, tenant *registry.Tenant) { t.Helper() reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index 59057696c..c4c06204d 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -56,6 +56,10 @@ PULSE_EMAIL_REPLY_TO=support@example.com # Non-mutating operational status: # docker compose run --rm control-plane provider-msp status # +# Pre-upgrade / pre-maintenance status gate. This fails unless a verified +# provider MSP backup archive is available: +# docker compose run --rm control-plane provider-msp status --require-backup +# # Fresh-install provider proof. This bootstraps the owner account, checks install # readiness, creates two proof client workspaces, verifies tenant-local agent # ingest and portal handoff, creates and verifies a backup, dry-runs restore and diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 8fcc4d827..815ebf482 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -166,7 +166,11 @@ surfaces. companion to that proof: it must report registry readiness, tenant state/health counts, stuck provisioning workspaces, Docker runtime prerequisites, storage guardrails, and the same license-backed plan identity - without pulling tenant images unless the operator asks for it. + without pulling tenant images unless the operator asks for it. It must also + surface backup readiness for upgrades and recovery drills by identifying the + latest verified provider MSP backup archive when one exists, warning when no + backup is available yet, and offering a strict `--require-backup` status gate + for pre-upgrade or pre-maintenance checks. 5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. `pulse-control-plane provider-msp backup create`, `backup verify`, and `backup restore` must diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 5876e58a8..6ebb7122c 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -60,6 +60,7 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "docker compose run --rm control-plane provider-msp bootstrap", "docker compose run --rm control-plane provider-msp preflight", "docker compose run --rm control-plane provider-msp status", + "docker compose run --rm control-plane provider-msp status --require-backup", "./run-install-proof.sh", "docker compose run --rm control-plane provider-msp install-proof", "docker compose run --rm control-plane provider-msp recover --all-degraded --dry-run",