From a2e860dc8c777ac17fda9c800339f11d95a1b5a4 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 2 Jun 2026 14:14:07 +0100 Subject: [PATCH] Add provider MSP backup restore --- .../provider_msp_backup.go | 75 ++++ .../provider_msp_backup_test.go | 6 + deploy/provider-msp/.env.example | 6 + .../v6/internal/subsystems/cloud-paid.md | 9 +- .../subsystems/deployment-installability.md | 10 +- internal/cloudcp/provider_msp_backup.go | 324 ++++++++++++++++++ internal/cloudcp/provider_msp_backup_test.go | 137 ++++++++ .../installtests/provider_msp_deploy_test.go | 3 + 8 files changed, 562 insertions(+), 8 deletions(-) diff --git a/cmd/pulse-control-plane/provider_msp_backup.go b/cmd/pulse-control-plane/provider_msp_backup.go index eb2469d95..0d09c064c 100644 --- a/cmd/pulse-control-plane/provider_msp_backup.go +++ b/cmd/pulse-control-plane/provider_msp_backup.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "strings" "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" @@ -14,6 +15,7 @@ func newProviderMSPBackupCmd() *cobra.Command { Short: "Create and verify provider-hosted MSP recovery archives", } cmd.AddCommand(newProviderMSPBackupCreateCmd()) + cmd.AddCommand(newProviderMSPBackupRestoreCmd()) cmd.AddCommand(newProviderMSPBackupVerifyCmd()) return cmd } @@ -40,6 +42,48 @@ func newProviderMSPBackupCreateCmd() *cobra.Command { return cmd } +func newProviderMSPBackupRestoreCmd() *cobra.Command { + var archivePath string + var targetDataDir string + var licenseOutputPath string + var replaceExisting bool + var dryRun bool + cmd := &cobra.Command{ + Use: "restore [archive]", + Short: "Restore a provider-hosted MSP recovery archive", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 1 { + if strings.TrimSpace(archivePath) != "" && archivePath != args[0] { + return fmt.Errorf("archive path provided both as --archive and positional argument") + } + archivePath = args[0] + } + if strings.TrimSpace(targetDataDir) == "" { + targetDataDir = providerMSPRestoreDefaultDataDir() + } + result, err := cloudcp.RestoreProviderMSPBackup(cmd.Context(), cloudcp.ProviderMSPBackupRestoreOptions{ + ArchivePath: archivePath, + TargetDataDir: targetDataDir, + LicenseOutputPath: licenseOutputPath, + ReplaceExisting: replaceExisting, + DryRun: dryRun, + }) + if err != nil { + return err + } + printProviderMSPBackupRestoreResult(result) + return nil + }, + } + cmd.Flags().StringVar(&archivePath, "archive", "", "Backup archive path to restore") + cmd.Flags().StringVar(&targetDataDir, "target-data-dir", "", "Target CP_DATA_DIR to restore into (default: CP_DATA_DIR or /data)") + cmd.Flags().StringVar(&licenseOutputPath, "license-output", "", "Where to restore the provider MSP license file (default: /provider-msp-license.jwt)") + cmd.Flags().BoolVar(&replaceExisting, "replace", false, "Replace existing restored control-plane/tenant state after the control plane has been stopped") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Verify and report restore targets without writing files") + return cmd +} + func newProviderMSPBackupVerifyCmd() *cobra.Command { var archivePath string cmd := &cobra.Command{ @@ -65,6 +109,13 @@ func newProviderMSPBackupVerifyCmd() *cobra.Command { return cmd } +func providerMSPRestoreDefaultDataDir() string { + if value := strings.TrimSpace(os.Getenv("CP_DATA_DIR")); value != "" { + return value + } + return "/data" +} + func printProviderMSPBackupCreateResult(result *cloudcp.ProviderMSPBackupCreateResult) { if result == nil { fmt.Println("provider_msp_backup_created=false") @@ -79,6 +130,30 @@ func printProviderMSPBackupCreateResult(result *cloudcp.ProviderMSPBackupCreateR fmt.Printf("license_entries=%d\n", result.LicenseEntries) } +func printProviderMSPBackupRestoreResult(result *cloudcp.ProviderMSPBackupRestoreResult) { + if result == nil { + fmt.Println("provider_msp_backup_restored=false") + return + } + fmt.Printf("provider_msp_backup_restored=%t\n", !result.DryRun) + fmt.Printf("provider_msp_backup_restore_dry_run=%t\n", result.DryRun) + fmt.Printf("archive_path=%s\n", result.ArchivePath) + fmt.Printf("archive_bytes=%d\n", result.VerifiedArchiveBytes) + fmt.Printf("target_data_dir=%s\n", result.TargetDataDir) + fmt.Printf("control_plane_dir=%s\n", result.ControlPlaneDir) + fmt.Printf("tenants_dir=%s\n", result.TenantsDir) + fmt.Printf("license_output_path=%s\n", result.LicenseOutputPath) + fmt.Printf("replace_existing=%t\n", result.ReplaceExisting) + printProviderMSPBackupManifest(result.Manifest) + fmt.Printf("control_plane_entries_restored=%d\n", result.ControlPlaneEntriesRestored) + fmt.Printf("tenant_entries_restored=%d\n", result.TenantEntriesRestored) + fmt.Printf("license_entries_restored=%d\n", result.LicenseEntriesRestored) + fmt.Printf("restored_registry_tenant_count=%d\n", result.RestoredRegistryTenantCount) + for _, tenantID := range result.RestoredRuntimeTenantIDs { + fmt.Printf("restored_runtime_tenant_id=%s\n", tenantID) + } +} + func printProviderMSPBackupVerifyResult(result *cloudcp.ProviderMSPBackupVerifyResult) { if result == nil { fmt.Println("provider_msp_backup_verified=false") diff --git a/cmd/pulse-control-plane/provider_msp_backup_test.go b/cmd/pulse-control-plane/provider_msp_backup_test.go index 2f73ef518..225b46613 100644 --- a/cmd/pulse-control-plane/provider_msp_backup_test.go +++ b/cmd/pulse-control-plane/provider_msp_backup_test.go @@ -7,11 +7,14 @@ func TestProviderMSPCommandExposesBackup(t *testing.T) { for _, child := range cmd.Commands() { if child.Name() == "backup" { foundCreate := false + foundRestore := false foundVerify := false for _, backupChild := range child.Commands() { switch backupChild.Name() { case "create": foundCreate = true + case "restore": + foundRestore = true case "verify": foundVerify = true } @@ -19,6 +22,9 @@ func TestProviderMSPCommandExposesBackup(t *testing.T) { if !foundCreate { t.Fatal("provider-msp backup create command is not registered") } + if !foundRestore { + t.Fatal("provider-msp backup restore command is not registered") + } if !foundVerify { t.Fatal("provider-msp backup verify command is not registered") } diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index 17b8f8060..6e5ba6c64 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -50,6 +50,12 @@ PULSE_EMAIL_REPLY_TO=support@example.com # docker compose run --rm control-plane provider-msp backup verify \ # /data/backups/provider-msp/provider-msp-backup-YYYYMMDDTHHMMSSZ.tar.gz # +# Recovery dry-run on a fresh target data directory: +# docker compose run --rm control-plane provider-msp backup restore \ +# /data/backups/provider-msp/provider-msp-backup-YYYYMMDDTHHMMSSZ.tar.gz \ +# --target-data-dir /data-restore-drill \ +# --dry-run +# # End-to-end provider proof. This creates two proof client workspaces, generates # client-bound agent install tokens, verifies tenant-local agent report ingest, # and exercises the portal handoff path. Keep --cleanup once you have captured diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 76d58e830..e17bfffc3 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -29,8 +29,8 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and stuck-provisioning threshold as the control-plane cleanup loop, so operator readiness reports and automated failure handling cannot drift into separate definitions of a failed provider workspace. -Provider-hosted MSP backup and verification are part of the cloud-paid operator -contract because the recovery artifact must preserve the provider's +Provider-hosted MSP backup, verification, and restore are part of the cloud-paid +operator contract because the recovery artifact must preserve the provider's license-backed plan identity, control-plane account/workspace registry, and tenant-local runtime state without depending on Stripe billing surfaces. @@ -190,8 +190,9 @@ tenant-local runtime state without depending on Stripe billing surfaces. 10. `internal/cloudcp/provider_msp_backup.go` shared with `deployment-installability`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. License-backed provider MSP backups must include the signed MSP license file as a recovery artifact while exposing only license metadata in command - output, and they must keep the archive Stripe-free so provider-hosted MSP - recovery does not inherit Pulse-hosted SaaS billing assumptions. + output, restore must recover that license as an explicit operator artifact, + and the archive must stay Stripe-free so provider-hosted MSP recovery does + not inherit Pulse-hosted SaaS billing assumptions. 11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. Hosted tenant runtime reconciliation must treat a registered tenant with preserved tenant data but no live Docker runtime as a recoverable managed diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 0d2aa7c04..75d219b89 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -140,14 +140,16 @@ surfaces. prerequisites, storage guardrails, and the same license-backed plan identity without pulling tenant images unless the operator asks for it. 5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. - `pulse-control-plane provider-msp backup create` and `backup verify` must + `pulse-control-plane provider-msp backup create`, `backup verify`, and + `backup restore` must create a Stripe-free recovery archive outside the live control-plane/tenant source trees, snapshot SQLite control-plane databases through an online backup path, include tenant runtime directories for all non-deleted registry workspaces, include the signed MSP license file when - the plan source is license-backed, and verify the manifest, tenant registry - snapshot, license artifact, and tenant runtime directories before the - archive is treated as usable for upgrades or recovery drills. + the plan source is license-backed, verify the manifest, tenant registry + snapshot, license artifact, and tenant runtime directories, and fail closed + on restore when target provider MSP state already exists unless the operator + explicitly uses the replace gate after stopping the control plane. 6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. 7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. It must expose a non-mutating preflight for the exact Windows agent diff --git a/internal/cloudcp/provider_msp_backup.go b/internal/cloudcp/provider_msp_backup.go index 01780b1a1..d749abb3a 100644 --- a/internal/cloudcp/provider_msp_backup.go +++ b/internal/cloudcp/provider_msp_backup.go @@ -80,6 +80,35 @@ type ProviderMSPBackupVerifyResult struct { VerifiedArchiveBytes int64 } +// ProviderMSPBackupRestoreOptions controls restoring a provider MSP recovery +// archive into a target control-plane data directory. +type ProviderMSPBackupRestoreOptions struct { + ArchivePath string + TargetDataDir string + LicenseOutputPath string + ReplaceExisting bool + DryRun bool +} + +// ProviderMSPBackupRestoreResult describes a completed or dry-run provider MSP +// recovery operation. +type ProviderMSPBackupRestoreResult struct { + ArchivePath string + TargetDataDir string + ControlPlaneDir string + TenantsDir string + LicenseOutputPath string + DryRun bool + ReplaceExisting bool + Manifest ProviderMSPBackupManifest + VerifiedArchiveBytes int64 + ControlPlaneEntriesRestored int + TenantEntriesRestored int + LicenseEntriesRestored int + RestoredRegistryTenantCount int + RestoredRuntimeTenantIDs []string +} + // DefaultProviderMSPBackupPath returns the compose-friendly default backup // location under CP_DATA_DIR without placing the archive inside the source trees. func DefaultProviderMSPBackupPath(cfg *CPConfig, now time.Time) string { @@ -205,6 +234,81 @@ func CreateProviderMSPBackup(ctx context.Context, cfg *CPConfig, outputPath stri return result, nil } +// RestoreProviderMSPBackup verifies and restores a provider MSP recovery +// archive. It intentionally does not require a fully validated CPConfig because +// restore may need to recover the MSP license before normal provider-hosted +// config validation can pass. +func RestoreProviderMSPBackup(ctx context.Context, opts ProviderMSPBackupRestoreOptions) (*ProviderMSPBackupRestoreResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + opts.ArchivePath = strings.TrimSpace(opts.ArchivePath) + if opts.ArchivePath == "" { + return nil, fmt.Errorf("backup archive path is required") + } + targetDataDir, err := resolveProviderMSPRestoreTargetDataDir(opts.TargetDataDir) + if err != nil { + return nil, err + } + licenseOutputPath, err := resolveProviderMSPRestoreLicenseOutputPath(opts.LicenseOutputPath, targetDataDir) + if err != nil { + return nil, err + } + controlPlaneDir := filepath.Join(targetDataDir, providerMSPBackupControlPlaneDir) + tenantsDir := filepath.Join(targetDataDir, providerMSPBackupTenantsDir) + if pathIsInside(licenseOutputPath, controlPlaneDir) || pathIsInside(licenseOutputPath, tenantsDir) { + return nil, fmt.Errorf("license output path must not be inside restored %s or %s", providerMSPBackupControlPlaneDir, providerMSPBackupTenantsDir) + } + + verified, err := VerifyProviderMSPBackup(ctx, opts.ArchivePath) + if err != nil { + return nil, err + } + conflicts, err := providerMSPRestoreConflicts(controlPlaneDir, tenantsDir, licenseOutputPath, verified.Manifest.LicenseIncluded) + if err != nil { + return nil, err + } + if len(conflicts) > 0 && !opts.ReplaceExisting { + return nil, fmt.Errorf("restore target already contains provider MSP state (%s); rerun with replace enabled only after stopping the control plane", strings.Join(conflicts, ", ")) + } + + result := &ProviderMSPBackupRestoreResult{ + ArchivePath: verified.ArchivePath, + TargetDataDir: targetDataDir, + ControlPlaneDir: controlPlaneDir, + TenantsDir: tenantsDir, + LicenseOutputPath: licenseOutputPath, + DryRun: opts.DryRun, + ReplaceExisting: opts.ReplaceExisting, + Manifest: verified.Manifest, + VerifiedArchiveBytes: verified.VerifiedArchiveBytes, + } + if opts.DryRun { + result.ControlPlaneEntriesRestored = verified.ControlPlaneEntries + result.TenantEntriesRestored = verified.TenantEntries + result.LicenseEntriesRestored = verified.LicenseEntries + result.RestoredRegistryTenantCount = verified.Manifest.RegistryTenantCount + result.RestoredRuntimeTenantIDs = append([]string(nil), verified.Manifest.RuntimeTenantIDs...) + return result, nil + } + + if opts.ReplaceExisting { + if err := removeProviderMSPRestoreTargets(controlPlaneDir, tenantsDir, licenseOutputPath, verified.Manifest.LicenseIncluded); err != nil { + return nil, err + } + } + if err := os.MkdirAll(targetDataDir, 0o755); err != nil { + return nil, fmt.Errorf("create restore target data dir: %w", err) + } + if err := extractProviderMSPBackupArchive(ctx, verified.ArchivePath, targetDataDir, licenseOutputPath, result); err != nil { + return nil, err + } + if err := validateRestoredProviderMSPBackup(result); err != nil { + return nil, err + } + return result, nil +} + // VerifyProviderMSPBackup validates that a provider MSP backup contains the // manifest, tenant registry snapshot, required tenant directories, and license // artifact declared by the manifest. @@ -328,6 +432,226 @@ func VerifyProviderMSPBackup(ctx context.Context, archivePath string) (*Provider return result, nil } +func resolveProviderMSPRestoreTargetDataDir(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", fmt.Errorf("restore target data dir is required") + } + resolved, err := filepath.Abs(filepath.Clean(raw)) + if err != nil { + return "", fmt.Errorf("resolve restore target data dir: %w", err) + } + return resolved, nil +} + +func resolveProviderMSPRestoreLicenseOutputPath(raw, targetDataDir string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + raw = filepath.Join(targetDataDir, providerMSPBackupLicenseName) + } + resolved, err := filepath.Abs(filepath.Clean(raw)) + if err != nil { + return "", fmt.Errorf("resolve provider MSP license output path: %w", err) + } + return resolved, nil +} + +func providerMSPRestoreConflicts(controlPlaneDir, tenantsDir, licenseOutputPath string, requireLicense bool) ([]string, error) { + var conflicts []string + for _, item := range []struct { + label string + path string + }{ + {label: providerMSPBackupControlPlaneDir, path: controlPlaneDir}, + {label: providerMSPBackupTenantsDir, path: tenantsDir}, + } { + exists, err := providerMSPRestorePathExists(item.path) + if err != nil { + return nil, err + } + if exists { + conflicts = append(conflicts, item.label) + } + } + if requireLicense { + exists, err := providerMSPRestorePathExists(licenseOutputPath) + if err != nil { + return nil, err + } + if exists { + conflicts = append(conflicts, "provider-msp-license") + } + } + return conflicts, nil +} + +func providerMSPRestorePathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("stat restore target %s: %w", path, err) +} + +func removeProviderMSPRestoreTargets(controlPlaneDir, tenantsDir, licenseOutputPath string, removeLicense bool) error { + for _, dir := range []string{controlPlaneDir, tenantsDir} { + if err := os.RemoveAll(dir); err != nil { + return fmt.Errorf("remove existing restore target %s: %w", dir, err) + } + } + if removeLicense { + if err := os.Remove(licenseOutputPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove existing provider MSP license output: %w", err) + } + } + return nil +} + +func extractProviderMSPBackupArchive(ctx context.Context, archivePath, targetDataDir, licenseOutputPath string, result *ProviderMSPBackupRestoreResult) error { + file, err := os.Open(archivePath) + if err != nil { + return fmt.Errorf("open backup archive for restore: %w", err) + } + defer file.Close() + gz, err := gzip.NewReader(file) + if err != nil { + return fmt.Errorf("open backup gzip stream for restore: %w", err) + } + defer gz.Close() + + tr := tar.NewReader(gz) + for { + if err := ctx.Err(); err != nil { + return err + } + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("read backup archive for restore: %w", err) + } + name, err := cleanProviderMSPArchiveName(header.Name) + if err != nil { + return err + } + switch { + case name == providerMSPBackupManifestName: + continue + case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"): + if err := restoreProviderMSPArchiveEntry(tr, header, targetDataDir, name); err != nil { + return err + } + result.ControlPlaneEntriesRestored++ + case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"): + if err := restoreProviderMSPArchiveEntry(tr, header, targetDataDir, name); err != nil { + return err + } + result.TenantEntriesRestored++ + case name == providerMSPBackupLicenseDir: + continue + case name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName: + if err := restoreProviderMSPArchiveFile(tr, header, licenseOutputPath); err != nil { + return err + } + result.LicenseEntriesRestored++ + default: + return fmt.Errorf("unexpected backup entry during restore %q", name) + } + } + return nil +} + +func restoreProviderMSPArchiveEntry(reader io.Reader, header *tar.Header, targetDataDir, archiveName string) error { + destPath := filepath.Join(targetDataDir, filepath.FromSlash(archiveName)) + if !pathIsInside(destPath, targetDataDir) { + return fmt.Errorf("backup restore entry escapes target data dir: %s", archiveName) + } + switch header.Typeflag { + case tar.TypeDir: + return restoreProviderMSPArchiveDir(header, destPath) + case tar.TypeReg, tar.TypeRegA: + return restoreProviderMSPArchiveFile(reader, header, destPath) + default: + return fmt.Errorf("unsupported backup restore entry type for %s", archiveName) + } +} + +func restoreProviderMSPArchiveDir(header *tar.Header, destPath string) error { + mode := os.FileMode(header.Mode) & 0o777 + if mode == 0 { + mode = 0o755 + } + if err := os.MkdirAll(destPath, mode); err != nil { + return fmt.Errorf("create restored directory %s: %w", destPath, err) + } + return nil +} + +func restoreProviderMSPArchiveFile(reader io.Reader, header *tar.Header, destPath string) error { + if header.Typeflag != tar.TypeReg && header.Typeflag != tar.TypeRegA { + return fmt.Errorf("backup restore expected regular file at %s", header.Name) + } + mode := os.FileMode(header.Mode) & 0o777 + if mode == 0 { + mode = 0o600 + } + if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { + return fmt.Errorf("create restored file parent %s: %w", destPath, err) + } + file, err := os.OpenFile(destPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, mode) + if err != nil { + return fmt.Errorf("create restored file %s: %w", destPath, err) + } + _, copyErr := io.Copy(file, reader) + closeErr := file.Close() + if copyErr != nil { + return fmt.Errorf("restore file %s: %w", destPath, copyErr) + } + if closeErr != nil { + return fmt.Errorf("close restored file %s: %w", destPath, closeErr) + } + return nil +} + +func validateRestoredProviderMSPBackup(result *ProviderMSPBackupRestoreResult) error { + if result == nil { + return fmt.Errorf("restore result is required") + } + if err := requireRegularFile(filepath.Join(result.ControlPlaneDir, "tenants.db"), "restored tenant registry database"); err != nil { + return err + } + if result.Manifest.LicenseIncluded { + if err := requireRegularFile(result.LicenseOutputPath, "restored provider MSP license file"); err != nil { + return err + } + } + reg, err := registry.NewTenantRegistry(result.ControlPlaneDir) + if err != nil { + return fmt.Errorf("open restored tenant registry: %w", err) + } + tenants, listErr := reg.List() + closeErr := reg.Close() + if listErr != nil { + return fmt.Errorf("list restored tenant registry rows: %w", listErr) + } + if closeErr != nil { + return fmt.Errorf("close restored tenant registry: %w", closeErr) + } + if len(tenants) != result.Manifest.RegistryTenantCount { + return fmt.Errorf("restored tenant registry count = %d, want %d", len(tenants), result.Manifest.RegistryTenantCount) + } + if err := requireProviderMSPRuntimeTenantDirs(result.TenantsDir, result.Manifest.RuntimeTenantIDs); err != nil { + return fmt.Errorf("validate restored tenant runtime dirs: %w", err) + } + result.RestoredRegistryTenantCount = len(tenants) + result.RestoredRuntimeTenantIDs = append([]string(nil), result.Manifest.RuntimeTenantIDs...) + return nil +} + func buildProviderMSPBackupManifest(cfg *CPConfig, tenants []*registry.Tenant, accountCount int) (ProviderMSPBackupManifest, error) { workspaceLimit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion) if !known { diff --git a/internal/cloudcp/provider_msp_backup_test.go b/internal/cloudcp/provider_msp_backup_test.go index a04dfabbf..8e7ba1a74 100644 --- a/internal/cloudcp/provider_msp_backup_test.go +++ b/internal/cloudcp/provider_msp_backup_test.go @@ -129,6 +129,102 @@ func TestProviderMSPBackupRequiresRuntimeTenantDirs(t *testing.T) { } } +func TestProviderMSPBackupRestoreRecoversStateAndLicense(t *testing.T) { + _, archivePath := createProviderMSPBackupArchiveForRestoreTest(t) + targetDataDir := filepath.Join(t.TempDir(), "restored-data") + + result, err := RestoreProviderMSPBackup(context.Background(), ProviderMSPBackupRestoreOptions{ + ArchivePath: archivePath, + TargetDataDir: targetDataDir, + }) + if err != nil { + t.Fatalf("RestoreProviderMSPBackup: %v", err) + } + if result.DryRun { + t.Fatal("restore result unexpectedly marked as dry-run") + } + if result.RestoredRegistryTenantCount != 2 { + t.Fatalf("RestoredRegistryTenantCount = %d, want 2", result.RestoredRegistryTenantCount) + } + if len(result.RestoredRuntimeTenantIDs) != 1 || result.RestoredRuntimeTenantIDs[0] != "t-ACTIVE001" { + t.Fatalf("RestoredRuntimeTenantIDs = %#v", result.RestoredRuntimeTenantIDs) + } + assertProviderMSPBackupRestoredTenantCount(t, filepath.Join(targetDataDir, "control-plane", "tenants.db"), 2) + if got, err := os.ReadFile(filepath.Join(targetDataDir, "tenants", "t-ACTIVE001", "runtime.json")); err != nil { + t.Fatalf("read restored tenant runtime file: %v", err) + } else if string(got) != `{"ok":true}` { + t.Fatalf("restored tenant runtime file = %q", got) + } + if got, err := os.ReadFile(filepath.Join(targetDataDir, "provider-msp-license.jwt")); err != nil { + t.Fatalf("read restored license: %v", err) + } else if string(got) != "signed-provider-msp-license" { + t.Fatalf("restored license = %q", got) + } + if _, err := os.Stat(filepath.Join(targetDataDir, "tenants", "t-DELETED001")); !os.IsNotExist(err) { + t.Fatalf("deleted tenant runtime dir should not be restored, stat err=%v", err) + } +} + +func TestProviderMSPBackupRestoreDryRunDoesNotWriteTarget(t *testing.T) { + _, archivePath := createProviderMSPBackupArchiveForRestoreTest(t) + targetDataDir := filepath.Join(t.TempDir(), "restore-dry-run") + + result, err := RestoreProviderMSPBackup(context.Background(), ProviderMSPBackupRestoreOptions{ + ArchivePath: archivePath, + TargetDataDir: targetDataDir, + DryRun: true, + }) + if err != nil { + t.Fatalf("RestoreProviderMSPBackup dry-run: %v", err) + } + if !result.DryRun { + t.Fatal("restore result should be marked dry-run") + } + if _, err := os.Stat(filepath.Join(targetDataDir, "control-plane")); !os.IsNotExist(err) { + t.Fatalf("dry-run should not create control-plane dir, stat err=%v", err) + } +} + +func TestProviderMSPBackupRestoreRequiresReplaceForExistingState(t *testing.T) { + _, archivePath := createProviderMSPBackupArchiveForRestoreTest(t) + targetDataDir := filepath.Join(t.TempDir(), "restore-replace") + stalePath := filepath.Join(targetDataDir, "control-plane", "stale.txt") + if err := os.MkdirAll(filepath.Dir(stalePath), 0o755); err != nil { + t.Fatalf("create stale control-plane dir: %v", err) + } + if err := os.WriteFile(stalePath, []byte("stale"), 0o600); err != nil { + t.Fatalf("write stale file: %v", err) + } + + _, err := RestoreProviderMSPBackup(context.Background(), ProviderMSPBackupRestoreOptions{ + ArchivePath: archivePath, + TargetDataDir: targetDataDir, + }) + if err == nil { + t.Fatal("expected restore to reject existing provider MSP state") + } + if !strings.Contains(err.Error(), "replace enabled") { + t.Fatalf("error = %v", err) + } + + result, err := RestoreProviderMSPBackup(context.Background(), ProviderMSPBackupRestoreOptions{ + ArchivePath: archivePath, + TargetDataDir: targetDataDir, + ReplaceExisting: true, + LicenseOutputPath: filepath.Join(targetDataDir, "provider-msp-license.jwt"), + }) + if err != nil { + t.Fatalf("RestoreProviderMSPBackup replace: %v", err) + } + if !result.ReplaceExisting { + t.Fatal("restore result should record replace mode") + } + if _, err := os.Stat(stalePath); !os.IsNotExist(err) { + t.Fatalf("replace should remove stale file, stat err=%v", err) + } + assertProviderMSPBackupRestoredTenantCount(t, filepath.Join(targetDataDir, "control-plane", "tenants.db"), 2) +} + func testProviderMSPBackupConfig(t *testing.T) *CPConfig { t.Helper() root := t.TempDir() @@ -183,6 +279,31 @@ func seedProviderMSPBackupRegistry(t *testing.T, cfg *CPConfig) *registry.Tenant return reg } +func createProviderMSPBackupArchiveForRestoreTest(t *testing.T) (*CPConfig, string) { + t.Helper() + cfg := testProviderMSPBackupConfig(t) + reg := seedProviderMSPBackupRegistry(t, cfg) + if err := reg.Close(); err != nil { + t.Fatalf("close registry: %v", err) + } + magicLinks, err := cpauth.NewService(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewService: %v", err) + } + magicLinks.Close() + if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil { + t.Fatalf("create active tenant dir: %v", err) + } + if err := os.WriteFile(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001", "runtime.json"), []byte(`{"ok":true}`), 0o600); err != nil { + t.Fatalf("write active tenant runtime file: %v", err) + } + archivePath := filepath.Join(t.TempDir(), "provider-msp-backup.tar.gz") + if _, err := CreateProviderMSPBackup(context.Background(), cfg, archivePath); err != nil { + t.Fatalf("CreateProviderMSPBackup: %v", err) + } + return cfg, archivePath +} + func readProviderMSPBackupEntries(t *testing.T, archivePath string) map[string][]byte { t.Helper() file, err := os.Open(archivePath) @@ -239,6 +360,22 @@ func assertProviderMSPBackupRegistrySnapshotCount(t *testing.T, dbBytes []byte, } } +func assertProviderMSPBackupRestoredTenantCount(t *testing.T, dbPath string, want int) { + t.Helper() + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open restored db: %v", err) + } + defer db.Close() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM tenants`).Scan(&got); err != nil { + t.Fatalf("query restored db: %v", err) + } + if got != want { + t.Fatalf("restored tenant count = %d, want %d", got, want) + } +} + func sortedEntryNames(entries map[string][]byte) []string { names := make([]string, 0, len(entries)) for name := range entries { diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 9581ad281..204357218 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -51,6 +51,9 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "docker compose run --rm control-plane provider-msp status", "docker compose run --rm control-plane provider-msp backup create", "docker compose run --rm control-plane provider-msp backup verify", + "docker compose run --rm control-plane provider-msp backup restore", + "--target-data-dir", + "--dry-run", "docker compose run --rm control-plane provider-msp proof", "--account-name", "--owner-email",