From c9f84c51929b7bdca363e91c23e250f984f99e41 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 2 Jun 2026 14:03:19 +0100 Subject: [PATCH] Add provider MSP backup command --- cmd/pulse-control-plane/provider_msp.go | 1 + .../provider_msp_backup.go | 125 +++ .../provider_msp_backup_test.go | 29 + deploy/provider-msp/.env.example | 7 + .../v6/internal/subsystems/cloud-paid.md | 12 +- .../subsystems/deployment-installability.md | 31 +- .../v6/internal/subsystems/registry.json | 14 + internal/cloudcp/provider_msp_backup.go | 754 ++++++++++++++++++ internal/cloudcp/provider_msp_backup_test.go | 258 ++++++ .../installtests/provider_msp_deploy_test.go | 2 + .../release_control/subsystem_lookup_test.py | 2 + 11 files changed, 1224 insertions(+), 11 deletions(-) create mode 100644 cmd/pulse-control-plane/provider_msp_backup.go create mode 100644 cmd/pulse-control-plane/provider_msp_backup_test.go create mode 100644 internal/cloudcp/provider_msp_backup.go create mode 100644 internal/cloudcp/provider_msp_backup_test.go diff --git a/cmd/pulse-control-plane/provider_msp.go b/cmd/pulse-control-plane/provider_msp.go index 394efdc15..3aab05813 100644 --- a/cmd/pulse-control-plane/provider_msp.go +++ b/cmd/pulse-control-plane/provider_msp.go @@ -13,6 +13,7 @@ func newProviderMSPCmd() *cobra.Command { Short: "Operate a provider-hosted MSP control plane", } cmd.AddCommand(newProviderMSPBootstrapCmd()) + cmd.AddCommand(newProviderMSPBackupCmd()) cmd.AddCommand(newProviderMSPPreflightCmd()) cmd.AddCommand(newProviderMSPProofCmd()) cmd.AddCommand(newProviderMSPStatusCmd()) diff --git a/cmd/pulse-control-plane/provider_msp_backup.go b/cmd/pulse-control-plane/provider_msp_backup.go new file mode 100644 index 000000000..eb2469d95 --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_backup.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + "github.com/spf13/cobra" +) + +func newProviderMSPBackupCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "backup", + Short: "Create and verify provider-hosted MSP recovery archives", + } + cmd.AddCommand(newProviderMSPBackupCreateCmd()) + cmd.AddCommand(newProviderMSPBackupVerifyCmd()) + return cmd +} + +func newProviderMSPBackupCreateCmd() *cobra.Command { + var outputPath string + cmd := &cobra.Command{ + Use: "create", + Short: "Create a provider-hosted MSP recovery archive", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := cloudcp.LoadConfig() + if err != nil { + return fmt.Errorf("load control plane config: %w", err) + } + result, err := cloudcp.CreateProviderMSPBackup(cmd.Context(), cfg, outputPath) + if err != nil { + return err + } + printProviderMSPBackupCreateResult(result) + return nil + }, + } + cmd.Flags().StringVar(&outputPath, "output", "", "Backup archive path (default: /backups/provider-msp/provider-msp-backup-.tar.gz)") + return cmd +} + +func newProviderMSPBackupVerifyCmd() *cobra.Command { + var archivePath string + cmd := &cobra.Command{ + Use: "verify [archive]", + Short: "Verify a provider-hosted MSP recovery archive", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) == 1 { + if strings.TrimSpace(archivePath) != "" && archivePath != args[0] { + return fmt.Errorf("archive path provided both as --archive and positional argument") + } + archivePath = args[0] + } + result, err := cloudcp.VerifyProviderMSPBackup(cmd.Context(), archivePath) + if err != nil { + return err + } + printProviderMSPBackupVerifyResult(result) + return nil + }, + } + cmd.Flags().StringVar(&archivePath, "archive", "", "Backup archive path to verify") + return cmd +} + +func printProviderMSPBackupCreateResult(result *cloudcp.ProviderMSPBackupCreateResult) { + if result == nil { + fmt.Println("provider_msp_backup_created=false") + return + } + fmt.Println("provider_msp_backup_created=true") + fmt.Printf("archive_path=%s\n", result.ArchivePath) + fmt.Printf("archive_bytes=%d\n", result.BytesWritten) + printProviderMSPBackupManifest(result.Manifest) + fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries) + fmt.Printf("tenant_entries=%d\n", result.TenantEntries) + fmt.Printf("license_entries=%d\n", result.LicenseEntries) +} + +func printProviderMSPBackupVerifyResult(result *cloudcp.ProviderMSPBackupVerifyResult) { + if result == nil { + fmt.Println("provider_msp_backup_verified=false") + return + } + fmt.Println("provider_msp_backup_verified=true") + fmt.Printf("archive_path=%s\n", result.ArchivePath) + fmt.Printf("archive_bytes=%d\n", result.VerifiedArchiveBytes) + printProviderMSPBackupManifest(result.Manifest) + fmt.Printf("control_plane_entries=%d\n", result.ControlPlaneEntries) + fmt.Printf("tenant_entries=%d\n", result.TenantEntries) + fmt.Printf("license_entries=%d\n", result.LicenseEntries) + fmt.Printf("tenant_registry_db_present=%t\n", result.HasTenantRegistryDB) + fmt.Printf("license_file_present=%t\n", result.HasLicenseFile) + for _, dbFile := range result.ControlPlaneDBFiles { + fmt.Printf("control_plane_db_backup=%s\n", dbFile) + } + for _, tenantID := range result.RuntimeTenantDirs { + fmt.Printf("runtime_tenant_dir=%s\n", tenantID) + } +} + +func printProviderMSPBackupManifest(manifest cloudcp.ProviderMSPBackupManifest) { + fmt.Printf("manifest_version=%s\n", manifest.Version) + fmt.Printf("created_at=%s\n", manifest.CreatedAt.Format("2006-01-02T15:04:05Z07:00")) + fmt.Printf("control_plane_mode=%s\n", manifest.ControlPlaneMode) + fmt.Printf("environment=%s\n", manifest.Environment) + fmt.Printf("base_url=%s\n", manifest.BaseURL) + fmt.Printf("plan_version=%s\n", manifest.PlanVersion) + fmt.Printf("plan_source=%s\n", manifest.PlanSource) + fmt.Printf("license_id=%s\n", manifest.LicenseID) + fmt.Printf("license_email=%s\n", manifest.LicenseEmail) + fmt.Printf("license_included=%t\n", manifest.LicenseIncluded) + fmt.Printf("workspace_limit=%d\n", manifest.WorkspaceLimit) + fmt.Printf("registry_account_count=%d\n", manifest.RegistryAccountCount) + fmt.Printf("registry_tenant_count=%d\n", manifest.RegistryTenantCount) + fmt.Printf("runtime_tenant_count=%d\n", manifest.RuntimeTenantCount) + for _, tenantID := range manifest.RuntimeTenantIDs { + fmt.Printf("runtime_tenant_id=%s\n", tenantID) + } + for _, dbFile := range manifest.ControlPlaneDBBackups { + fmt.Printf("manifest_db_backup=%s\n", dbFile) + } +} diff --git a/cmd/pulse-control-plane/provider_msp_backup_test.go b/cmd/pulse-control-plane/provider_msp_backup_test.go new file mode 100644 index 000000000..2f73ef518 --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_backup_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +func TestProviderMSPCommandExposesBackup(t *testing.T) { + cmd := newProviderMSPCmd() + for _, child := range cmd.Commands() { + if child.Name() == "backup" { + foundCreate := false + foundVerify := false + for _, backupChild := range child.Commands() { + switch backupChild.Name() { + case "create": + foundCreate = true + case "verify": + foundVerify = true + } + } + if !foundCreate { + t.Fatal("provider-msp backup create command is not registered") + } + if !foundVerify { + t.Fatal("provider-msp backup verify command is not registered") + } + return + } + } + t.Fatal("provider-msp backup command is not registered") +} diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index 94c993487..17b8f8060 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -43,6 +43,13 @@ PULSE_EMAIL_REPLY_TO=support@example.com # Non-mutating operational status: # docker compose run --rm control-plane provider-msp status # +# Recovery backup before upgrades or recovery drills: +# docker compose run --rm control-plane provider-msp backup create +# +# Verify a recovery archive before relying on it: +# docker compose run --rm control-plane provider-msp backup verify \ +# /data/backups/provider-msp/provider-msp-backup-YYYYMMDDTHHMMSSZ.tar.gz +# # End-to-end provider proof. This creates two proof client workspaces, generates # client-bound agent install tokens, verifies tenant-local agent report ingest, # and exercises the portal handoff path. Keep --cleanup once you have captured diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 12b003438..76d58e830 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -29,6 +29,10 @@ Provider-hosted MSP status uses the same tenant registry, health summary, and stuck-provisioning threshold as the control-plane cleanup loop, so operator readiness reports and automated failure handling cannot drift into separate definitions of a failed provider workspace. +Provider-hosted MSP backup and verification are part of the cloud-paid operator +contract because the recovery artifact must preserve the provider's +license-backed plan identity, control-plane account/workspace registry, and +tenant-local runtime state without depending on Stripe billing surfaces. ## Canonical Files @@ -112,6 +116,7 @@ definitions of a failed provider workspace. 88. `internal/hosted/hosted_metrics.go`, `internal/hosted/reaper.go` 89. `internal/cloudcp/public_msp_signup_handlers.go` 90. `internal/cloudcp/provider_msp_bootstrap.go` +91. `internal/cloudcp/provider_msp_backup.go` ## Shared Boundaries @@ -182,7 +187,12 @@ definitions of a failed provider workspace. Pulse Pro operations bundle on a recurring systemd timer, write durable status/log output, and emit Prometheus textfile metrics so a clean GA baseline is continuously monitored rather than manually rediscovered. -10. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. +10. `internal/cloudcp/provider_msp_backup.go` shared with `deployment-installability`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. + License-backed provider MSP backups must include the signed MSP license + file as a recovery artifact while exposing only license metadata in command + output, and they must keep the archive Stripe-free so provider-hosted MSP + recovery does not inherit Pulse-hosted SaaS billing assumptions. +11. `internal/cloudcp/tenant_runtime_rollout.go` shared with `deployment-installability`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. Hosted tenant runtime reconciliation must treat a registered tenant with preserved tenant data but no live Docker runtime as a recoverable managed state, not as a terminal skip. The control-plane-owned reconcile path must diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 575d28277..0d2aa7c04 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -27,12 +27,14 @@ surfaces. 4. `cmd/pulse-control-plane/main.go` 5. `cmd/pulse-control-plane/mobile_proof_cmd.go` 6. `cmd/pulse-control-plane/provider_msp.go` -7. `cmd/pulse-control-plane/provider_msp_preflight.go` -8. `cmd/pulse-control-plane/provider_msp_proof.go` -9. `cmd/pulse-control-plane/provider_msp_status.go` -10. `internal/cloudcp/docker/manager.go` -11. `internal/cloudcp/docker/labels.go` -12. `internal/cloudcp/tenant_runtime_rollout.go` +7. `cmd/pulse-control-plane/provider_msp_backup.go` +8. `cmd/pulse-control-plane/provider_msp_preflight.go` +9. `cmd/pulse-control-plane/provider_msp_proof.go` +10. `cmd/pulse-control-plane/provider_msp_status.go` +11. `internal/cloudcp/provider_msp_backup.go` +12. `internal/cloudcp/docker/manager.go` +13. `internal/cloudcp/docker/labels.go` +14. `internal/cloudcp/tenant_runtime_rollout.go` 13. `.github/workflows/create-release.yml` 14. `.github/workflows/deploy-demo-server.yml` 15. `.github/workflows/helm-pages.yml` @@ -117,7 +119,7 @@ surfaces. 2. `internal/api/updates.go` shared with `api-contracts`: update handlers are both a deployment-installability control surface and a canonical API payload contract boundary. 3. `internal/cloudcp/docker/labels.go` shared with `cloud-paid`: hosted tenant Docker labels are both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary. 4. `internal/cloudcp/docker/manager.go` shared with `cloud-paid`: hosted tenant container management is both a Pulse Cloud runtime contract boundary and a deployment-installability rollout boundary. - Tenant runtime containers must be created with bounded Docker `json-file` + Tenant runtime containers must use bounded Docker `json-file` logging so rollout and canary fleets cannot consume unbounded production host storage while they remain running. Provider-hosted MSP workspace creation and preflight must prepare or report @@ -137,13 +139,22 @@ surfaces. state/health counts, stuck provisioning workspaces, Docker runtime prerequisites, storage guardrails, and the same license-backed plan identity without pulling tenant images unless the operator asks for it. -5. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. -6. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. +5. `internal/cloudcp/provider_msp_backup.go` shared with `cloud-paid`: provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary. + `pulse-control-plane provider-msp backup create` and `backup verify` must + create a Stripe-free recovery archive outside the live + control-plane/tenant source trees, snapshot SQLite control-plane databases + through an online backup path, include tenant runtime directories for all + non-deleted registry workspaces, include the signed MSP license file when + the plan source is license-backed, and verify the manifest, tenant registry + snapshot, license artifact, and tenant runtime directories before the + archive is treated as usable for upgrades or recovery drills. +6. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. +7. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. It must expose a non-mutating preflight for the exact Windows agent architecture before Administrator-only install changes, accept token-file enrollment input, and avoid interactive download-failure prompts when launched by generated non-interactive onboarding commands. -7. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. +8. `scripts/install.sh` shared with `agent-lifecycle`: the shell installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. ## Extension Points diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index a86f0c189..43257345e 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -634,6 +634,14 @@ "deployment-installability" ] }, + { + "path": "internal/cloudcp/provider_msp_backup.go", + "rationale": "provider-hosted MSP backup is both a cloud-paid license/account/runtime continuity boundary and a deployment-installability recovery artifact boundary", + "subsystems": [ + "cloud-paid", + "deployment-installability" + ] + }, { "path": "internal/cloudcp/tenant_runtime_rollout.go", "rationale": "hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary", @@ -2624,6 +2632,7 @@ ".github/workflows/validate-release-assets.yml", "cmd/pulse-control-plane/main.go", "cmd/pulse-control-plane/provider_msp.go", + "cmd/pulse-control-plane/provider_msp_backup.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", "cmd/pulse-control-plane/provider_msp_status.go", @@ -2644,6 +2653,7 @@ "internal/api/updates.go", "internal/cloudcp/docker/labels.go", "internal/cloudcp/docker/manager.go", + "internal/cloudcp/provider_msp_backup.go", "internal/cloudcp/tenant_runtime_rollout.go", "Makefile", "package-lock.json", @@ -2893,20 +2903,24 @@ "match_files": [ "cmd/pulse-control-plane/main.go", "cmd/pulse-control-plane/provider_msp.go", + "cmd/pulse-control-plane/provider_msp_backup.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", "cmd/pulse-control-plane/provider_msp_status.go", "internal/cloudcp/docker/labels.go", "internal/cloudcp/docker/manager.go", + "internal/cloudcp/provider_msp_backup.go", "internal/cloudcp/tenant_runtime_rollout.go" ], "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ + "cmd/pulse-control-plane/provider_msp_backup_test.go", "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", + "internal/cloudcp/provider_msp_backup_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go" ] diff --git a/internal/cloudcp/provider_msp_backup.go b/internal/cloudcp/provider_msp_backup.go new file mode 100644 index 000000000..01780b1a1 --- /dev/null +++ b/internal/cloudcp/provider_msp_backup.go @@ -0,0 +1,754 @@ +package cloudcp + +import ( + "archive/tar" + "compress/gzip" + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "net/url" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" + pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" + _ "modernc.org/sqlite" +) + +const ( + ProviderMSPBackupManifestVersion = "provider-msp-backup/v1" + providerMSPBackupManifestName = "manifest.json" + providerMSPBackupControlPlaneDir = "control-plane" + providerMSPBackupTenantsDir = "tenants" + providerMSPBackupLicenseDir = "license" + providerMSPBackupLicenseName = "provider-msp-license.jwt" + maxProviderMSPBackupManifestSize = 1024 * 1024 +) + +// ProviderMSPBackupManifest is the recovery contract stored inside every +// provider-hosted MSP backup archive. +type ProviderMSPBackupManifest struct { + Version string `json:"version"` + CreatedAt time.Time `json:"created_at"` + ControlPlaneMode string `json:"control_plane_mode"` + Environment string `json:"environment"` + BaseURL string `json:"base_url"` + PlanVersion string `json:"plan_version"` + PlanSource string `json:"plan_source"` + LicenseID string `json:"license_id,omitempty"` + LicenseEmail string `json:"license_email,omitempty"` + LicenseIncluded bool `json:"license_included"` + WorkspaceLimit int `json:"workspace_limit"` + ControlPlaneDir string `json:"control_plane_dir"` + TenantsDir string `json:"tenants_dir"` + RegistryAccountCount int `json:"registry_account_count"` + RegistryTenantCount int `json:"registry_tenant_count"` + RuntimeTenantCount int `json:"runtime_tenant_count"` + RuntimeTenantIDs []string `json:"runtime_tenant_ids"` + RegistryTenantIDs []string `json:"registry_tenant_ids"` + RegistryStateCounts map[string]int `json:"registry_state_counts"` + ControlPlaneDBBackups []string `json:"control_plane_db_backups"` +} + +// ProviderMSPBackupCreateResult describes a newly written provider MSP backup. +type ProviderMSPBackupCreateResult struct { + ArchivePath string + Manifest ProviderMSPBackupManifest + ControlPlaneEntries int + TenantEntries int + LicenseEntries int + BytesWritten int64 +} + +// ProviderMSPBackupVerifyResult describes a verified provider MSP backup. +type ProviderMSPBackupVerifyResult struct { + ArchivePath string + Manifest ProviderMSPBackupManifest + ControlPlaneEntries int + TenantEntries int + LicenseEntries int + HasTenantRegistryDB bool + HasLicenseFile bool + RuntimeTenantDirs []string + ControlPlaneDBFiles []string + VerifiedArchiveBytes int64 +} + +// DefaultProviderMSPBackupPath returns the compose-friendly default backup +// location under CP_DATA_DIR without placing the archive inside the source trees. +func DefaultProviderMSPBackupPath(cfg *CPConfig, now time.Time) string { + if now.IsZero() { + now = time.Now().UTC() + } + name := "provider-msp-backup-" + now.UTC().Format("20060102T150405Z") + ".tar.gz" + if cfg == nil || strings.TrimSpace(cfg.DataDir) == "" { + return name + } + return filepath.Join(cfg.DataDir, "backups", "provider-msp", name) +} + +// CreateProviderMSPBackup creates a recovery archive for a provider-hosted MSP +// control plane. It backs up the control-plane registry through SQLite's online +// VACUUM INTO path instead of copying live WAL files directly. +func CreateProviderMSPBackup(ctx context.Context, cfg *CPConfig, outputPath string) (*ProviderMSPBackupCreateResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if cfg == nil { + return nil, fmt.Errorf("control plane config is required") + } + if !cfg.IsProviderHostedMSP() { + return nil, fmt.Errorf("provider MSP backup requires CP_CONTROL_PLANE_MODE=%s", ControlPlaneModeProviderHostedMSP) + } + if cfg.UsesStripeBilling() { + return nil, fmt.Errorf("provider MSP backup is unavailable for Stripe-backed control planes") + } + if strings.TrimSpace(outputPath) == "" { + outputPath = DefaultProviderMSPBackupPath(cfg, time.Now().UTC()) + } + + archivePath, err := filepath.Abs(filepath.Clean(outputPath)) + if err != nil { + return nil, fmt.Errorf("resolve backup archive path: %w", err) + } + controlPlaneDir, err := filepath.Abs(filepath.Clean(cfg.ControlPlaneDir())) + if err != nil { + return nil, fmt.Errorf("resolve control-plane dir: %w", err) + } + tenantsDir, err := filepath.Abs(filepath.Clean(cfg.TenantsDir())) + if err != nil { + return nil, fmt.Errorf("resolve tenants dir: %w", err) + } + if pathIsInside(archivePath, controlPlaneDir) || pathIsInside(archivePath, tenantsDir) { + return nil, fmt.Errorf("backup archive must not be written inside %s or %s", cfg.ControlPlaneDir(), cfg.TenantsDir()) + } + + if err := requireDirectory(controlPlaneDir, "control-plane dir"); err != nil { + return nil, err + } + registryDB := filepath.Join(controlPlaneDir, "tenants.db") + if err := requireRegularFile(registryDB, "tenant registry database"); err != nil { + return nil, err + } + + reg, err := registry.NewTenantRegistry(controlPlaneDir) + if err != nil { + return nil, fmt.Errorf("open tenant registry: %w", err) + } + tenants, listErr := reg.List() + accounts, accountsErr := reg.ListAccounts() + closeErr := reg.Close() + if listErr != nil { + return nil, fmt.Errorf("list tenant registry rows: %w", listErr) + } + if accountsErr != nil { + return nil, fmt.Errorf("list registry accounts: %w", accountsErr) + } + if closeErr != nil { + return nil, fmt.Errorf("close tenant registry before backup: %w", closeErr) + } + + manifest, err := buildProviderMSPBackupManifest(cfg, tenants, len(accounts)) + if err != nil { + return nil, err + } + if err := requireProviderMSPRuntimeTenantDirs(tenantsDir, manifest.RuntimeTenantIDs); err != nil { + return nil, err + } + licensePath, licenseIncluded, err := resolveProviderMSPBackupLicensePath(cfg) + if err != nil { + return nil, err + } + manifest.LicenseIncluded = licenseIncluded + + if err := os.MkdirAll(filepath.Dir(archivePath), 0o755); err != nil { + return nil, fmt.Errorf("create backup archive parent: %w", err) + } + tempDir, err := os.MkdirTemp(filepath.Dir(archivePath), ".provider-msp-backup-*") + if err != nil { + return nil, fmt.Errorf("create backup staging dir: %w", err) + } + defer os.RemoveAll(tempDir) + + tmpArchive := archivePath + ".tmp" + _ = os.Remove(tmpArchive) + file, err := os.OpenFile(tmpArchive, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return nil, fmt.Errorf("create backup archive: %w", err) + } + result, err := writeProviderMSPBackupArchive(ctx, file, tempDir, controlPlaneDir, tenantsDir, licensePath, manifest) + closeArchiveErr := file.Close() + if err != nil { + _ = os.Remove(tmpArchive) + return nil, err + } + if closeArchiveErr != nil { + _ = os.Remove(tmpArchive) + return nil, fmt.Errorf("close backup archive: %w", closeArchiveErr) + } + if err := os.Rename(tmpArchive, archivePath); err != nil { + _ = os.Remove(tmpArchive) + return nil, fmt.Errorf("publish backup archive: %w", err) + } + info, err := os.Stat(archivePath) + if err != nil { + return nil, fmt.Errorf("stat backup archive: %w", err) + } + result.ArchivePath = archivePath + result.BytesWritten = info.Size() + return result, nil +} + +// VerifyProviderMSPBackup validates that a provider MSP backup contains the +// manifest, tenant registry snapshot, required tenant directories, and license +// artifact declared by the manifest. +func VerifyProviderMSPBackup(ctx context.Context, archivePath string) (*ProviderMSPBackupVerifyResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + archivePath = strings.TrimSpace(archivePath) + if archivePath == "" { + return nil, fmt.Errorf("backup archive path is required") + } + resolvedArchivePath, err := filepath.Abs(filepath.Clean(archivePath)) + if err != nil { + return nil, fmt.Errorf("resolve backup archive path: %w", err) + } + file, err := os.Open(resolvedArchivePath) + if err != nil { + return nil, fmt.Errorf("open backup archive: %w", err) + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("stat backup archive: %w", err) + } + gz, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("open backup gzip stream: %w", err) + } + defer gz.Close() + + result := &ProviderMSPBackupVerifyResult{ + ArchivePath: resolvedArchivePath, + VerifiedArchiveBytes: info.Size(), + } + tenantDirs := map[string]bool{} + dbFiles := map[string]bool{} + var manifestBytes []byte + tr := tar.NewReader(gz) + for { + if err := ctx.Err(); err != nil { + return nil, err + } + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("read backup archive: %w", err) + } + name, err := cleanProviderMSPArchiveName(header.Name) + if err != nil { + return nil, err + } + switch { + case name == providerMSPBackupManifestName: + manifestBytes, err = readProviderMSPBackupManifestBytes(tr, header.Size) + if err != nil { + return nil, err + } + case name == providerMSPBackupControlPlaneDir || strings.HasPrefix(name, providerMSPBackupControlPlaneDir+"/"): + result.ControlPlaneEntries++ + if strings.HasSuffix(name, ".db") { + dbFiles[name] = true + } + if name == providerMSPBackupControlPlaneDir+"/tenants.db" { + result.HasTenantRegistryDB = true + } + case name == providerMSPBackupTenantsDir || strings.HasPrefix(name, providerMSPBackupTenantsDir+"/"): + result.TenantEntries++ + if tenantID := archiveTenantID(name); tenantID != "" { + tenantDirs[tenantID] = true + } + case name == providerMSPBackupLicenseDir || strings.HasPrefix(name, providerMSPBackupLicenseDir+"/"): + result.LicenseEntries++ + if name == providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName { + result.HasLicenseFile = true + } + default: + return nil, fmt.Errorf("unexpected top-level backup entry %q", name) + } + } + if len(manifestBytes) == 0 { + return nil, fmt.Errorf("backup manifest is missing") + } + if err := json.Unmarshal(manifestBytes, &result.Manifest); err != nil { + return nil, fmt.Errorf("parse backup manifest: %w", err) + } + if err := validateProviderMSPBackupManifest(result.Manifest); err != nil { + return nil, err + } + if !result.HasTenantRegistryDB { + return nil, fmt.Errorf("backup is missing control-plane tenant registry snapshot") + } + if result.ControlPlaneEntries == 0 { + return nil, fmt.Errorf("backup is missing control-plane data") + } + if result.TenantEntries == 0 { + return nil, fmt.Errorf("backup is missing tenants directory") + } + if result.Manifest.LicenseIncluded && !result.HasLicenseFile { + return nil, fmt.Errorf("backup manifest declares included MSP license but license file is missing") + } + for _, dbFile := range result.Manifest.ControlPlaneDBBackups { + if !dbFiles[dbFile] { + return nil, fmt.Errorf("backup manifest declares control-plane DB snapshot %q but it is missing", dbFile) + } + } + for _, tenantID := range result.Manifest.RuntimeTenantIDs { + if !tenantDirs[tenantID] { + return nil, fmt.Errorf("backup is missing runtime tenant directory %q", tenantID) + } + } + for tenantID := range tenantDirs { + result.RuntimeTenantDirs = append(result.RuntimeTenantDirs, tenantID) + } + sort.Strings(result.RuntimeTenantDirs) + for name := range dbFiles { + result.ControlPlaneDBFiles = append(result.ControlPlaneDBFiles, name) + } + sort.Strings(result.ControlPlaneDBFiles) + return result, nil +} + +func buildProviderMSPBackupManifest(cfg *CPConfig, tenants []*registry.Tenant, accountCount int) (ProviderMSPBackupManifest, error) { + workspaceLimit, known := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion) + if !known { + return ProviderMSPBackupManifest{}, fmt.Errorf("provider MSP plan %q has no known workspace limit", cfg.ProviderMSPPlanVersion) + } + manifest := ProviderMSPBackupManifest{ + Version: ProviderMSPBackupManifestVersion, + CreatedAt: time.Now().UTC(), + ControlPlaneMode: string(cfg.ControlPlaneMode), + Environment: strings.TrimSpace(cfg.Environment), + BaseURL: strings.TrimSpace(cfg.BaseURL), + PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion), + PlanSource: providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource), + LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID), + LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)), + WorkspaceLimit: workspaceLimit, + ControlPlaneDir: providerMSPBackupControlPlaneDir, + TenantsDir: providerMSPBackupTenantsDir, + RegistryAccountCount: accountCount, + RegistryStateCounts: map[string]int{}, + ControlPlaneDBBackups: []string{}, + } + for _, tenant := range tenants { + if tenant == nil { + continue + } + tenantID := strings.TrimSpace(tenant.ID) + if tenantID == "" { + continue + } + manifest.RegistryTenantIDs = append(manifest.RegistryTenantIDs, tenantID) + manifest.RegistryStateCounts[string(tenant.State)]++ + if providerMSPBackupRequiresRuntimeTenantDir(tenant.State) { + manifest.RuntimeTenantIDs = append(manifest.RuntimeTenantIDs, tenantID) + } + } + sort.Strings(manifest.RegistryTenantIDs) + sort.Strings(manifest.RuntimeTenantIDs) + manifest.RegistryTenantCount = len(manifest.RegistryTenantIDs) + manifest.RuntimeTenantCount = len(manifest.RuntimeTenantIDs) + return manifest, nil +} + +func providerMSPBackupRequiresRuntimeTenantDir(state registry.TenantState) bool { + switch state { + case registry.TenantStateCanceled, registry.TenantStateDeleting, registry.TenantStateDeleted: + return false + default: + return true + } +} + +func resolveProviderMSPBackupLicensePath(cfg *CPConfig) (string, bool, error) { + if providerMSPPlanSourceOrDefault(cfg.ProviderMSPPlanSource) != ProviderMSPPlanSourceLicenseFile { + return "", false, nil + } + licensePath := strings.TrimSpace(cfg.ProviderMSPLicenseFile) + if licensePath == "" { + return "", false, fmt.Errorf("provider MSP backup requires CP_PROVIDER_MSP_LICENSE_FILE when plan_source=%s", ProviderMSPPlanSourceLicenseFile) + } + if err := requireRegularFile(licensePath, "provider MSP license file"); err != nil { + return "", false, err + } + return licensePath, true, nil +} + +func requireProviderMSPRuntimeTenantDirs(tenantsDir string, tenantIDs []string) error { + if len(tenantIDs) == 0 { + return nil + } + if err := requireDirectory(tenantsDir, "tenants dir"); err != nil { + return err + } + for _, tenantID := range tenantIDs { + dir := filepath.Join(tenantsDir, tenantID) + if err := requireDirectory(dir, "runtime tenant dir "+tenantID); err != nil { + return err + } + } + return nil +} + +func writeProviderMSPBackupArchive(ctx context.Context, output io.Writer, tempDir, controlPlaneDir, tenantsDir, licensePath string, manifest ProviderMSPBackupManifest) (*ProviderMSPBackupCreateResult, error) { + gz := gzip.NewWriter(output) + tw := tar.NewWriter(gz) + result := &ProviderMSPBackupCreateResult{ + Manifest: manifest, + } + controlPlaneEntries, dbBackups, err := addProviderMSPBackupDir(ctx, tw, tempDir, controlPlaneDir, providerMSPBackupControlPlaneDir, true, manifest.CreatedAt) + if err != nil { + return nil, err + } + result.ControlPlaneEntries = controlPlaneEntries + manifest.ControlPlaneDBBackups = dbBackups + tenantEntries, _, err := addProviderMSPBackupDir(ctx, tw, tempDir, tenantsDir, providerMSPBackupTenantsDir, false, manifest.CreatedAt) + if err != nil { + return nil, err + } + result.TenantEntries = tenantEntries + if strings.TrimSpace(licensePath) != "" { + if err := writeProviderMSPBackupRootDir(tw, providerMSPBackupLicenseDir, manifest.CreatedAt); err != nil { + return nil, err + } + result.LicenseEntries++ + if err := addProviderMSPBackupFile(tw, licensePath, providerMSPBackupLicenseDir+"/"+providerMSPBackupLicenseName, 0); err != nil { + return nil, err + } + result.LicenseEntries++ + } + result.Manifest = manifest + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, fmt.Errorf("encode backup manifest: %w", err) + } + manifestBytes = append(manifestBytes, '\n') + if err := writeProviderMSPBackupBytes(tw, providerMSPBackupManifestName, manifestBytes, 0o600, manifest.CreatedAt); err != nil { + return nil, err + } + if err := tw.Close(); err != nil { + return nil, fmt.Errorf("close backup tar stream: %w", err) + } + if err := gz.Close(); err != nil { + return nil, fmt.Errorf("close backup gzip stream: %w", err) + } + return result, nil +} + +func addProviderMSPBackupDir(ctx context.Context, tw *tar.Writer, tempDir, sourceDir, archiveRoot string, snapshotSQLite bool, createdAt time.Time) (int, []string, error) { + if strings.TrimSpace(sourceDir) == "" { + return 0, nil, fmt.Errorf("source dir is required for %s", archiveRoot) + } + entries := 0 + dbBackups := []string{} + if err := writeProviderMSPBackupRootDir(tw, archiveRoot, createdAt); err != nil { + return 0, nil, err + } + entries++ + if _, err := os.Stat(sourceDir); os.IsNotExist(err) { + return entries, dbBackups, nil + } else if err != nil { + return 0, nil, fmt.Errorf("stat %s: %w", sourceDir, err) + } + err := filepath.WalkDir(sourceDir, func(filePath string, entry os.DirEntry, walkErr error) error { + if err := ctx.Err(); err != nil { + return err + } + if walkErr != nil { + return fmt.Errorf("walk %s: %w", filePath, walkErr) + } + rel, err := filepath.Rel(sourceDir, filePath) + if err != nil { + return fmt.Errorf("resolve backup relative path: %w", err) + } + if rel == "." { + return nil + } + archiveName := filepath.ToSlash(filepath.Join(archiveRoot, rel)) + info, err := entry.Info() + if err != nil { + return fmt.Errorf("stat backup source %s: %w", filePath, err) + } + if entry.Type()&os.ModeSymlink != 0 { + if err := addProviderMSPBackupSymlink(tw, filePath, archiveName, info); err != nil { + return err + } + entries++ + return nil + } + if entry.IsDir() { + if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil { + return err + } + entries++ + return nil + } + if !entry.Type().IsRegular() { + return fmt.Errorf("unsupported backup source file type at %s", filePath) + } + if snapshotSQLite && isProviderMSPSQLiteSidecar(rel) { + return nil + } + if snapshotSQLite && strings.EqualFold(filepath.Ext(filePath), ".db") { + snapshotPath := filepath.Join(tempDir, archiveRoot, rel) + if err := snapshotSQLiteDatabase(filePath, snapshotPath); err != nil { + return err + } + if err := addProviderMSPBackupFile(tw, snapshotPath, archiveName, info.Mode().Perm()); err != nil { + return err + } + dbBackups = append(dbBackups, archiveName) + entries++ + return nil + } + if err := addProviderMSPBackupFile(tw, filePath, archiveName, 0); err != nil { + return err + } + entries++ + return nil + }) + sort.Strings(dbBackups) + if err != nil { + return 0, nil, err + } + return entries, dbBackups, nil +} + +func snapshotSQLiteDatabase(sourcePath, destPath string) error { + if err := requireRegularFile(sourcePath, "sqlite database"); err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil { + return fmt.Errorf("create sqlite snapshot parent: %w", err) + } + _ = os.Remove(destPath) + dsn := sourcePath + "?" + url.Values{ + "_pragma": []string{ + "busy_timeout(30000)", + "foreign_keys(ON)", + }, + }.Encode() + db, err := sql.Open("sqlite", dsn) + if err != nil { + return fmt.Errorf("open sqlite database for backup: %w", err) + } + defer db.Close() + if _, err := db.Exec("VACUUM INTO ?", destPath); err != nil { + return fmt.Errorf("snapshot sqlite database %s: %w", filepath.Base(sourcePath), err) + } + return nil +} + +func addProviderMSPBackupFile(tw *tar.Writer, sourcePath, archiveName string, modeOverride os.FileMode) error { + info, err := os.Lstat(sourcePath) + if err != nil { + return fmt.Errorf("stat backup file %s: %w", sourcePath, err) + } + if modeOverride != 0 { + info = providerMSPFileInfoWithMode{FileInfo: info, mode: modeOverride} + } + file, err := os.Open(sourcePath) + if err != nil { + return fmt.Errorf("open backup file %s: %w", sourcePath, err) + } + defer file.Close() + if err := addProviderMSPBackupHeader(tw, info, archiveName, ""); err != nil { + return err + } + if _, err := io.Copy(tw, file); err != nil { + return fmt.Errorf("write backup file %s: %w", archiveName, err) + } + return nil +} + +func addProviderMSPBackupSymlink(tw *tar.Writer, sourcePath, archiveName string, info os.FileInfo) error { + target, err := os.Readlink(sourcePath) + if err != nil { + return fmt.Errorf("read backup symlink %s: %w", sourcePath, err) + } + return addProviderMSPBackupHeader(tw, info, archiveName, target) +} + +func addProviderMSPBackupHeader(tw *tar.Writer, info os.FileInfo, archiveName, linkTarget string) error { + header, err := tar.FileInfoHeader(info, linkTarget) + if err != nil { + return fmt.Errorf("create backup tar header %s: %w", archiveName, err) + } + header.Name = archiveName + header.Uid = 0 + header.Gid = 0 + header.Uname = "" + header.Gname = "" + if info.IsDir() && !strings.HasSuffix(header.Name, "/") { + header.Name += "/" + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("write backup tar header %s: %w", archiveName, err) + } + return nil +} + +func writeProviderMSPBackupRootDir(tw *tar.Writer, archiveName string, modTime time.Time) error { + name := strings.TrimSuffix(archiveName, "/") + "/" + header := &tar.Header{ + Name: name, + Mode: 0o755, + Typeflag: tar.TypeDir, + ModTime: modTime, + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("write backup tar header %s: %w", archiveName, err) + } + return nil +} + +func writeProviderMSPBackupBytes(tw *tar.Writer, archiveName string, content []byte, mode int64, modTime time.Time) error { + header := &tar.Header{ + Name: archiveName, + Mode: mode, + Size: int64(len(content)), + Typeflag: tar.TypeReg, + ModTime: modTime, + } + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("write backup tar header %s: %w", archiveName, err) + } + if _, err := tw.Write(content); err != nil { + return fmt.Errorf("write backup entry %s: %w", archiveName, err) + } + return nil +} + +func validateProviderMSPBackupManifest(manifest ProviderMSPBackupManifest) error { + if manifest.Version != ProviderMSPBackupManifestVersion { + return fmt.Errorf("unsupported provider MSP backup manifest version %q", manifest.Version) + } + if manifest.ControlPlaneMode != string(ControlPlaneModeProviderHostedMSP) { + return fmt.Errorf("backup control_plane_mode = %q, want %q", manifest.ControlPlaneMode, ControlPlaneModeProviderHostedMSP) + } + if strings.TrimSpace(manifest.PlanVersion) == "" { + return fmt.Errorf("backup manifest is missing plan_version") + } + if strings.TrimSpace(manifest.PlanSource) == "" { + return fmt.Errorf("backup manifest is missing plan_source") + } + if manifest.ControlPlaneDir != providerMSPBackupControlPlaneDir { + return fmt.Errorf("backup control_plane_dir = %q, want %q", manifest.ControlPlaneDir, providerMSPBackupControlPlaneDir) + } + if manifest.TenantsDir != providerMSPBackupTenantsDir { + return fmt.Errorf("backup tenants_dir = %q, want %q", manifest.TenantsDir, providerMSPBackupTenantsDir) + } + if manifest.RegistryTenantCount != len(manifest.RegistryTenantIDs) { + return fmt.Errorf("backup registry_tenant_count = %d, want %d ids", manifest.RegistryTenantCount, len(manifest.RegistryTenantIDs)) + } + if manifest.RuntimeTenantCount != len(manifest.RuntimeTenantIDs) { + return fmt.Errorf("backup runtime_tenant_count = %d, want %d ids", manifest.RuntimeTenantCount, len(manifest.RuntimeTenantIDs)) + } + return nil +} + +func readProviderMSPBackupManifestBytes(reader io.Reader, size int64) ([]byte, error) { + if size < 0 || size > maxProviderMSPBackupManifestSize { + return nil, fmt.Errorf("backup manifest size %d is outside allowed range", size) + } + limited := io.LimitReader(reader, maxProviderMSPBackupManifestSize+1) + content, err := io.ReadAll(limited) + if err != nil { + return nil, fmt.Errorf("read backup manifest: %w", err) + } + if len(content) > maxProviderMSPBackupManifestSize { + return nil, fmt.Errorf("backup manifest exceeds %d bytes", maxProviderMSPBackupManifestSize) + } + return content, nil +} + +func cleanProviderMSPArchiveName(raw string) (string, error) { + name := strings.TrimSpace(strings.ReplaceAll(raw, "\\", "/")) + if name == "" { + return "", fmt.Errorf("backup archive contains empty entry name") + } + name = strings.TrimPrefix(name, "./") + cleaned := path.Clean(name) + if cleaned == "." || strings.HasPrefix(cleaned, "../") || path.IsAbs(cleaned) { + return "", fmt.Errorf("backup archive contains unsafe entry name %q", raw) + } + return cleaned, nil +} + +func archiveTenantID(name string) string { + if name == providerMSPBackupTenantsDir || !strings.HasPrefix(name, providerMSPBackupTenantsDir+"/") { + return "" + } + rest := strings.TrimPrefix(name, providerMSPBackupTenantsDir+"/") + parts := strings.Split(rest, "/") + if len(parts) == 0 { + return "" + } + return strings.TrimSpace(parts[0]) +} + +func isProviderMSPSQLiteSidecar(rel string) bool { + name := strings.ToLower(filepath.Base(rel)) + return strings.HasSuffix(name, ".db-wal") || strings.HasSuffix(name, ".db-shm") +} + +func pathIsInside(candidate, parent string) bool { + candidate = filepath.Clean(candidate) + parent = filepath.Clean(parent) + rel, err := filepath.Rel(parent, candidate) + if err != nil { + return false + } + return rel == "." || (rel != ".." && !strings.HasPrefix(rel, ".."+string(filepath.Separator))) +} + +func requireDirectory(dir, label string) error { + info, err := os.Stat(dir) + if err != nil { + return fmt.Errorf("%s unavailable: %w", label, err) + } + if !info.IsDir() { + return fmt.Errorf("%s %q is not a directory", label, dir) + } + return nil +} + +func requireRegularFile(filePath, label string) error { + info, err := os.Stat(filePath) + if err != nil { + return fmt.Errorf("%s unavailable: %w", label, err) + } + if !info.Mode().IsRegular() { + return fmt.Errorf("%s %q is not a regular file", label, filePath) + } + return nil +} + +type providerMSPFileInfoWithMode struct { + os.FileInfo + mode os.FileMode +} + +func (i providerMSPFileInfoWithMode) Mode() os.FileMode { + return i.mode +} diff --git a/internal/cloudcp/provider_msp_backup_test.go b/internal/cloudcp/provider_msp_backup_test.go new file mode 100644 index 000000000..a04dfabbf --- /dev/null +++ b/internal/cloudcp/provider_msp_backup_test.go @@ -0,0 +1,258 @@ +package cloudcp + +import ( + "archive/tar" + "compress/gzip" + "context" + "database/sql" + "io" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + cpauth "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/auth" + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" +) + +func TestProviderMSPBackupCreateVerifyIncludesRecoveryContract(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + reg := seedProviderMSPBackupRegistry(t, cfg) + if err := reg.Close(); err != nil { + t.Fatalf("close registry: %v", err) + } + magicLinks, err := cpauth.NewService(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewService: %v", err) + } + magicLinks.Close() + if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil { + t.Fatalf("create tenant dir: %v", err) + } + if err := os.WriteFile(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001", "runtime.json"), []byte(`{"ok":true}`), 0o600); err != nil { + t.Fatalf("write tenant runtime file: %v", err) + } + if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "operator-note.txt"), []byte("keep me"), 0o600); err != nil { + t.Fatalf("write control-plane note: %v", err) + } + if err := os.WriteFile(filepath.Join(cfg.ControlPlaneDir(), "tenants.db-wal"), []byte("stale wal sidecar"), 0o600); err != nil { + t.Fatalf("write wal sidecar: %v", err) + } + + archivePath := filepath.Join(t.TempDir(), "provider-msp-backup.tar.gz") + result, err := CreateProviderMSPBackup(context.Background(), cfg, archivePath) + if err != nil { + t.Fatalf("CreateProviderMSPBackup: %v", err) + } + if result.Manifest.Version != ProviderMSPBackupManifestVersion { + t.Fatalf("manifest version = %q", result.Manifest.Version) + } + if !result.Manifest.LicenseIncluded { + t.Fatal("expected provider MSP license to be included in recovery archive") + } + if result.Manifest.RegistryAccountCount != 1 || result.Manifest.RegistryTenantCount != 2 || result.Manifest.RuntimeTenantCount != 1 { + t.Fatalf("manifest counts = accounts %d registry tenants %d runtime tenants %d", result.Manifest.RegistryAccountCount, result.Manifest.RegistryTenantCount, result.Manifest.RuntimeTenantCount) + } + if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/tenants.db") { + t.Fatalf("manifest db backups = %#v, want tenant registry", result.Manifest.ControlPlaneDBBackups) + } + if !providerMSPBackupTestContainsString(result.Manifest.ControlPlaneDBBackups, "control-plane/cp_magic_links.db") { + t.Fatalf("manifest db backups = %#v, want magic-link db", result.Manifest.ControlPlaneDBBackups) + } + + verify, err := VerifyProviderMSPBackup(context.Background(), archivePath) + if err != nil { + t.Fatalf("VerifyProviderMSPBackup: %v", err) + } + if !verify.HasTenantRegistryDB || !verify.HasLicenseFile { + t.Fatalf("verify flags = registry %t license %t", verify.HasTenantRegistryDB, verify.HasLicenseFile) + } + if len(verify.RuntimeTenantDirs) != 1 || verify.RuntimeTenantDirs[0] != "t-ACTIVE001" { + t.Fatalf("runtime tenant dirs = %#v", verify.RuntimeTenantDirs) + } + + entries := readProviderMSPBackupEntries(t, archivePath) + for _, want := range []string{ + "manifest.json", + "control-plane/tenants.db", + "control-plane/cp_magic_links.db", + "control-plane/operator-note.txt", + "tenants/t-ACTIVE001/runtime.json", + "license/provider-msp-license.jwt", + } { + if _, ok := entries[want]; !ok { + t.Fatalf("backup missing %q; entries=%#v", want, sortedEntryNames(entries)) + } + } + if _, ok := entries["control-plane/tenants.db-wal"]; ok { + t.Fatalf("backup included raw sqlite WAL sidecar") + } + assertProviderMSPBackupRegistrySnapshotCount(t, entries["control-plane/tenants.db"], 2) +} + +func TestProviderMSPBackupRejectsArchiveInsideSourceTrees(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + reg := seedProviderMSPBackupRegistry(t, cfg) + if err := reg.Close(); err != nil { + t.Fatalf("close registry: %v", err) + } + if err := os.MkdirAll(filepath.Join(cfg.TenantsDir(), "t-ACTIVE001"), 0o755); err != nil { + t.Fatalf("create tenant dir: %v", err) + } + + _, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(cfg.ControlPlaneDir(), "backup.tar.gz")) + if err == nil { + t.Fatal("expected archive inside control-plane dir to be rejected") + } + if !strings.Contains(err.Error(), "must not be written inside") { + t.Fatalf("error = %v", err) + } +} + +func TestProviderMSPBackupRequiresRuntimeTenantDirs(t *testing.T) { + cfg := testProviderMSPBackupConfig(t) + reg := seedProviderMSPBackupRegistry(t, cfg) + if err := reg.Close(); err != nil { + t.Fatalf("close registry: %v", err) + } + if err := os.MkdirAll(cfg.TenantsDir(), 0o755); err != nil { + t.Fatalf("create tenants dir: %v", err) + } + + _, err := CreateProviderMSPBackup(context.Background(), cfg, filepath.Join(t.TempDir(), "backup.tar.gz")) + if err == nil { + t.Fatal("expected missing active tenant runtime dir to fail") + } + if !strings.Contains(err.Error(), "runtime tenant dir t-ACTIVE001") { + t.Fatalf("error = %v", err) + } +} + +func testProviderMSPBackupConfig(t *testing.T) *CPConfig { + t.Helper() + root := t.TempDir() + licenseFile := filepath.Join(root, "provider-msp-license.jwt") + if err := os.WriteFile(licenseFile, []byte("signed-provider-msp-license"), 0o600); err != nil { + t.Fatalf("write license: %v", err) + } + return &CPConfig{ + DataDir: filepath.Join(root, "data"), + Environment: "production", + ControlPlaneMode: ControlPlaneModeProviderHostedMSP, + BaseURL: "https://msp.example.com", + ProviderMSPPlanVersion: "msp_growth", + ProviderMSPPlanSource: ProviderMSPPlanSourceLicenseFile, + ProviderMSPLicenseFile: licenseFile, + ProviderMSPLicenseID: "lic_provider_msp_backup", + ProviderMSPLicenseEmail: "provider@example.com", + } +} + +func seedProviderMSPBackupRegistry(t *testing.T, cfg *CPConfig) *registry.TenantRegistry { + t.Helper() + reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewTenantRegistry: %v", err) + } + if err := reg.CreateAccount(®istry.Account{ + ID: "acct_backup", + Kind: registry.AccountKindMSP, + DisplayName: "Backup MSP", + }); err != nil { + t.Fatalf("CreateAccount: %v", err) + } + if err := reg.Create(®istry.Tenant{ + ID: "t-ACTIVE001", + AccountID: "acct_backup", + Email: "client-a@example.com", + DisplayName: "Client A", + State: registry.TenantStateActive, + }); err != nil { + t.Fatalf("Create active tenant: %v", err) + } + if err := reg.Create(®istry.Tenant{ + ID: "t-DELETED001", + AccountID: "acct_backup", + Email: "client-old@example.com", + DisplayName: "Deleted Client", + State: registry.TenantStateDeleted, + }); err != nil { + t.Fatalf("Create deleted tenant: %v", err) + } + return reg +} + +func readProviderMSPBackupEntries(t *testing.T, archivePath string) map[string][]byte { + t.Helper() + file, err := os.Open(archivePath) + if err != nil { + t.Fatalf("open archive: %v", err) + } + defer file.Close() + gz, err := gzip.NewReader(file) + if err != nil { + t.Fatalf("open gzip: %v", err) + } + defer gz.Close() + entries := map[string][]byte{} + tr := tar.NewReader(gz) + for { + header, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("read tar: %v", err) + } + name := strings.TrimSuffix(header.Name, "/") + if header.Typeflag == tar.TypeReg { + content, err := io.ReadAll(tr) + if err != nil { + t.Fatalf("read %s: %v", header.Name, err) + } + entries[name] = content + } else { + entries[name] = nil + } + } + return entries +} + +func assertProviderMSPBackupRegistrySnapshotCount(t *testing.T, dbBytes []byte, want int) { + t.Helper() + dbPath := filepath.Join(t.TempDir(), "tenants.db") + if err := os.WriteFile(dbPath, dbBytes, 0o600); err != nil { + t.Fatalf("write snapshot db: %v", err) + } + db, err := sql.Open("sqlite", dbPath) + if err != nil { + t.Fatalf("open snapshot db: %v", err) + } + defer db.Close() + var got int + if err := db.QueryRow(`SELECT COUNT(*) FROM tenants`).Scan(&got); err != nil { + t.Fatalf("query snapshot db: %v", err) + } + if got != want { + t.Fatalf("snapshot tenant count = %d, want %d", got, want) + } +} + +func sortedEntryNames(entries map[string][]byte) []string { + names := make([]string, 0, len(entries)) + for name := range entries { + names = append(names, name) + } + sort.Strings(names) + return names +} + +func providerMSPBackupTestContainsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 9ba8e4d87..9581ad281 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -49,6 +49,8 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "docker compose run --rm control-plane provider-msp bootstrap", "docker compose run --rm control-plane provider-msp preflight", "docker compose run --rm control-plane provider-msp status", + "docker compose run --rm control-plane provider-msp backup create", + "docker compose run --rm control-plane provider-msp backup verify", "docker compose run --rm control-plane provider-msp proof", "--account-name", "--owner-email", diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 08cfef40c..40570daca 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -3405,10 +3405,12 @@ class SubsystemLookupTest(unittest.TestCase): self.assertEqual( match["verification_requirement"]["exact_files"], [ + "cmd/pulse-control-plane/provider_msp_backup_test.go", "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", + "internal/cloudcp/provider_msp_backup_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go", ],