diff --git a/cmd/pulse-control-plane/provider_msp.go b/cmd/pulse-control-plane/provider_msp.go index 1d0cd67ac..394efdc15 100644 --- a/cmd/pulse-control-plane/provider_msp.go +++ b/cmd/pulse-control-plane/provider_msp.go @@ -15,6 +15,7 @@ func newProviderMSPCmd() *cobra.Command { cmd.AddCommand(newProviderMSPBootstrapCmd()) cmd.AddCommand(newProviderMSPPreflightCmd()) cmd.AddCommand(newProviderMSPProofCmd()) + cmd.AddCommand(newProviderMSPStatusCmd()) return cmd } diff --git a/cmd/pulse-control-plane/provider_msp_status.go b/cmd/pulse-control-plane/provider_msp_status.go new file mode 100644 index 000000000..e54ee6de3 --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_status.go @@ -0,0 +1,269 @@ +package main + +import ( + "context" + "fmt" + "sort" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" + pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing" + "github.com/spf13/cobra" +) + +type providerMSPStatusOptions struct { + AllowEnvPlan bool + PullImage bool +} + +type providerMSPStatusReport struct { + OK bool + Environment string + ControlMode string + BaseURL string + PlanVersion string + PlanSource string + LicenseID string + LicenseEmail string + WorkspaceLimit int + RegistryReady bool + TotalTenants int + HealthyTenants int + UnhealthyTenants int + FailedTenants int + ProvisioningTenants int + StuckProvisioningTimeout time.Duration + StuckProvisioningTenants []string + CountsByState map[registry.TenantState]int + Preflight *providerMSPPreflightReport + Failures []string +} + +type providerMSPStatusDependencies struct { + OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error) + RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) + Now func() time.Time +} + +func newProviderMSPStatusCmd() *cobra.Command { + opts := providerMSPStatusOptions{} + cmd := &cobra.Command{ + Use: "status", + Short: "Report provider-hosted MSP operational readiness", + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := cloudcp.LoadConfig() + if err != nil { + return fmt.Errorf("load control plane config: %w", err) + } + report, err := runProviderMSPStatus(cmd.Context(), cfg, opts) + printProviderMSPStatusReport(report) + return err + }, + } + cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development") + cmd.Flags().BoolVar(&opts.PullImage, "pull-image", false, "Pull the tenant runtime image during status instead of inspecting only") + return cmd +} + +func runProviderMSPStatus(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions) (*providerMSPStatusReport, error) { + return runProviderMSPStatusWithDependencies(ctx, cfg, opts, providerMSPStatusDependencies{}) +} + +func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions, deps providerMSPStatusDependencies) (*providerMSPStatusReport, error) { + if cfg == nil { + return nil, fmt.Errorf("control plane config is required") + } + deps = normalizeProviderMSPStatusDependencies(deps) + + workspaceLimit, _ := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion) + report := &providerMSPStatusReport{ + OK: true, + Environment: strings.TrimSpace(cfg.Environment), + ControlMode: string(cfg.ControlPlaneMode), + BaseURL: strings.TrimSpace(cfg.BaseURL), + PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion), + PlanSource: strings.TrimSpace(cfg.ProviderMSPPlanSource), + LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID), + LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)), + WorkspaceLimit: workspaceLimit, + CountsByState: map[registry.TenantState]int{}, + } + addFailure := func(format string, args ...any) { + report.OK = false + report.Failures = append(report.Failures, fmt.Sprintf(format, args...)) + } + + preflight, preflightErr := deps.RunPreflight(ctx, cfg, providerMSPPreflightOptions{ + AllowEnvPlan: opts.AllowEnvPlan, + SkipImagePull: !opts.PullImage, + }) + report.Preflight = preflight + if preflightErr != nil { + if preflight != nil && len(preflight.Failures) > 0 { + for _, failure := range preflight.Failures { + addFailure("preflight: %s", failure) + } + } else { + addFailure("preflight: %v", preflightErr) + } + } + if preflight != nil { + report.RegistryReady = preflight.RegistryReady + if preflight.WorkspaceLimit > 0 { + report.WorkspaceLimit = preflight.WorkspaceLimit + } + if preflight.LicenseID != "" { + report.LicenseID = preflight.LicenseID + } + if preflight.LicenseEmail != "" { + report.LicenseEmail = strings.ToLower(preflight.LicenseEmail) + } + if !preflight.OK { + for _, failure := range preflight.Failures { + if !containsProviderMSPStatusFailure(report.Failures, "preflight: "+failure) { + addFailure("preflight: %s", failure) + } + } + } + } + + reg, err := deps.OpenRegistry(cfg) + if err != nil { + addFailure("open tenant registry: %v", err) + return report, providerMSPStatusError(report) + } + defer reg.Close() + + if err := reg.Ping(); err != nil { + addFailure("tenant registry ping: %v", err) + } else { + report.RegistryReady = true + } + + counts, err := reg.CountByState() + if err != nil { + addFailure("tenant state counts: %v", err) + } else { + report.CountsByState = counts + for _, count := range counts { + report.TotalTenants += count + } + report.FailedTenants = counts[registry.TenantStateFailed] + report.ProvisioningTenants = counts[registry.TenantStateProvisioning] + if report.FailedTenants > 0 { + addFailure("failed workspaces: %d", report.FailedTenants) + } + } + + healthy, unhealthy, err := reg.HealthSummary() + if err != nil { + addFailure("tenant health summary: %v", err) + } else { + report.HealthyTenants = healthy + report.UnhealthyTenants = unhealthy + if unhealthy > 0 { + addFailure("unhealthy active workspaces: %d", unhealthy) + } + } + + stuck, err := cloudcp.InspectStuckProvisioning(reg, deps.Now()) + if err != nil { + addFailure("stuck provisioning inspection: %v", err) + } else if stuck != nil { + report.StuckProvisioningTimeout = stuck.Timeout + report.StuckProvisioningTenants = stuck.TenantIDs + if stuck.Count > 0 { + addFailure("stuck provisioning workspaces: %s", strings.Join(stuck.TenantIDs, ",")) + } + } + + return report, providerMSPStatusError(report) +} + +func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) providerMSPStatusDependencies { + if deps.OpenRegistry == nil { + deps.OpenRegistry = func(cfg *cloudcp.CPConfig) (*registry.TenantRegistry, error) { + return registry.NewTenantRegistry(cfg.ControlPlaneDir()) + } + } + if deps.RunPreflight == nil { + deps.RunPreflight = runProviderMSPPreflight + } + if deps.Now == nil { + deps.Now = func() time.Time { return time.Now().UTC() } + } + return deps +} + +func providerMSPStatusError(report *providerMSPStatusReport) error { + if report == nil || report.OK { + return nil + } + return fmt.Errorf("provider MSP status failed: %s", strings.Join(report.Failures, "; ")) +} + +func containsProviderMSPStatusFailure(failures []string, want string) bool { + for _, failure := range failures { + if failure == want { + return true + } + } + return false +} + +func printProviderMSPStatusReport(report *providerMSPStatusReport) { + if report == nil { + fmt.Println("provider_msp_status_ok=false") + return + } + fmt.Printf("provider_msp_status_ok=%t\n", report.OK) + fmt.Printf("environment=%s\n", report.Environment) + fmt.Printf("control_plane_mode=%s\n", report.ControlMode) + fmt.Printf("base_url=%s\n", report.BaseURL) + fmt.Printf("plan_version=%s\n", report.PlanVersion) + fmt.Printf("plan_source=%s\n", report.PlanSource) + fmt.Printf("license_id=%s\n", report.LicenseID) + fmt.Printf("license_email=%s\n", report.LicenseEmail) + fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit) + fmt.Printf("registry_ready=%t\n", report.RegistryReady) + fmt.Printf("total_tenants=%d\n", report.TotalTenants) + fmt.Printf("healthy_tenants=%d\n", report.HealthyTenants) + fmt.Printf("unhealthy_tenants=%d\n", report.UnhealthyTenants) + fmt.Printf("failed_tenants=%d\n", report.FailedTenants) + fmt.Printf("provisioning_tenants=%d\n", report.ProvisioningTenants) + fmt.Printf("stuck_provisioning_timeout=%s\n", report.StuckProvisioningTimeout) + fmt.Printf("stuck_provisioning_count=%d\n", len(report.StuckProvisioningTenants)) + for _, tenantID := range report.StuckProvisioningTenants { + fmt.Printf("stuck_provisioning_tenant=%s\n", tenantID) + } + for _, state := range sortedProviderMSPStatusStates(report.CountsByState) { + fmt.Printf("tenant_state=%s count=%d\n", state, report.CountsByState[state]) + } + if report.Preflight != nil && report.Preflight.Docker != nil { + fmt.Printf("docker_reachable=%t\n", report.Preflight.Docker.DockerReachable) + fmt.Printf("docker_network_ok=%t\n", report.Preflight.Docker.NetworkOK) + fmt.Printf("tenant_runtime_image=%s\n", report.Preflight.Docker.ImageRef) + fmt.Printf("tenant_runtime_image_available=%t\n", report.Preflight.Docker.ImageAvailable) + fmt.Printf("tenant_runtime_image_pulled=%t\n", report.Preflight.Docker.ImagePulled) + } + if report.Preflight != nil && report.Preflight.Storage != nil { + fmt.Printf("storage_guardrails_enabled=%t\n", report.Preflight.Storage.Enabled) + fmt.Printf("storage_guardrails_ok=%t\n", report.Preflight.Storage.OK) + } + for _, failure := range report.Failures { + fmt.Printf("failure=%s\n", failure) + } +} + +func sortedProviderMSPStatusStates(counts map[registry.TenantState]int) []registry.TenantState { + states := make([]registry.TenantState, 0, len(counts)) + for state := range counts { + states = append(states, state) + } + sort.Slice(states, func(i, j int) bool { + return string(states[i]) < string(states[j]) + }) + return states +} diff --git a/cmd/pulse-control-plane/provider_msp_status_test.go b/cmd/pulse-control-plane/provider_msp_status_test.go new file mode 100644 index 000000000..74c299b1b --- /dev/null +++ b/cmd/pulse-control-plane/provider_msp_status_test.go @@ -0,0 +1,142 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp" + cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker" + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" +) + +func TestProviderMSPCommandExposesStatus(t *testing.T) { + cmd := newProviderMSPCmd() + for _, child := range cmd.Commands() { + if child.Name() == "status" { + return + } + } + t.Fatal("provider-msp status command is not registered") +} + +func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) { + cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile) + now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) + createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ + ID: "t-HEALTHY", + State: registry.TenantStateActive, + CreatedAt: now.Add(-time.Hour), + HealthCheckOK: true, + }) + + var gotPreflight providerMSPPreflightOptions + report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{ + RunPreflight: func(_ context.Context, _ *cloudcp.CPConfig, opts providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { + gotPreflight = opts + return healthyProviderMSPStatusPreflightReport(), nil + }, + Now: func() time.Time { return now }, + }) + if err != nil { + t.Fatalf("runProviderMSPStatusWithDependencies: %v", err) + } + if !report.OK { + t.Fatalf("report.OK = false, failures = %v", report.Failures) + } + if !gotPreflight.SkipImagePull { + t.Fatal("status preflight should inspect the tenant runtime image by default") + } + if report.TotalTenants != 1 || report.HealthyTenants != 1 || report.UnhealthyTenants != 0 { + t.Fatalf("tenant health summary = total %d healthy %d unhealthy %d", report.TotalTenants, report.HealthyTenants, report.UnhealthyTenants) + } + if report.WorkspaceLimit != 15 || report.PlanSource != cloudcp.ProviderMSPPlanSourceLicenseFile { + t.Fatalf("plan status = %q limit=%d", report.PlanSource, report.WorkspaceLimit) + } + if report.LicenseID != "lic_provider_msp_test" || report.LicenseEmail != "provider@example.com" { + t.Fatalf("license status = %q %q", report.LicenseID, report.LicenseEmail) + } +} + +func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) { + cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile) + now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) + createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ + ID: "t-UNHEALTHY", + State: registry.TenantStateActive, + CreatedAt: now.Add(-time.Hour), + HealthCheckOK: false, + }) + createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ + ID: "t-FAILED", + State: registry.TenantStateFailed, + CreatedAt: now.Add(-time.Hour), + }) + createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{ + ID: "t-STUCK", + State: registry.TenantStateProvisioning, + CreatedAt: now.Add(-time.Hour), + }) + + report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{ + RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) { + return healthyProviderMSPStatusPreflightReport(), nil + }, + Now: func() time.Time { return now }, + }) + if err == nil { + t.Fatal("expected provider MSP status to fail for degraded workspaces") + } + if report == nil || report.OK { + t.Fatalf("report.OK = %v, want false", report != nil && report.OK) + } + if report.FailedTenants != 1 || report.UnhealthyTenants != 1 || len(report.StuckProvisioningTenants) != 1 { + t.Fatalf("degraded summary = failed %d unhealthy %d stuck %#v", report.FailedTenants, report.UnhealthyTenants, report.StuckProvisioningTenants) + } + failures := strings.Join(report.Failures, "; ") + for _, want := range []string{"failed workspaces: 1", "unhealthy active workspaces: 1", "stuck provisioning workspaces: t-STUCK"} { + if !strings.Contains(failures, want) { + t.Fatalf("failures = %q, want %q", failures, want) + } + } +} + +func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport { + return &providerMSPPreflightReport{ + OK: true, + Environment: "production", + ControlMode: string(cloudcp.ControlPlaneModeProviderHostedMSP), + BaseURL: "https://msp.example.com", + PlanVersion: "msp_growth", + PlanSource: cloudcp.ProviderMSPPlanSourceLicenseFile, + LicenseID: "lic_provider_msp_test", + LicenseEmail: "provider@example.com", + WorkspaceLimit: 15, + RegistryReady: true, + Docker: &cpDocker.RuntimePrerequisiteReport{ + OK: true, + DockerReachable: true, + NetworkName: "pulse-provider-msp", + NetworkOK: true, + ImageRef: "pulse:test", + ImageAvailable: true, + }, + Storage: &cloudcp.StorageGuardrailReport{ + Enabled: true, + OK: true, + }, + } +} + +func createProviderMSPStatusTenant(t *testing.T, cfg *cloudcp.CPConfig, tenant *registry.Tenant) { + t.Helper() + reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir()) + if err != nil { + t.Fatalf("NewTenantRegistry: %v", err) + } + defer reg.Close() + if err := reg.Create(tenant); err != nil { + t.Fatalf("Create(%s): %v", tenant.ID, err) + } +} diff --git a/deploy/provider-msp/.env.example b/deploy/provider-msp/.env.example index 76ea703e0..94c993487 100644 --- a/deploy/provider-msp/.env.example +++ b/deploy/provider-msp/.env.example @@ -40,6 +40,9 @@ PULSE_EMAIL_REPLY_TO=support@example.com # Install readiness check: # docker compose run --rm control-plane provider-msp preflight # +# Non-mutating operational status: +# docker compose run --rm control-plane provider-msp status +# # End-to-end provider proof. This creates two proof client workspaces, generates # client-bound agent install tokens, verifies tenant-local agent report ingest, # and exercises the portal handoff path. Keep --cleanup once you have captured diff --git a/docs/release-control/v6/internal/subsystems/cloud-paid.md b/docs/release-control/v6/internal/subsystems/cloud-paid.md index 5a11a6e4b..12b003438 100644 --- a/docs/release-control/v6/internal/subsystems/cloud-paid.md +++ b/docs/release-control/v6/internal/subsystems/cloud-paid.md @@ -25,6 +25,10 @@ contract. It must carry the resolved MSP plan version, plan source, workspace limit, and validated provider MSP license id/email so installability proof can distinguish signed license-backed activation from local environment fallback configuration. +Provider-hosted MSP status uses the same tenant registry, health summary, and +stuck-provisioning threshold as the control-plane cleanup loop, so operator +readiness reports and automated failure handling cannot drift into separate +definitions of a failed provider workspace. ## Canonical Files diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index c74c9d588..a6c56f292 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -29,20 +29,21 @@ surfaces. 6. `cmd/pulse-control-plane/provider_msp.go` 7. `cmd/pulse-control-plane/provider_msp_preflight.go` 8. `cmd/pulse-control-plane/provider_msp_proof.go` -9. `internal/cloudcp/docker/manager.go` -10. `internal/cloudcp/docker/labels.go` -11. `internal/cloudcp/tenant_runtime_rollout.go` -12. `.github/workflows/create-release.yml` -13. `.github/workflows/deploy-demo-server.yml` -14. `.github/workflows/helm-pages.yml` -15. `.github/workflows/promote-floating-tags.yml` -16. `.github/workflows/publish-docker.yml` -17. `.github/workflows/publish-helm-chart.yml` -18. `.github/workflows/release-dry-run.yml` -19. `.github/workflows/update-demo-server.yml` -20. `.github/workflows/validate-release-assets.yml` -21. `.github/workflows/install-sh-smoke.yml` -22. `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml` +9. `cmd/pulse-control-plane/provider_msp_status.go` +10. `internal/cloudcp/docker/manager.go` +11. `internal/cloudcp/docker/labels.go` +12. `internal/cloudcp/tenant_runtime_rollout.go` +13. `.github/workflows/create-release.yml` +14. `.github/workflows/deploy-demo-server.yml` +15. `.github/workflows/helm-pages.yml` +16. `.github/workflows/promote-floating-tags.yml` +17. `.github/workflows/publish-docker.yml` +18. `.github/workflows/publish-helm-chart.yml` +19. `.github/workflows/release-dry-run.yml` +20. `.github/workflows/update-demo-server.yml` +21. `.github/workflows/validate-release-assets.yml` +22. `.github/workflows/install-sh-smoke.yml` +23. `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml` 23. `docs/RELEASE_NOTES.md` 24. `docs/releases/` 25. `docs/UPGRADE_v6.md` @@ -130,6 +131,11 @@ surfaces. treated as proven. The proof is license-backed by default: `license_file` must be the resolved provider MSP plan source unless the operator explicitly opts into the local-development `--allow-env-plan` escape hatch. + `pulse-control-plane provider-msp status` is the non-mutating operational + companion to that proof: it must report registry readiness, tenant + state/health counts, stuck provisioning workspaces, Docker runtime + prerequisites, storage guardrails, and the same license-backed plan identity + without pulling tenant images unless the operator asks for it. 5. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary. 6. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary. It must expose a non-mutating preflight for the exact Windows agent diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 9650f61dc..a86f0c189 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -2626,6 +2626,7 @@ "cmd/pulse-control-plane/provider_msp.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", + "cmd/pulse-control-plane/provider_msp_status.go", "docker-compose.yml", "Dockerfile", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", @@ -2894,6 +2895,7 @@ "cmd/pulse-control-plane/provider_msp.go", "cmd/pulse-control-plane/provider_msp_preflight.go", "cmd/pulse-control-plane/provider_msp_proof.go", + "cmd/pulse-control-plane/provider_msp_status.go", "internal/cloudcp/docker/labels.go", "internal/cloudcp/docker/manager.go", "internal/cloudcp/tenant_runtime_rollout.go" @@ -2903,6 +2905,7 @@ "exact_files": [ "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", + "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go" diff --git a/internal/cloudcp/health_stuck_provisioning.go b/internal/cloudcp/health_stuck_provisioning.go index 422425834..7ec0886b1 100644 --- a/internal/cloudcp/health_stuck_provisioning.go +++ b/internal/cloudcp/health_stuck_provisioning.go @@ -13,6 +13,39 @@ const ( provisioningTimeout = 15 * time.Minute ) +// StuckProvisioningReport describes tenants that have exceeded the +// provider-control-plane provisioning grace period. +type StuckProvisioningReport struct { + Timeout time.Duration + Count int + TenantIDs []string +} + +// InspectStuckProvisioning returns provisioning tenants that are old enough to +// be treated as failed by the cleanup loop. +func InspectStuckProvisioning(reg *registry.TenantRegistry, now time.Time) (*StuckProvisioningReport, error) { + if reg == nil { + return nil, nil + } + if now.IsZero() { + now = time.Now().UTC() + } + tenants, err := reg.ListByState(registry.TenantStateProvisioning) + if err != nil { + return nil, err + } + cutoff := now.UTC().Add(-provisioningTimeout) + report := &StuckProvisioningReport{Timeout: provisioningTimeout} + for _, tenant := range tenants { + if tenant == nil || tenant.CreatedAt.After(cutoff) { + continue + } + report.Count++ + report.TenantIDs = append(report.TenantIDs, tenant.ID) + } + return report, nil +} + // StuckProvisioningCleanup transitions tenants stuck in provisioning state // for longer than provisioningTimeout to failed state. type StuckProvisioningCleanup struct { @@ -43,24 +76,25 @@ func (s *StuckProvisioningCleanup) Run(ctx context.Context) { } func (s *StuckProvisioningCleanup) cleanup(ctx context.Context) { - tenants, err := s.registry.ListByState(registry.TenantStateProvisioning) + report, err := InspectStuckProvisioning(s.registry, time.Now().UTC()) if err != nil { log.Error().Err(err).Msg("Stuck provisioning cleanup: failed to list provisioning tenants") return } - - cutoff := time.Now().UTC().Add(-provisioningTimeout) - - for _, tenant := range tenants { + if report == nil || report.Count == 0 { + return + } + for _, tenantID := range report.TenantIDs { if ctx.Err() != nil { return } - if tenant == nil { + tenant, err := s.registry.Get(tenantID) + if err != nil { + log.Error().Err(err).Str("tenant_id", tenantID).Msg("Stuck provisioning cleanup: failed to reload tenant") continue } - - if tenant.CreatedAt.After(cutoff) { - continue // Still within provisioning window + if tenant == nil || tenant.State != registry.TenantStateProvisioning { + continue } log.Warn(). diff --git a/internal/cloudcp/health_stuck_provisioning_test.go b/internal/cloudcp/health_stuck_provisioning_test.go new file mode 100644 index 000000000..d64888d76 --- /dev/null +++ b/internal/cloudcp/health_stuck_provisioning_test.go @@ -0,0 +1,57 @@ +package cloudcp + +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry" +) + +func TestInspectStuckProvisioningUsesCleanupThreshold(t *testing.T) { + reg, err := registry.NewTenantRegistry(t.TempDir()) + if err != nil { + t.Fatalf("NewTenantRegistry: %v", err) + } + defer reg.Close() + + now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC) + for _, tenant := range []*registry.Tenant{ + { + ID: "t-STUCK", + State: registry.TenantStateProvisioning, + CreatedAt: now.Add(-provisioningTimeout - time.Second), + }, + { + ID: "t-RECENT", + State: registry.TenantStateProvisioning, + CreatedAt: now.Add(-provisioningTimeout + time.Second), + }, + { + ID: "t-UNHEALTHY", + State: registry.TenantStateActive, + CreatedAt: now.Add(-time.Hour), + HealthCheckOK: false, + }, + } { + if err := reg.Create(tenant); err != nil { + t.Fatalf("Create(%s): %v", tenant.ID, err) + } + } + + report, err := InspectStuckProvisioning(reg, now) + if err != nil { + t.Fatalf("InspectStuckProvisioning: %v", err) + } + if report == nil { + t.Fatal("report = nil") + } + if report.Timeout != provisioningTimeout { + t.Fatalf("Timeout = %s, want %s", report.Timeout, provisioningTimeout) + } + if report.Count != 1 { + t.Fatalf("Count = %d, want 1", report.Count) + } + if len(report.TenantIDs) != 1 || report.TenantIDs[0] != "t-STUCK" { + t.Fatalf("TenantIDs = %#v, want [t-STUCK]", report.TenantIDs) + } +} diff --git a/scripts/installtests/provider_msp_deploy_test.go b/scripts/installtests/provider_msp_deploy_test.go index 0294540ca..9ba8e4d87 100644 --- a/scripts/installtests/provider_msp_deploy_test.go +++ b/scripts/installtests/provider_msp_deploy_test.go @@ -48,6 +48,7 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) { "CP_TRIAL_ACTIVATION_PRIVATE_KEY=", "docker compose run --rm control-plane provider-msp bootstrap", "docker compose run --rm control-plane provider-msp preflight", + "docker compose run --rm control-plane provider-msp status", "docker compose run --rm control-plane provider-msp proof", "--account-name", "--owner-email", diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 106f6cd4b..08cfef40c 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -3407,6 +3407,7 @@ class SubsystemLookupTest(unittest.TestCase): [ "cmd/pulse-control-plane/provider_msp_preflight_test.go", "cmd/pulse-control-plane/provider_msp_proof_test.go", + "cmd/pulse-control-plane/provider_msp_status_test.go", "internal/cloudcp/docker/manager_test.go", "internal/cloudcp/tenant_runtime_rollout_test.go", "scripts/installtests/provider_msp_deploy_test.go",