mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Add provider MSP status command
This commit is contained in:
parent
c9c415d7a6
commit
f128c0b8cd
11 changed files with 544 additions and 23 deletions
|
|
@ -15,6 +15,7 @@ func newProviderMSPCmd() *cobra.Command {
|
|||
cmd.AddCommand(newProviderMSPBootstrapCmd())
|
||||
cmd.AddCommand(newProviderMSPPreflightCmd())
|
||||
cmd.AddCommand(newProviderMSPProofCmd())
|
||||
cmd.AddCommand(newProviderMSPStatusCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
|
|
|||
269
cmd/pulse-control-plane/provider_msp_status.go
Normal file
269
cmd/pulse-control-plane/provider_msp_status.go
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type providerMSPStatusOptions struct {
|
||||
AllowEnvPlan bool
|
||||
PullImage bool
|
||||
}
|
||||
|
||||
type providerMSPStatusReport struct {
|
||||
OK bool
|
||||
Environment string
|
||||
ControlMode string
|
||||
BaseURL string
|
||||
PlanVersion string
|
||||
PlanSource string
|
||||
LicenseID string
|
||||
LicenseEmail string
|
||||
WorkspaceLimit int
|
||||
RegistryReady bool
|
||||
TotalTenants int
|
||||
HealthyTenants int
|
||||
UnhealthyTenants int
|
||||
FailedTenants int
|
||||
ProvisioningTenants int
|
||||
StuckProvisioningTimeout time.Duration
|
||||
StuckProvisioningTenants []string
|
||||
CountsByState map[registry.TenantState]int
|
||||
Preflight *providerMSPPreflightReport
|
||||
Failures []string
|
||||
}
|
||||
|
||||
type providerMSPStatusDependencies struct {
|
||||
OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error)
|
||||
RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func newProviderMSPStatusCmd() *cobra.Command {
|
||||
opts := providerMSPStatusOptions{}
|
||||
cmd := &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Report provider-hosted MSP operational readiness",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := cloudcp.LoadConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load control plane config: %w", err)
|
||||
}
|
||||
report, err := runProviderMSPStatus(cmd.Context(), cfg, opts)
|
||||
printProviderMSPStatusReport(report)
|
||||
return err
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development")
|
||||
cmd.Flags().BoolVar(&opts.PullImage, "pull-image", false, "Pull the tenant runtime image during status instead of inspecting only")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runProviderMSPStatus(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions) (*providerMSPStatusReport, error) {
|
||||
return runProviderMSPStatusWithDependencies(ctx, cfg, opts, providerMSPStatusDependencies{})
|
||||
}
|
||||
|
||||
func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions, deps providerMSPStatusDependencies) (*providerMSPStatusReport, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("control plane config is required")
|
||||
}
|
||||
deps = normalizeProviderMSPStatusDependencies(deps)
|
||||
|
||||
workspaceLimit, _ := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
|
||||
report := &providerMSPStatusReport{
|
||||
OK: true,
|
||||
Environment: strings.TrimSpace(cfg.Environment),
|
||||
ControlMode: string(cfg.ControlPlaneMode),
|
||||
BaseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion),
|
||||
PlanSource: strings.TrimSpace(cfg.ProviderMSPPlanSource),
|
||||
LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID),
|
||||
LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
|
||||
WorkspaceLimit: workspaceLimit,
|
||||
CountsByState: map[registry.TenantState]int{},
|
||||
}
|
||||
addFailure := func(format string, args ...any) {
|
||||
report.OK = false
|
||||
report.Failures = append(report.Failures, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
preflight, preflightErr := deps.RunPreflight(ctx, cfg, providerMSPPreflightOptions{
|
||||
AllowEnvPlan: opts.AllowEnvPlan,
|
||||
SkipImagePull: !opts.PullImage,
|
||||
})
|
||||
report.Preflight = preflight
|
||||
if preflightErr != nil {
|
||||
if preflight != nil && len(preflight.Failures) > 0 {
|
||||
for _, failure := range preflight.Failures {
|
||||
addFailure("preflight: %s", failure)
|
||||
}
|
||||
} else {
|
||||
addFailure("preflight: %v", preflightErr)
|
||||
}
|
||||
}
|
||||
if preflight != nil {
|
||||
report.RegistryReady = preflight.RegistryReady
|
||||
if preflight.WorkspaceLimit > 0 {
|
||||
report.WorkspaceLimit = preflight.WorkspaceLimit
|
||||
}
|
||||
if preflight.LicenseID != "" {
|
||||
report.LicenseID = preflight.LicenseID
|
||||
}
|
||||
if preflight.LicenseEmail != "" {
|
||||
report.LicenseEmail = strings.ToLower(preflight.LicenseEmail)
|
||||
}
|
||||
if !preflight.OK {
|
||||
for _, failure := range preflight.Failures {
|
||||
if !containsProviderMSPStatusFailure(report.Failures, "preflight: "+failure) {
|
||||
addFailure("preflight: %s", failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reg, err := deps.OpenRegistry(cfg)
|
||||
if err != nil {
|
||||
addFailure("open tenant registry: %v", err)
|
||||
return report, providerMSPStatusError(report)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
if err := reg.Ping(); err != nil {
|
||||
addFailure("tenant registry ping: %v", err)
|
||||
} else {
|
||||
report.RegistryReady = true
|
||||
}
|
||||
|
||||
counts, err := reg.CountByState()
|
||||
if err != nil {
|
||||
addFailure("tenant state counts: %v", err)
|
||||
} else {
|
||||
report.CountsByState = counts
|
||||
for _, count := range counts {
|
||||
report.TotalTenants += count
|
||||
}
|
||||
report.FailedTenants = counts[registry.TenantStateFailed]
|
||||
report.ProvisioningTenants = counts[registry.TenantStateProvisioning]
|
||||
if report.FailedTenants > 0 {
|
||||
addFailure("failed workspaces: %d", report.FailedTenants)
|
||||
}
|
||||
}
|
||||
|
||||
healthy, unhealthy, err := reg.HealthSummary()
|
||||
if err != nil {
|
||||
addFailure("tenant health summary: %v", err)
|
||||
} else {
|
||||
report.HealthyTenants = healthy
|
||||
report.UnhealthyTenants = unhealthy
|
||||
if unhealthy > 0 {
|
||||
addFailure("unhealthy active workspaces: %d", unhealthy)
|
||||
}
|
||||
}
|
||||
|
||||
stuck, err := cloudcp.InspectStuckProvisioning(reg, deps.Now())
|
||||
if err != nil {
|
||||
addFailure("stuck provisioning inspection: %v", err)
|
||||
} else if stuck != nil {
|
||||
report.StuckProvisioningTimeout = stuck.Timeout
|
||||
report.StuckProvisioningTenants = stuck.TenantIDs
|
||||
if stuck.Count > 0 {
|
||||
addFailure("stuck provisioning workspaces: %s", strings.Join(stuck.TenantIDs, ","))
|
||||
}
|
||||
}
|
||||
|
||||
return report, providerMSPStatusError(report)
|
||||
}
|
||||
|
||||
func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) providerMSPStatusDependencies {
|
||||
if deps.OpenRegistry == nil {
|
||||
deps.OpenRegistry = func(cfg *cloudcp.CPConfig) (*registry.TenantRegistry, error) {
|
||||
return registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
}
|
||||
}
|
||||
if deps.RunPreflight == nil {
|
||||
deps.RunPreflight = runProviderMSPPreflight
|
||||
}
|
||||
if deps.Now == nil {
|
||||
deps.Now = func() time.Time { return time.Now().UTC() }
|
||||
}
|
||||
return deps
|
||||
}
|
||||
|
||||
func providerMSPStatusError(report *providerMSPStatusReport) error {
|
||||
if report == nil || report.OK {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("provider MSP status failed: %s", strings.Join(report.Failures, "; "))
|
||||
}
|
||||
|
||||
func containsProviderMSPStatusFailure(failures []string, want string) bool {
|
||||
for _, failure := range failures {
|
||||
if failure == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printProviderMSPStatusReport(report *providerMSPStatusReport) {
|
||||
if report == nil {
|
||||
fmt.Println("provider_msp_status_ok=false")
|
||||
return
|
||||
}
|
||||
fmt.Printf("provider_msp_status_ok=%t\n", report.OK)
|
||||
fmt.Printf("environment=%s\n", report.Environment)
|
||||
fmt.Printf("control_plane_mode=%s\n", report.ControlMode)
|
||||
fmt.Printf("base_url=%s\n", report.BaseURL)
|
||||
fmt.Printf("plan_version=%s\n", report.PlanVersion)
|
||||
fmt.Printf("plan_source=%s\n", report.PlanSource)
|
||||
fmt.Printf("license_id=%s\n", report.LicenseID)
|
||||
fmt.Printf("license_email=%s\n", report.LicenseEmail)
|
||||
fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit)
|
||||
fmt.Printf("registry_ready=%t\n", report.RegistryReady)
|
||||
fmt.Printf("total_tenants=%d\n", report.TotalTenants)
|
||||
fmt.Printf("healthy_tenants=%d\n", report.HealthyTenants)
|
||||
fmt.Printf("unhealthy_tenants=%d\n", report.UnhealthyTenants)
|
||||
fmt.Printf("failed_tenants=%d\n", report.FailedTenants)
|
||||
fmt.Printf("provisioning_tenants=%d\n", report.ProvisioningTenants)
|
||||
fmt.Printf("stuck_provisioning_timeout=%s\n", report.StuckProvisioningTimeout)
|
||||
fmt.Printf("stuck_provisioning_count=%d\n", len(report.StuckProvisioningTenants))
|
||||
for _, tenantID := range report.StuckProvisioningTenants {
|
||||
fmt.Printf("stuck_provisioning_tenant=%s\n", tenantID)
|
||||
}
|
||||
for _, state := range sortedProviderMSPStatusStates(report.CountsByState) {
|
||||
fmt.Printf("tenant_state=%s count=%d\n", state, report.CountsByState[state])
|
||||
}
|
||||
if report.Preflight != nil && report.Preflight.Docker != nil {
|
||||
fmt.Printf("docker_reachable=%t\n", report.Preflight.Docker.DockerReachable)
|
||||
fmt.Printf("docker_network_ok=%t\n", report.Preflight.Docker.NetworkOK)
|
||||
fmt.Printf("tenant_runtime_image=%s\n", report.Preflight.Docker.ImageRef)
|
||||
fmt.Printf("tenant_runtime_image_available=%t\n", report.Preflight.Docker.ImageAvailable)
|
||||
fmt.Printf("tenant_runtime_image_pulled=%t\n", report.Preflight.Docker.ImagePulled)
|
||||
}
|
||||
if report.Preflight != nil && report.Preflight.Storage != nil {
|
||||
fmt.Printf("storage_guardrails_enabled=%t\n", report.Preflight.Storage.Enabled)
|
||||
fmt.Printf("storage_guardrails_ok=%t\n", report.Preflight.Storage.OK)
|
||||
}
|
||||
for _, failure := range report.Failures {
|
||||
fmt.Printf("failure=%s\n", failure)
|
||||
}
|
||||
}
|
||||
|
||||
func sortedProviderMSPStatusStates(counts map[registry.TenantState]int) []registry.TenantState {
|
||||
states := make([]registry.TenantState, 0, len(counts))
|
||||
for state := range counts {
|
||||
states = append(states, state)
|
||||
}
|
||||
sort.Slice(states, func(i, j int) bool {
|
||||
return string(states[i]) < string(states[j])
|
||||
})
|
||||
return states
|
||||
}
|
||||
142
cmd/pulse-control-plane/provider_msp_status_test.go
Normal file
142
cmd/pulse-control-plane/provider_msp_status_test.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
)
|
||||
|
||||
func TestProviderMSPCommandExposesStatus(t *testing.T) {
|
||||
cmd := newProviderMSPCmd()
|
||||
for _, child := range cmd.Commands() {
|
||||
if child.Name() == "status" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("provider-msp status command is not registered")
|
||||
}
|
||||
|
||||
func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
|
||||
cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile)
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-HEALTHY",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: true,
|
||||
})
|
||||
|
||||
var gotPreflight providerMSPPreflightOptions
|
||||
report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{
|
||||
RunPreflight: func(_ context.Context, _ *cloudcp.CPConfig, opts providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
gotPreflight = opts
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProviderMSPStatusWithDependencies: %v", err)
|
||||
}
|
||||
if !report.OK {
|
||||
t.Fatalf("report.OK = false, failures = %v", report.Failures)
|
||||
}
|
||||
if !gotPreflight.SkipImagePull {
|
||||
t.Fatal("status preflight should inspect the tenant runtime image by default")
|
||||
}
|
||||
if report.TotalTenants != 1 || report.HealthyTenants != 1 || report.UnhealthyTenants != 0 {
|
||||
t.Fatalf("tenant health summary = total %d healthy %d unhealthy %d", report.TotalTenants, report.HealthyTenants, report.UnhealthyTenants)
|
||||
}
|
||||
if report.WorkspaceLimit != 15 || report.PlanSource != cloudcp.ProviderMSPPlanSourceLicenseFile {
|
||||
t.Fatalf("plan status = %q limit=%d", report.PlanSource, report.WorkspaceLimit)
|
||||
}
|
||||
if report.LicenseID != "lic_provider_msp_test" || report.LicenseEmail != "provider@example.com" {
|
||||
t.Fatalf("license status = %q %q", report.LicenseID, report.LicenseEmail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) {
|
||||
cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile)
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-UNHEALTHY",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: false,
|
||||
})
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-FAILED",
|
||||
State: registry.TenantStateFailed,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
})
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-STUCK",
|
||||
State: registry.TenantStateProvisioning,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
})
|
||||
|
||||
report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{
|
||||
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected provider MSP status to fail for degraded workspaces")
|
||||
}
|
||||
if report == nil || report.OK {
|
||||
t.Fatalf("report.OK = %v, want false", report != nil && report.OK)
|
||||
}
|
||||
if report.FailedTenants != 1 || report.UnhealthyTenants != 1 || len(report.StuckProvisioningTenants) != 1 {
|
||||
t.Fatalf("degraded summary = failed %d unhealthy %d stuck %#v", report.FailedTenants, report.UnhealthyTenants, report.StuckProvisioningTenants)
|
||||
}
|
||||
failures := strings.Join(report.Failures, "; ")
|
||||
for _, want := range []string{"failed workspaces: 1", "unhealthy active workspaces: 1", "stuck provisioning workspaces: t-STUCK"} {
|
||||
if !strings.Contains(failures, want) {
|
||||
t.Fatalf("failures = %q, want %q", failures, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
|
||||
return &providerMSPPreflightReport{
|
||||
OK: true,
|
||||
Environment: "production",
|
||||
ControlMode: string(cloudcp.ControlPlaneModeProviderHostedMSP),
|
||||
BaseURL: "https://msp.example.com",
|
||||
PlanVersion: "msp_growth",
|
||||
PlanSource: cloudcp.ProviderMSPPlanSourceLicenseFile,
|
||||
LicenseID: "lic_provider_msp_test",
|
||||
LicenseEmail: "provider@example.com",
|
||||
WorkspaceLimit: 15,
|
||||
RegistryReady: true,
|
||||
Docker: &cpDocker.RuntimePrerequisiteReport{
|
||||
OK: true,
|
||||
DockerReachable: true,
|
||||
NetworkName: "pulse-provider-msp",
|
||||
NetworkOK: true,
|
||||
ImageRef: "pulse:test",
|
||||
ImageAvailable: true,
|
||||
},
|
||||
Storage: &cloudcp.StorageGuardrailReport{
|
||||
Enabled: true,
|
||||
OK: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func createProviderMSPStatusTenant(t *testing.T, cfg *cloudcp.CPConfig, tenant *registry.Tenant) {
|
||||
t.Helper()
|
||||
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewTenantRegistry: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
if err := reg.Create(tenant); err != nil {
|
||||
t.Fatalf("Create(%s): %v", tenant.ID, err)
|
||||
}
|
||||
}
|
||||
|
|
@ -40,6 +40,9 @@ PULSE_EMAIL_REPLY_TO=support@example.com
|
|||
# Install readiness check:
|
||||
# docker compose run --rm control-plane provider-msp preflight
|
||||
#
|
||||
# Non-mutating operational status:
|
||||
# docker compose run --rm control-plane provider-msp status
|
||||
#
|
||||
# End-to-end provider proof. This creates two proof client workspaces, generates
|
||||
# client-bound agent install tokens, verifies tenant-local agent report ingest,
|
||||
# and exercises the portal handoff path. Keep --cleanup once you have captured
|
||||
|
|
|
|||
|
|
@ -25,6 +25,10 @@ contract. It must carry the resolved MSP plan version, plan source, workspace
|
|||
limit, and validated provider MSP license id/email so installability proof can
|
||||
distinguish signed license-backed activation from local environment fallback
|
||||
configuration.
|
||||
Provider-hosted MSP status uses the same tenant registry, health summary, and
|
||||
stuck-provisioning threshold as the control-plane cleanup loop, so operator
|
||||
readiness reports and automated failure handling cannot drift into separate
|
||||
definitions of a failed provider workspace.
|
||||
|
||||
## Canonical Files
|
||||
|
||||
|
|
|
|||
|
|
@ -29,20 +29,21 @@ surfaces.
|
|||
6. `cmd/pulse-control-plane/provider_msp.go`
|
||||
7. `cmd/pulse-control-plane/provider_msp_preflight.go`
|
||||
8. `cmd/pulse-control-plane/provider_msp_proof.go`
|
||||
9. `internal/cloudcp/docker/manager.go`
|
||||
10. `internal/cloudcp/docker/labels.go`
|
||||
11. `internal/cloudcp/tenant_runtime_rollout.go`
|
||||
12. `.github/workflows/create-release.yml`
|
||||
13. `.github/workflows/deploy-demo-server.yml`
|
||||
14. `.github/workflows/helm-pages.yml`
|
||||
15. `.github/workflows/promote-floating-tags.yml`
|
||||
16. `.github/workflows/publish-docker.yml`
|
||||
17. `.github/workflows/publish-helm-chart.yml`
|
||||
18. `.github/workflows/release-dry-run.yml`
|
||||
19. `.github/workflows/update-demo-server.yml`
|
||||
20. `.github/workflows/validate-release-assets.yml`
|
||||
21. `.github/workflows/install-sh-smoke.yml`
|
||||
22. `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml`
|
||||
9. `cmd/pulse-control-plane/provider_msp_status.go`
|
||||
10. `internal/cloudcp/docker/manager.go`
|
||||
11. `internal/cloudcp/docker/labels.go`
|
||||
12. `internal/cloudcp/tenant_runtime_rollout.go`
|
||||
13. `.github/workflows/create-release.yml`
|
||||
14. `.github/workflows/deploy-demo-server.yml`
|
||||
15. `.github/workflows/helm-pages.yml`
|
||||
16. `.github/workflows/promote-floating-tags.yml`
|
||||
17. `.github/workflows/publish-docker.yml`
|
||||
18. `.github/workflows/publish-helm-chart.yml`
|
||||
19. `.github/workflows/release-dry-run.yml`
|
||||
20. `.github/workflows/update-demo-server.yml`
|
||||
21. `.github/workflows/validate-release-assets.yml`
|
||||
22. `.github/workflows/install-sh-smoke.yml`
|
||||
23. `.github/ISSUE_TEMPLATE/v6_rc_feedback.yml`
|
||||
23. `docs/RELEASE_NOTES.md`
|
||||
24. `docs/releases/`
|
||||
25. `docs/UPGRADE_v6.md`
|
||||
|
|
@ -130,6 +131,11 @@ surfaces.
|
|||
treated as proven. The proof is license-backed by default: `license_file`
|
||||
must be the resolved provider MSP plan source unless the operator explicitly
|
||||
opts into the local-development `--allow-env-plan` escape hatch.
|
||||
`pulse-control-plane provider-msp status` is the non-mutating operational
|
||||
companion to that proof: it must report registry readiness, tenant
|
||||
state/health counts, stuck provisioning workspaces, Docker runtime
|
||||
prerequisites, storage guardrails, and the same license-backed plan identity
|
||||
without pulling tenant images unless the operator asks for it.
|
||||
5. `internal/cloudcp/tenant_runtime_rollout.go` shared with `cloud-paid`: hosted tenant runtime rollout is both a Pulse Cloud runtime contract boundary and a deployment-installability release-rollout boundary.
|
||||
6. `scripts/install.ps1` shared with `agent-lifecycle`: the Windows installer is both a deployment installability entry point and a canonical agent lifecycle runtime continuity boundary.
|
||||
It must expose a non-mutating preflight for the exact Windows agent
|
||||
|
|
|
|||
|
|
@ -2626,6 +2626,7 @@
|
|||
"cmd/pulse-control-plane/provider_msp.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status.go",
|
||||
"docker-compose.yml",
|
||||
"Dockerfile",
|
||||
"docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md",
|
||||
|
|
@ -2894,6 +2895,7 @@
|
|||
"cmd/pulse-control-plane/provider_msp.go",
|
||||
"cmd/pulse-control-plane/provider_msp_preflight.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status.go",
|
||||
"internal/cloudcp/docker/labels.go",
|
||||
"internal/cloudcp/docker/manager.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout.go"
|
||||
|
|
@ -2903,6 +2905,7 @@
|
|||
"exact_files": [
|
||||
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status_test.go",
|
||||
"internal/cloudcp/docker/manager_test.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout_test.go",
|
||||
"scripts/installtests/provider_msp_deploy_test.go"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,39 @@ const (
|
|||
provisioningTimeout = 15 * time.Minute
|
||||
)
|
||||
|
||||
// StuckProvisioningReport describes tenants that have exceeded the
|
||||
// provider-control-plane provisioning grace period.
|
||||
type StuckProvisioningReport struct {
|
||||
Timeout time.Duration
|
||||
Count int
|
||||
TenantIDs []string
|
||||
}
|
||||
|
||||
// InspectStuckProvisioning returns provisioning tenants that are old enough to
|
||||
// be treated as failed by the cleanup loop.
|
||||
func InspectStuckProvisioning(reg *registry.TenantRegistry, now time.Time) (*StuckProvisioningReport, error) {
|
||||
if reg == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now().UTC()
|
||||
}
|
||||
tenants, err := reg.ListByState(registry.TenantStateProvisioning)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cutoff := now.UTC().Add(-provisioningTimeout)
|
||||
report := &StuckProvisioningReport{Timeout: provisioningTimeout}
|
||||
for _, tenant := range tenants {
|
||||
if tenant == nil || tenant.CreatedAt.After(cutoff) {
|
||||
continue
|
||||
}
|
||||
report.Count++
|
||||
report.TenantIDs = append(report.TenantIDs, tenant.ID)
|
||||
}
|
||||
return report, nil
|
||||
}
|
||||
|
||||
// StuckProvisioningCleanup transitions tenants stuck in provisioning state
|
||||
// for longer than provisioningTimeout to failed state.
|
||||
type StuckProvisioningCleanup struct {
|
||||
|
|
@ -43,24 +76,25 @@ func (s *StuckProvisioningCleanup) Run(ctx context.Context) {
|
|||
}
|
||||
|
||||
func (s *StuckProvisioningCleanup) cleanup(ctx context.Context) {
|
||||
tenants, err := s.registry.ListByState(registry.TenantStateProvisioning)
|
||||
report, err := InspectStuckProvisioning(s.registry, time.Now().UTC())
|
||||
if err != nil {
|
||||
log.Error().Err(err).Msg("Stuck provisioning cleanup: failed to list provisioning tenants")
|
||||
return
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-provisioningTimeout)
|
||||
|
||||
for _, tenant := range tenants {
|
||||
if report == nil || report.Count == 0 {
|
||||
return
|
||||
}
|
||||
for _, tenantID := range report.TenantIDs {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if tenant == nil {
|
||||
tenant, err := s.registry.Get(tenantID)
|
||||
if err != nil {
|
||||
log.Error().Err(err).Str("tenant_id", tenantID).Msg("Stuck provisioning cleanup: failed to reload tenant")
|
||||
continue
|
||||
}
|
||||
|
||||
if tenant.CreatedAt.After(cutoff) {
|
||||
continue // Still within provisioning window
|
||||
if tenant == nil || tenant.State != registry.TenantStateProvisioning {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Warn().
|
||||
|
|
|
|||
57
internal/cloudcp/health_stuck_provisioning_test.go
Normal file
57
internal/cloudcp/health_stuck_provisioning_test.go
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
package cloudcp
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
)
|
||||
|
||||
func TestInspectStuckProvisioningUsesCleanupThreshold(t *testing.T) {
|
||||
reg, err := registry.NewTenantRegistry(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewTenantRegistry: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
for _, tenant := range []*registry.Tenant{
|
||||
{
|
||||
ID: "t-STUCK",
|
||||
State: registry.TenantStateProvisioning,
|
||||
CreatedAt: now.Add(-provisioningTimeout - time.Second),
|
||||
},
|
||||
{
|
||||
ID: "t-RECENT",
|
||||
State: registry.TenantStateProvisioning,
|
||||
CreatedAt: now.Add(-provisioningTimeout + time.Second),
|
||||
},
|
||||
{
|
||||
ID: "t-UNHEALTHY",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: false,
|
||||
},
|
||||
} {
|
||||
if err := reg.Create(tenant); err != nil {
|
||||
t.Fatalf("Create(%s): %v", tenant.ID, err)
|
||||
}
|
||||
}
|
||||
|
||||
report, err := InspectStuckProvisioning(reg, now)
|
||||
if err != nil {
|
||||
t.Fatalf("InspectStuckProvisioning: %v", err)
|
||||
}
|
||||
if report == nil {
|
||||
t.Fatal("report = nil")
|
||||
}
|
||||
if report.Timeout != provisioningTimeout {
|
||||
t.Fatalf("Timeout = %s, want %s", report.Timeout, provisioningTimeout)
|
||||
}
|
||||
if report.Count != 1 {
|
||||
t.Fatalf("Count = %d, want 1", report.Count)
|
||||
}
|
||||
if len(report.TenantIDs) != 1 || report.TenantIDs[0] != "t-STUCK" {
|
||||
t.Fatalf("TenantIDs = %#v, want [t-STUCK]", report.TenantIDs)
|
||||
}
|
||||
}
|
||||
|
|
@ -48,6 +48,7 @@ func TestProviderMSPDeployEnvExampleMatchesBootstrapPath(t *testing.T) {
|
|||
"CP_TRIAL_ACTIVATION_PRIVATE_KEY=",
|
||||
"docker compose run --rm control-plane provider-msp bootstrap",
|
||||
"docker compose run --rm control-plane provider-msp preflight",
|
||||
"docker compose run --rm control-plane provider-msp status",
|
||||
"docker compose run --rm control-plane provider-msp proof",
|
||||
"--account-name",
|
||||
"--owner-email",
|
||||
|
|
|
|||
|
|
@ -3407,6 +3407,7 @@ class SubsystemLookupTest(unittest.TestCase):
|
|||
[
|
||||
"cmd/pulse-control-plane/provider_msp_preflight_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_proof_test.go",
|
||||
"cmd/pulse-control-plane/provider_msp_status_test.go",
|
||||
"internal/cloudcp/docker/manager_test.go",
|
||||
"internal/cloudcp/tenant_runtime_rollout_test.go",
|
||||
"scripts/installtests/provider_msp_deploy_test.go",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue