mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Add provider MSP status command
This commit is contained in:
rcourtman
parent
c9c415d7a6
commit
f128c0b8cd
11 changed files with 544 additions and 23 deletions
|
|
@ -15,6 +15,7 @@ func newProviderMSPCmd() *cobra.Command {
|
|||
cmd.AddCommand(newProviderMSPBootstrapCmd())
|
||||
cmd.AddCommand(newProviderMSPPreflightCmd())
|
||||
cmd.AddCommand(newProviderMSPProofCmd())
|
||||
cmd.AddCommand(newProviderMSPStatusCmd())
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
|
|
|||
269
cmd/pulse-control-plane/provider_msp_status.go
Normal file
269
cmd/pulse-control-plane/provider_msp_status.go
Normal file
|
|
@ -0,0 +1,269 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
pkglicensing "github.com/rcourtman/pulse-go-rewrite/pkg/licensing"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
type providerMSPStatusOptions struct {
|
||||
AllowEnvPlan bool
|
||||
PullImage bool
|
||||
}
|
||||
|
||||
type providerMSPStatusReport struct {
|
||||
OK bool
|
||||
Environment string
|
||||
ControlMode string
|
||||
BaseURL string
|
||||
PlanVersion string
|
||||
PlanSource string
|
||||
LicenseID string
|
||||
LicenseEmail string
|
||||
WorkspaceLimit int
|
||||
RegistryReady bool
|
||||
TotalTenants int
|
||||
HealthyTenants int
|
||||
UnhealthyTenants int
|
||||
FailedTenants int
|
||||
ProvisioningTenants int
|
||||
StuckProvisioningTimeout time.Duration
|
||||
StuckProvisioningTenants []string
|
||||
CountsByState map[registry.TenantState]int
|
||||
Preflight *providerMSPPreflightReport
|
||||
Failures []string
|
||||
}
|
||||
|
||||
type providerMSPStatusDependencies struct {
|
||||
OpenRegistry func(*cloudcp.CPConfig) (*registry.TenantRegistry, error)
|
||||
RunPreflight func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error)
|
||||
Now func() time.Time
|
||||
}
|
||||
|
||||
func newProviderMSPStatusCmd() *cobra.Command {
|
||||
opts := providerMSPStatusOptions{}
|
||||
cmd := &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "Report provider-hosted MSP operational readiness",
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := cloudcp.LoadConfig()
|
||||
if err != nil {
|
||||
return fmt.Errorf("load control plane config: %w", err)
|
||||
}
|
||||
report, err := runProviderMSPStatus(cmd.Context(), cfg, opts)
|
||||
printProviderMSPStatusReport(report)
|
||||
return err
|
||||
},
|
||||
}
|
||||
cmd.Flags().BoolVar(&opts.AllowEnvPlan, "allow-env-plan", false, "Allow CP_PROVIDER_MSP_PLAN_VERSION fallback instead of a signed provider MSP license file for local development")
|
||||
cmd.Flags().BoolVar(&opts.PullImage, "pull-image", false, "Pull the tenant runtime image during status instead of inspecting only")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func runProviderMSPStatus(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions) (*providerMSPStatusReport, error) {
|
||||
return runProviderMSPStatusWithDependencies(ctx, cfg, opts, providerMSPStatusDependencies{})
|
||||
}
|
||||
|
||||
func runProviderMSPStatusWithDependencies(ctx context.Context, cfg *cloudcp.CPConfig, opts providerMSPStatusOptions, deps providerMSPStatusDependencies) (*providerMSPStatusReport, error) {
|
||||
if cfg == nil {
|
||||
return nil, fmt.Errorf("control plane config is required")
|
||||
}
|
||||
deps = normalizeProviderMSPStatusDependencies(deps)
|
||||
|
||||
workspaceLimit, _ := pkglicensing.WorkspaceLimitForPlan(cfg.ProviderMSPPlanVersion)
|
||||
report := &providerMSPStatusReport{
|
||||
OK: true,
|
||||
Environment: strings.TrimSpace(cfg.Environment),
|
||||
ControlMode: string(cfg.ControlPlaneMode),
|
||||
BaseURL: strings.TrimSpace(cfg.BaseURL),
|
||||
PlanVersion: strings.TrimSpace(cfg.ProviderMSPPlanVersion),
|
||||
PlanSource: strings.TrimSpace(cfg.ProviderMSPPlanSource),
|
||||
LicenseID: strings.TrimSpace(cfg.ProviderMSPLicenseID),
|
||||
LicenseEmail: strings.ToLower(strings.TrimSpace(cfg.ProviderMSPLicenseEmail)),
|
||||
WorkspaceLimit: workspaceLimit,
|
||||
CountsByState: map[registry.TenantState]int{},
|
||||
}
|
||||
addFailure := func(format string, args ...any) {
|
||||
report.OK = false
|
||||
report.Failures = append(report.Failures, fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
preflight, preflightErr := deps.RunPreflight(ctx, cfg, providerMSPPreflightOptions{
|
||||
AllowEnvPlan: opts.AllowEnvPlan,
|
||||
SkipImagePull: !opts.PullImage,
|
||||
})
|
||||
report.Preflight = preflight
|
||||
if preflightErr != nil {
|
||||
if preflight != nil && len(preflight.Failures) > 0 {
|
||||
for _, failure := range preflight.Failures {
|
||||
addFailure("preflight: %s", failure)
|
||||
}
|
||||
} else {
|
||||
addFailure("preflight: %v", preflightErr)
|
||||
}
|
||||
}
|
||||
if preflight != nil {
|
||||
report.RegistryReady = preflight.RegistryReady
|
||||
if preflight.WorkspaceLimit > 0 {
|
||||
report.WorkspaceLimit = preflight.WorkspaceLimit
|
||||
}
|
||||
if preflight.LicenseID != "" {
|
||||
report.LicenseID = preflight.LicenseID
|
||||
}
|
||||
if preflight.LicenseEmail != "" {
|
||||
report.LicenseEmail = strings.ToLower(preflight.LicenseEmail)
|
||||
}
|
||||
if !preflight.OK {
|
||||
for _, failure := range preflight.Failures {
|
||||
if !containsProviderMSPStatusFailure(report.Failures, "preflight: "+failure) {
|
||||
addFailure("preflight: %s", failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
reg, err := deps.OpenRegistry(cfg)
|
||||
if err != nil {
|
||||
addFailure("open tenant registry: %v", err)
|
||||
return report, providerMSPStatusError(report)
|
||||
}
|
||||
defer reg.Close()
|
||||
|
||||
if err := reg.Ping(); err != nil {
|
||||
addFailure("tenant registry ping: %v", err)
|
||||
} else {
|
||||
report.RegistryReady = true
|
||||
}
|
||||
|
||||
counts, err := reg.CountByState()
|
||||
if err != nil {
|
||||
addFailure("tenant state counts: %v", err)
|
||||
} else {
|
||||
report.CountsByState = counts
|
||||
for _, count := range counts {
|
||||
report.TotalTenants += count
|
||||
}
|
||||
report.FailedTenants = counts[registry.TenantStateFailed]
|
||||
report.ProvisioningTenants = counts[registry.TenantStateProvisioning]
|
||||
if report.FailedTenants > 0 {
|
||||
addFailure("failed workspaces: %d", report.FailedTenants)
|
||||
}
|
||||
}
|
||||
|
||||
healthy, unhealthy, err := reg.HealthSummary()
|
||||
if err != nil {
|
||||
addFailure("tenant health summary: %v", err)
|
||||
} else {
|
||||
report.HealthyTenants = healthy
|
||||
report.UnhealthyTenants = unhealthy
|
||||
if unhealthy > 0 {
|
||||
addFailure("unhealthy active workspaces: %d", unhealthy)
|
||||
}
|
||||
}
|
||||
|
||||
stuck, err := cloudcp.InspectStuckProvisioning(reg, deps.Now())
|
||||
if err != nil {
|
||||
addFailure("stuck provisioning inspection: %v", err)
|
||||
} else if stuck != nil {
|
||||
report.StuckProvisioningTimeout = stuck.Timeout
|
||||
report.StuckProvisioningTenants = stuck.TenantIDs
|
||||
if stuck.Count > 0 {
|
||||
addFailure("stuck provisioning workspaces: %s", strings.Join(stuck.TenantIDs, ","))
|
||||
}
|
||||
}
|
||||
|
||||
return report, providerMSPStatusError(report)
|
||||
}
|
||||
|
||||
func normalizeProviderMSPStatusDependencies(deps providerMSPStatusDependencies) providerMSPStatusDependencies {
|
||||
if deps.OpenRegistry == nil {
|
||||
deps.OpenRegistry = func(cfg *cloudcp.CPConfig) (*registry.TenantRegistry, error) {
|
||||
return registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
}
|
||||
}
|
||||
if deps.RunPreflight == nil {
|
||||
deps.RunPreflight = runProviderMSPPreflight
|
||||
}
|
||||
if deps.Now == nil {
|
||||
deps.Now = func() time.Time { return time.Now().UTC() }
|
||||
}
|
||||
return deps
|
||||
}
|
||||
|
||||
func providerMSPStatusError(report *providerMSPStatusReport) error {
|
||||
if report == nil || report.OK {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("provider MSP status failed: %s", strings.Join(report.Failures, "; "))
|
||||
}
|
||||
|
||||
func containsProviderMSPStatusFailure(failures []string, want string) bool {
|
||||
for _, failure := range failures {
|
||||
if failure == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func printProviderMSPStatusReport(report *providerMSPStatusReport) {
|
||||
if report == nil {
|
||||
fmt.Println("provider_msp_status_ok=false")
|
||||
return
|
||||
}
|
||||
fmt.Printf("provider_msp_status_ok=%t\n", report.OK)
|
||||
fmt.Printf("environment=%s\n", report.Environment)
|
||||
fmt.Printf("control_plane_mode=%s\n", report.ControlMode)
|
||||
fmt.Printf("base_url=%s\n", report.BaseURL)
|
||||
fmt.Printf("plan_version=%s\n", report.PlanVersion)
|
||||
fmt.Printf("plan_source=%s\n", report.PlanSource)
|
||||
fmt.Printf("license_id=%s\n", report.LicenseID)
|
||||
fmt.Printf("license_email=%s\n", report.LicenseEmail)
|
||||
fmt.Printf("workspace_limit=%d\n", report.WorkspaceLimit)
|
||||
fmt.Printf("registry_ready=%t\n", report.RegistryReady)
|
||||
fmt.Printf("total_tenants=%d\n", report.TotalTenants)
|
||||
fmt.Printf("healthy_tenants=%d\n", report.HealthyTenants)
|
||||
fmt.Printf("unhealthy_tenants=%d\n", report.UnhealthyTenants)
|
||||
fmt.Printf("failed_tenants=%d\n", report.FailedTenants)
|
||||
fmt.Printf("provisioning_tenants=%d\n", report.ProvisioningTenants)
|
||||
fmt.Printf("stuck_provisioning_timeout=%s\n", report.StuckProvisioningTimeout)
|
||||
fmt.Printf("stuck_provisioning_count=%d\n", len(report.StuckProvisioningTenants))
|
||||
for _, tenantID := range report.StuckProvisioningTenants {
|
||||
fmt.Printf("stuck_provisioning_tenant=%s\n", tenantID)
|
||||
}
|
||||
for _, state := range sortedProviderMSPStatusStates(report.CountsByState) {
|
||||
fmt.Printf("tenant_state=%s count=%d\n", state, report.CountsByState[state])
|
||||
}
|
||||
if report.Preflight != nil && report.Preflight.Docker != nil {
|
||||
fmt.Printf("docker_reachable=%t\n", report.Preflight.Docker.DockerReachable)
|
||||
fmt.Printf("docker_network_ok=%t\n", report.Preflight.Docker.NetworkOK)
|
||||
fmt.Printf("tenant_runtime_image=%s\n", report.Preflight.Docker.ImageRef)
|
||||
fmt.Printf("tenant_runtime_image_available=%t\n", report.Preflight.Docker.ImageAvailable)
|
||||
fmt.Printf("tenant_runtime_image_pulled=%t\n", report.Preflight.Docker.ImagePulled)
|
||||
}
|
||||
if report.Preflight != nil && report.Preflight.Storage != nil {
|
||||
fmt.Printf("storage_guardrails_enabled=%t\n", report.Preflight.Storage.Enabled)
|
||||
fmt.Printf("storage_guardrails_ok=%t\n", report.Preflight.Storage.OK)
|
||||
}
|
||||
for _, failure := range report.Failures {
|
||||
fmt.Printf("failure=%s\n", failure)
|
||||
}
|
||||
}
|
||||
|
||||
func sortedProviderMSPStatusStates(counts map[registry.TenantState]int) []registry.TenantState {
|
||||
states := make([]registry.TenantState, 0, len(counts))
|
||||
for state := range counts {
|
||||
states = append(states, state)
|
||||
}
|
||||
sort.Slice(states, func(i, j int) bool {
|
||||
return string(states[i]) < string(states[j])
|
||||
})
|
||||
return states
|
||||
}
|
||||
142
cmd/pulse-control-plane/provider_msp_status_test.go
Normal file
142
cmd/pulse-control-plane/provider_msp_status_test.go
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp"
|
||||
cpDocker "github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/docker"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/cloudcp/registry"
|
||||
)
|
||||
|
||||
func TestProviderMSPCommandExposesStatus(t *testing.T) {
|
||||
cmd := newProviderMSPCmd()
|
||||
for _, child := range cmd.Commands() {
|
||||
if child.Name() == "status" {
|
||||
return
|
||||
}
|
||||
}
|
||||
t.Fatal("provider-msp status command is not registered")
|
||||
}
|
||||
|
||||
func TestProviderMSPStatusReportsHealthyOperatorSurface(t *testing.T) {
|
||||
cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile)
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-HEALTHY",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: true,
|
||||
})
|
||||
|
||||
var gotPreflight providerMSPPreflightOptions
|
||||
report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{
|
||||
RunPreflight: func(_ context.Context, _ *cloudcp.CPConfig, opts providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
gotPreflight = opts
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("runProviderMSPStatusWithDependencies: %v", err)
|
||||
}
|
||||
if !report.OK {
|
||||
t.Fatalf("report.OK = false, failures = %v", report.Failures)
|
||||
}
|
||||
if !gotPreflight.SkipImagePull {
|
||||
t.Fatal("status preflight should inspect the tenant runtime image by default")
|
||||
}
|
||||
if report.TotalTenants != 1 || report.HealthyTenants != 1 || report.UnhealthyTenants != 0 {
|
||||
t.Fatalf("tenant health summary = total %d healthy %d unhealthy %d", report.TotalTenants, report.HealthyTenants, report.UnhealthyTenants)
|
||||
}
|
||||
if report.WorkspaceLimit != 15 || report.PlanSource != cloudcp.ProviderMSPPlanSourceLicenseFile {
|
||||
t.Fatalf("plan status = %q limit=%d", report.PlanSource, report.WorkspaceLimit)
|
||||
}
|
||||
if report.LicenseID != "lic_provider_msp_test" || report.LicenseEmail != "provider@example.com" {
|
||||
t.Fatalf("license status = %q %q", report.LicenseID, report.LicenseEmail)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProviderMSPStatusFailsOnFailedUnhealthyAndStuckWorkspaces(t *testing.T) {
|
||||
cfg := testProviderMSPPreflightConfig(t, cloudcp.ProviderMSPPlanSourceLicenseFile)
|
||||
now := time.Date(2026, 6, 2, 12, 0, 0, 0, time.UTC)
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-UNHEALTHY",
|
||||
State: registry.TenantStateActive,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
HealthCheckOK: false,
|
||||
})
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-FAILED",
|
||||
State: registry.TenantStateFailed,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
})
|
||||
createProviderMSPStatusTenant(t, cfg, ®istry.Tenant{
|
||||
ID: "t-STUCK",
|
||||
State: registry.TenantStateProvisioning,
|
||||
CreatedAt: now.Add(-time.Hour),
|
||||
})
|
||||
|
||||
report, err := runProviderMSPStatusWithDependencies(context.Background(), cfg, providerMSPStatusOptions{}, providerMSPStatusDependencies{
|
||||
RunPreflight: func(context.Context, *cloudcp.CPConfig, providerMSPPreflightOptions) (*providerMSPPreflightReport, error) {
|
||||
return healthyProviderMSPStatusPreflightReport(), nil
|
||||
},
|
||||
Now: func() time.Time { return now },
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("expected provider MSP status to fail for degraded workspaces")
|
||||
}
|
||||
if report == nil || report.OK {
|
||||
t.Fatalf("report.OK = %v, want false", report != nil && report.OK)
|
||||
}
|
||||
if report.FailedTenants != 1 || report.UnhealthyTenants != 1 || len(report.StuckProvisioningTenants) != 1 {
|
||||
t.Fatalf("degraded summary = failed %d unhealthy %d stuck %#v", report.FailedTenants, report.UnhealthyTenants, report.StuckProvisioningTenants)
|
||||
}
|
||||
failures := strings.Join(report.Failures, "; ")
|
||||
for _, want := range []string{"failed workspaces: 1", "unhealthy active workspaces: 1", "stuck provisioning workspaces: t-STUCK"} {
|
||||
if !strings.Contains(failures, want) {
|
||||
t.Fatalf("failures = %q, want %q", failures, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func healthyProviderMSPStatusPreflightReport() *providerMSPPreflightReport {
|
||||
return &providerMSPPreflightReport{
|
||||
OK: true,
|
||||
Environment: "production",
|
||||
ControlMode: string(cloudcp.ControlPlaneModeProviderHostedMSP),
|
||||
BaseURL: "https://msp.example.com",
|
||||
PlanVersion: "msp_growth",
|
||||
PlanSource: cloudcp.ProviderMSPPlanSourceLicenseFile,
|
||||
LicenseID: "lic_provider_msp_test",
|
||||
LicenseEmail: "provider@example.com",
|
||||
WorkspaceLimit: 15,
|
||||
RegistryReady: true,
|
||||
Docker: &cpDocker.RuntimePrerequisiteReport{
|
||||
OK: true,
|
||||
DockerReachable: true,
|
||||
NetworkName: "pulse-provider-msp",
|
||||
NetworkOK: true,
|
||||
ImageRef: "pulse:test",
|
||||
ImageAvailable: true,
|
||||
},
|
||||
Storage: &cloudcp.StorageGuardrailReport{
|
||||
Enabled: true,
|
||||
OK: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func createProviderMSPStatusTenant(t *testing.T, cfg *cloudcp.CPConfig, tenant *registry.Tenant) {
|
||||
t.Helper()
|
||||
reg, err := registry.NewTenantRegistry(cfg.ControlPlaneDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewTenantRegistry: %v", err)
|
||||
}
|
||||
defer reg.Close()
|
||||
if err := reg.Create(tenant); err != nil {
|
||||
t.Fatalf("Create(%s): %v", tenant.ID, err)
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue