From 9923e4b04f573ee7c846c50857229c75538efcc1 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 8 Jul 2026 09:49:03 +0100 Subject: [PATCH] Pace mock churn to homelab rates and stop phantom demo change spam The public demo wrote ~110K resource_changes rows/day (restart 60K/day, state_transition 45K/day), making the Changes timeline unreadable and keeping unified_resources.db churning. Four generator-level engines, all verified with before/after soaks against scratch mock backends: - Per-tick flap probabilities ran 43,200x/day on the 2s update loop (docker restart p=0.01/tick alone is ~430 restarts/day/container). Churn rates are now expressed as events per day per entity and converted per tick against the configured interval, tuned to a few fleet-wide events per day with dwell times long enough to see. - The pod scheduling reconciler fought the per-tick scenario re-pin and fabricated a fresh random StartTime on every recovery; derived uptime moved backwards, which change emission records as a restart. The reconciler is now idempotent: stable per-pod park/reschedule choices, StartTime never regenerated, and only pods it parked itself (NodeNotReady/ClusterOffline/NodeLost) get recovered, so curated Pending and ImagePullBackOff stories stay put. - Swarm cluster objects were fabricated per host under one shared cluster key, so registry dedupe alternated between the divergent candidates every poll (service renames, status flips, node re-parenting). One leader manager now reports services, tasks, secrets, configs and the node inventory, like a real control plane. - Demo docker host profiles cycled 2 hostnames across 4 online hosts, collapsing canonical identities, and scripted-offline hosts had their sighting refreshed right at the 2 minute staleness threshold, sawtoothing them online/offline. Four distinct host profiles now exist and offline hosts keep a stably stale sighting. Before/after soak with a live client: pre-fix ~110-160 rows/min sustained (restart ~45/min, matching the droplet's 60K/day); post-fix zero rows/min at steady state with the curated degraded stories (CrashLoop payments-worker, ImagePullBackOff, offline hosts) intact. --- internal/mock/demo_scenarios.go | 45 ++++++++- internal/mock/generator.go | 167 +++++++++++++++++++++++++------- 2 files changed, 178 insertions(+), 34 deletions(-) diff --git a/internal/mock/demo_scenarios.go b/internal/mock/demo_scenarios.go index 7b5c43545..b96a5729a 100644 --- a/internal/mock/demo_scenarios.go +++ b/internal/mock/demo_scenarios.go @@ -136,6 +136,28 @@ var demoDockerHostProfiles = []demoDockerHostProfile{ {Name: "grafana-agent", Image: "grafana/agent:v0.42.0", Tags: []string{"monitoring", "agent", "platform"}, Health: "healthy"}, }, }, + { + Hostname: "app-platform-01", + DisplayName: "App Platform 01", + Containers: []demoDockerContainerProfile{ + {Name: "orders-api", Image: "ghcr.io/pulse-demo/orders-api:2026.04", Tags: []string{"api", "production", "customer-facing"}, Health: "healthy"}, + {Name: "notifications-worker", Image: "ghcr.io/pulse-demo/notifications-worker:2026.04", Tags: []string{"queue", "worker", "production"}, Health: "healthy"}, + {Name: "redis-cache", Image: "redis:7.4.1", Tags: []string{"cache", "production", "latency-sensitive"}, Health: "healthy"}, + {Name: "nginx-edge", Image: "nginx:1.27.2", Tags: []string{"web", "ingress", "production"}, Health: "healthy"}, + {Name: "loki", Image: "grafana/loki:3.2.0", Tags: []string{"logging", "platform"}, Health: "healthy"}, + }, + }, + { + Hostname: "data-services-01", + DisplayName: "Data Services 01", + Containers: []demoDockerContainerProfile{ + {Name: "kafka-broker", Image: "confluentinc/cp-kafka:7.7.1", Tags: []string{"streaming", "platform", "internal"}, Health: "healthy"}, + {Name: "clickhouse", Image: "clickhouse/clickhouse-server:24.8", Tags: []string{"database", "analytics", "internal"}, Health: "healthy"}, + {Name: "etl-runner", Image: "ghcr.io/pulse-demo/etl-runner:2026.04", Tags: []string{"batch", "analytics", "worker"}, Health: "healthy"}, + {Name: "minio", Image: "minio/minio:RELEASE.2026-01-20T00-00-00Z", Tags: []string{"storage", "gateway", "internal"}, Health: "healthy"}, + {Name: "schema-registry", Image: "confluentinc/cp-schema-registry:7.7.1", Tags: []string{"streaming", "platform", "internal"}, Health: "healthy"}, + }, + }, } // Per-cluster demo identities. Each cluster carries its own node naming @@ -508,6 +530,7 @@ func applyDemoDockerScenario(state *models.StateSnapshot, now time.Time) { // One docker host index is forced offline so the demo Docker page exposes a // disconnected host with its containers shown as exited rather than running. const offlineDockerIndex = 2 + onlineOrdinal := 0 for i := range state.DockerHosts { host := &state.DockerHosts[i] if i == offlineDockerIndex { @@ -518,9 +541,20 @@ func applyDemoDockerScenario(state *models.StateSnapshot, now time.Time) { ensureMockDockerNativeInventory(host, i, now) continue } - hostProfile := demoDockerHostProfiles[i%len(demoDockerHostProfiles)] + hostProfile := demoDockerHostProfiles[onlineOrdinal%len(demoDockerHostProfiles)] + // Hostnames must stay unique across the fleet: unified resources + // treats hostname as identity, so two hosts sharing a profile name + // collapse into one canonical resource whose status, parent, and + // swarm services then flap on every poll as the two real hosts + // alternate underneath it. + round := onlineOrdinal / len(demoDockerHostProfiles) + onlineOrdinal++ host.Hostname = hostProfile.Hostname host.DisplayName = hostProfile.DisplayName + if round > 0 { + host.Hostname = fmt.Sprintf("%s-%d", hostProfile.Hostname, round+1) + host.DisplayName = fmt.Sprintf("%s (%d)", hostProfile.DisplayName, round+1) + } host.Status = "online" host.Containers = applyDemoDockerContainerProfiles(host.Containers, hostProfile.Containers, now) ensureMockDockerNativeInventory(host, i, now) @@ -918,6 +952,15 @@ func applyDemoHostScenario(state *models.StateSnapshot, now time.Time) { host.Disks[j].Free = host.Disks[j].Total host.Disks[j].Usage = -1 } + // An offline host stopped reporting, so its sighting must stay + // stale. Refreshing it like the online hosts below made the + // agent's LastSeen race the 2-minute source staleness threshold + // and sawtooth the resource between online and offline every + // couple of minutes. + if !now.IsZero() { + host.LastSeen = now.Add(-10 * time.Minute) + } + continue } if now.IsZero() { continue diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 6f5946e7a..fa30324c0 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -61,6 +61,61 @@ const ( hostConnectionPrefix = "host-" ) +// Churn pacing for the mock estate. Restart and state-change events are +// expressed as expected events per day per entity and converted to a +// per-update probability against the configured update interval, so the +// pacing does not depend on how often the mock loop ticks. The default +// 2-second tick runs 43,200 times a day: a raw per-tick probability like +// 0.03 means thousands of events per container per day, which floods +// resource_changes and makes the demo Changes timeline unreadable. These +// rates target what a real homelab produces: a few restarts and a few +// state changes per day across the whole fleet, while per-tick metric +// wiggle keeps the demo feeling alive. +const ( + // One container restart every ~12 days per container; across the + // ~56 running demo containers that is 4-5 restarts a day fleet-wide. + mockDockerRestartsPerDay = 0.08 + // Health-check blips (healthy -> unhealthy / starting). + mockDockerUnhealthyPerDay = 0.05 + mockDockerStartingPerDay = 0.05 + // A stopped container being started again. + mockDockerStoppedStartsPerDay = 0.25 + // An offline docker host coming back. + mockDockerHostRecoveriesPerDay = 0.25 + // Kubernetes background churn per cluster. + mockK8sClusterRecoveriesPerDay = 0.5 + mockK8sNodeReadyFlipsPerDay = 0.3 + mockK8sPodCrashFlipsPerDay = 0.5 + // Standalone host status churn per host. + mockHostDegradedPerDay = 0.1 + mockHostRecoveriesPerDay = 0.25 + mockDiskWearoutStepsPerDay = 0.1 +) + +// Mean dwell times for transient states, converted to a per-update recovery +// probability. Long enough to be seen on the dashboard, short enough that +// the estate does not look permanently broken. +const ( + mockDockerUnhealthyMeanRecovery = 12 * time.Minute + mockDockerStartingMeanRecovery = 90 * time.Second + mockHostDegradedMeanRecovery = 15 * time.Minute +) + +// mockEventsPerDayChance converts an expected events-per-day rate for a +// single entity into a probability for one mock update tick. +func mockEventsPerDayChance(eventsPerDay float64) float64 { + return eventsPerDay * currentMockUpdateStepSeconds() / (24 * 60 * 60) +} + +// mockRecoveryChance converts a mean time-to-recovery into a probability +// for one mock update tick. +func mockRecoveryChance(meanRecovery time.Duration) float64 { + if meanRecovery <= 0 { + return 1 + } + return currentMockUpdateStepSeconds() / meanRecovery.Seconds() +} + // Default fixture sizes target a mature small-to-mid homelab / SMB // environment so platform pages exercise table density, sorting, // grouping, drawer behavior, and responsive layout out of the box: @@ -1507,7 +1562,7 @@ func initializeMockKubernetesClusterUsage(cluster *models.KubernetesCluster, now return } - reconcileMockKubernetesPodScheduling(cluster, randomize) + reconcileMockKubernetesPodScheduling(cluster, now, randomize) nodeByName := make(map[string]*models.KubernetesNode, len(cluster.Nodes)) for i := range cluster.Nodes { @@ -1529,10 +1584,22 @@ func initializeMockKubernetesClusterUsage(cluster *models.KubernetesCluster, now recomputeMockKubernetesNodeUsage(cluster, now) } -func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, randomize bool) { +// reconcileMockKubernetesPodScheduling keeps pod placement coherent with +// node readiness. It runs on every mock update tick, both mid-tick (with +// randomize) and again after the demo scenario re-pins pod profiles, so it +// must be idempotent: the same inputs must produce the same pod state, and +// it must never fabricate a fresh StartTime for a pod that already has one. +// A regenerated StartTime moves the pod's derived uptime backwards, which +// unified resources records as a restart — an earlier version of this +// function re-anchored StartTime with rand on every recovery and flooded +// resource_changes with tens of thousands of phantom pod restarts per day. +func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, now time.Time, randomize bool) { if cluster == nil { return } + if now.IsZero() { + now = time.Now() + } clusterOffline := strings.EqualFold(strings.TrimSpace(cluster.Status), "offline") if clusterOffline { @@ -1590,11 +1657,14 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran pod.StartTime = nil continue } - if randomize && rand.Float64() < 0.25 { + // A stable per-pod choice, not a fresh roll per tick: the same + // pods on a dead node always park as Pending and the rest always + // reschedule, so repeated reconciles cannot flap a pod between + // the two outcomes. + if randomize && mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "notready-park")%4 == 0 { pod.Phase = "Pending" pod.Reason = "NodeNotReady" pod.Message = "Waiting for node recovery" - pod.StartTime = nil continue } assignIndex := int(mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "ready-node") % uint64(len(readyNodes))) @@ -1602,14 +1672,19 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran pod.Reason = "" pod.Message = "" if pod.StartTime == nil { - start := time.Now().Add(-time.Duration(30+rand.Intn(240)) * time.Second) + start := now pod.StartTime = &start } case "pending", "unknown": if len(readyNodes) == 0 { continue } - if randomize && rand.Float64() >= 0.22 { + // Only un-park pods this reconciler parked itself. Curated + // fixture stories (Unschedulable capacity pressure, + // ImagePullBackOff, CrashLoopBackOff) must stay put, or the + // scenario re-pins them every tick and the recover/re-pin cycle + // churns phantom state transitions and restarts. + if !mockReconcilerParkedPod(*pod) { continue } assignIndex := int(mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "recover-node") % uint64(len(readyNodes))) @@ -1617,8 +1692,10 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran pod.Phase = "Running" pod.Reason = "" pod.Message = "" - start := time.Now().Add(-time.Duration(20+rand.Intn(180)) * time.Second) - pod.StartTime = &start + if pod.StartTime == nil { + start := now + pod.StartTime = &start + } for j := range pod.Containers { if strings.EqualFold(pod.Containers[j].State, "terminated") { continue @@ -1632,6 +1709,22 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran } } +// mockReconcilerParkedPod reports whether a pending/unknown pod is in a +// transient state that the scheduling reconciler (or a node/cluster outage) +// produced, as opposed to a curated fixture story that must persist. +func mockReconcilerParkedPod(pod models.KubernetesPod) bool { + switch strings.TrimSpace(pod.Reason) { + case "NodeNotReady", "ClusterOffline", "NodeLost": + return true + case "Unschedulable": + // The reconciler's own capacity parking; the curated Unschedulable + // story uses "0/3 nodes available". + return strings.TrimSpace(pod.Message) == "No ready nodes available" + default: + return false + } +} + func normalizeMockKubernetesNodeCapacity(node *models.KubernetesNode) { if node == nil { return @@ -2248,6 +2341,7 @@ func generateDockerHosts(config MockConfig) []models.DockerHost { now := time.Now() hosts := make([]models.DockerHost, 0, hostCount) + swarmLeaderAssigned := false for i := 0; i < hostCount; i++ { agentVersion := dockerAgentVersions[rand.Intn(len(dockerAgentVersions))] @@ -2310,8 +2404,7 @@ func generateDockerHosts(config MockConfig) []models.DockerHost { // adapter to project Swarm services as docker-service rows // (`dockerSwarmClusterKey` returns empty without them). Anchor // the mock estate to a single named cluster so all manager and - // worker hosts share Swarm identity and their services - // deduplicate correctly across managers. + // worker hosts share Swarm identity. swarmInfo = &models.DockerSwarmInfo{ NodeID: fmt.Sprintf("%s-node", hostID), NodeRole: "worker", @@ -2324,6 +2417,17 @@ func generateDockerHosts(config MockConfig) []models.DockerHost { if i%2 == 0 { swarmInfo.NodeRole = "manager" + } + // Cluster-scoped Swarm objects (services, tasks, secrets, + // configs) come from exactly one control plane, mirroring how a + // real deployment queries them through a single manager. When + // several mock hosts each fabricated their own service list from + // their own containers under the shared cluster key, the + // registry's cross-host dedupe alternated between the divergent + // candidates on every poll and flooded resource_changes with + // phantom service renames and status flips. + if !swarmLeaderAssigned && strings.EqualFold(swarmInfo.NodeRole, "manager") { + swarmLeaderAssigned = true swarmInfo.ControlAvailable = true swarmInfo.Scope = "cluster" services, tasks = generateDockerServicesAndTasks(hostname, containers, now) @@ -2331,8 +2435,6 @@ func generateDockerHosts(config MockConfig) []models.DockerHost { if len(services) == 0 { swarmInfo.Scope = "node" } - } else if i%3 == 0 { - services, tasks = generateDockerServicesAndTasks(hostname, containers, now) } } @@ -2771,7 +2873,6 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time. } nodes := make([]models.DockerNode, 0, len(hosts)) - nodeByID := make(map[string]models.DockerNode, len(hosts)) for _, host := range hosts { if host.Swarm == nil || strings.EqualFold(host.Runtime, "podman") { continue @@ -2827,7 +2928,6 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time. CreatedAt: now.Add(-168 * time.Hour), } nodes = append(nodes, node) - nodeByID[nodeID] = node } for i := range hosts { @@ -2836,15 +2936,16 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time. host.Nodes = []models.DockerNode{} continue } + // Only the control-plane manager carries the node inventory, the + // same way `docker node ls` only answers on a manager. When every + // host repeated its own node entry, the registry saw one node from + // several reporting hosts and re-parented it to whichever host was + // polled last, spamming relationship_change rows. if strings.EqualFold(host.Swarm.NodeRole, "manager") && host.Swarm.ControlAvailable { host.Nodes = cloneMockDockerNodes(nodes) continue } - if node, ok := nodeByID[host.Swarm.NodeID]; ok { - host.Nodes = []models.DockerNode{node} - } else { - host.Nodes = []models.DockerNode{} - } + host.Nodes = []models.DockerNode{} } } @@ -5642,7 +5743,7 @@ func updateFixtureStateMetricsAt(data *models.StateSnapshot, config MockConfig, disk.Temperature = int(math.Round(SampleMetric("disk", resourceID, "smart_temp", refreshNow))) // Occasionally degrade SSD life - if disk.Wearout > 0 && rand.Float64() < 0.01 { + if disk.Wearout > 0 && rand.Float64() < mockEventsPerDayChance(mockDiskWearoutStepsPerDay) { disk.Wearout = disk.Wearout - 1 if disk.Wearout < 0 { disk.Wearout = 0 @@ -5735,7 +5836,7 @@ func updateKubernetesClusters(data *models.StateSnapshot, config MockConfig, now if cluster.Status != "offline" { cluster.LastSeen = now.Add(-time.Duration(rand.Intn(12)) * time.Second) - } else if config.RandomMetrics && rand.Float64() < 0.01 { + } else if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockK8sClusterRecoveriesPerDay) { cluster.Status = "online" cluster.LastSeen = now for nodeIdx := range cluster.Nodes { @@ -5745,13 +5846,13 @@ func updateKubernetesClusters(data *models.StateSnapshot, config MockConfig, now if config.RandomMetrics { // Small chance to flip a node Ready state. - if len(cluster.Nodes) > 0 && rand.Float64() < 0.05 { + if len(cluster.Nodes) > 0 && rand.Float64() < mockEventsPerDayChance(mockK8sNodeReadyFlipsPerDay) { idx := rand.Intn(len(cluster.Nodes)) cluster.Nodes[idx].Ready = !cluster.Nodes[idx].Ready } // Small chance to flip a pod into/out of CrashLoopBackOff. - if len(cluster.Pods) > 0 && rand.Float64() < 0.07 { + if len(cluster.Pods) > 0 && rand.Float64() < mockEventsPerDayChance(mockK8sPodCrashFlipsPerDay) { idx := rand.Intn(len(cluster.Pods)) pod := &cluster.Pods[idx] if kubernetesPodHealthy(*pod) { @@ -5821,7 +5922,7 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T } temperature := SampleMetric("dockerHost", host.ID, "temperature", now) host.Temperature = &temperature - } else if config.RandomMetrics && rand.Float64() < 0.01 { + } else if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockDockerHostRecoveriesPerDay) { // Occasionally bring an offline host back online host.Status = "online" host.LastSeen = now @@ -5861,7 +5962,7 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T flagged++ } - if config.RandomMetrics && (state == "exited" || state == "paused") && rand.Float64() < 0.02 { + if config.RandomMetrics && (state == "exited" || state == "paused") && rand.Float64() < mockEventsPerDayChance(mockDockerStoppedStartsPerDay) { container.State = "running" container.Status = "Up a few seconds" container.Health = "starting" @@ -5906,26 +6007,26 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T switch health { case "unhealthy": - if rand.Float64() < 0.3 { + if rand.Float64() < mockRecoveryChance(mockDockerUnhealthyMeanRecovery) { container.Health = "healthy" health = "healthy" } case "starting": - if rand.Float64() < 0.5 { + if rand.Float64() < mockRecoveryChance(mockDockerStartingMeanRecovery) { container.Health = "healthy" health = "healthy" } default: - if rand.Float64() < 0.03 { + if rand.Float64() < mockEventsPerDayChance(mockDockerUnhealthyPerDay) { container.Health = "unhealthy" health = "unhealthy" - } else if rand.Float64() < 0.04 { + } else if rand.Float64() < mockEventsPerDayChance(mockDockerStartingPerDay) { container.Health = "starting" health = "starting" } } - if rand.Float64() < 0.01 { + if rand.Float64() < mockEventsPerDayChance(mockDockerRestartsPerDay) { container.RestartCount++ } } @@ -6148,7 +6249,7 @@ func updateHosts(data *models.StateSnapshot, config MockConfig, now time.Time) { host.NetOutRate = 0 host.DiskReadRate = 0 host.DiskWriteRate = 0 - if config.RandomMetrics && rand.Float64() < 0.02 { + if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockHostRecoveriesPerDay) { host.Status = "online" host.LastSeen = now host.UptimeSeconds = int64(120 + rand.Intn(3600)) @@ -6191,10 +6292,10 @@ func updateHosts(data *models.StateSnapshot, config MockConfig, now time.Time) { } if host.Status == "degraded" { - if rand.Float64() < 0.25 { + if rand.Float64() < mockRecoveryChance(mockHostDegradedMeanRecovery) { host.Status = "online" } - } else if rand.Float64() < 0.05 { + } else if rand.Float64() < mockEventsPerDayChance(mockHostDegradedPerDay) { host.Status = "degraded" } }