mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Pace mock churn to homelab rates and stop phantom demo change spam
The public demo wrote ~110K resource_changes rows/day (restart 60K/day, state_transition 45K/day), making the Changes timeline unreadable and keeping unified_resources.db churning. Four generator-level engines, all verified with before/after soaks against scratch mock backends: - Per-tick flap probabilities ran 43,200x/day on the 2s update loop (docker restart p=0.01/tick alone is ~430 restarts/day/container). Churn rates are now expressed as events per day per entity and converted per tick against the configured interval, tuned to a few fleet-wide events per day with dwell times long enough to see. - The pod scheduling reconciler fought the per-tick scenario re-pin and fabricated a fresh random StartTime on every recovery; derived uptime moved backwards, which change emission records as a restart. The reconciler is now idempotent: stable per-pod park/reschedule choices, StartTime never regenerated, and only pods it parked itself (NodeNotReady/ClusterOffline/NodeLost) get recovered, so curated Pending and ImagePullBackOff stories stay put. - Swarm cluster objects were fabricated per host under one shared cluster key, so registry dedupe alternated between the divergent candidates every poll (service renames, status flips, node re-parenting). One leader manager now reports services, tasks, secrets, configs and the node inventory, like a real control plane. - Demo docker host profiles cycled 2 hostnames across 4 online hosts, collapsing canonical identities, and scripted-offline hosts had their sighting refreshed right at the 2 minute staleness threshold, sawtoothing them online/offline. Four distinct host profiles now exist and offline hosts keep a stably stale sighting. Before/after soak with a live client: pre-fix ~110-160 rows/min sustained (restart ~45/min, matching the droplet's 60K/day); post-fix zero rows/min at steady state with the curated degraded stories (CrashLoop payments-worker, ImagePullBackOff, offline hosts) intact.
This commit is contained in:
rcourtman
parent
ef2569b1fc
commit
9923e4b04f
2 changed files with 178 additions and 34 deletions
|
|
@ -136,6 +136,28 @@ var demoDockerHostProfiles = []demoDockerHostProfile{
|
|||
{Name: "grafana-agent", Image: "grafana/agent:v0.42.0", Tags: []string{"monitoring", "agent", "platform"}, Health: "healthy"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Hostname: "app-platform-01",
|
||||
DisplayName: "App Platform 01",
|
||||
Containers: []demoDockerContainerProfile{
|
||||
{Name: "orders-api", Image: "ghcr.io/pulse-demo/orders-api:2026.04", Tags: []string{"api", "production", "customer-facing"}, Health: "healthy"},
|
||||
{Name: "notifications-worker", Image: "ghcr.io/pulse-demo/notifications-worker:2026.04", Tags: []string{"queue", "worker", "production"}, Health: "healthy"},
|
||||
{Name: "redis-cache", Image: "redis:7.4.1", Tags: []string{"cache", "production", "latency-sensitive"}, Health: "healthy"},
|
||||
{Name: "nginx-edge", Image: "nginx:1.27.2", Tags: []string{"web", "ingress", "production"}, Health: "healthy"},
|
||||
{Name: "loki", Image: "grafana/loki:3.2.0", Tags: []string{"logging", "platform"}, Health: "healthy"},
|
||||
},
|
||||
},
|
||||
{
|
||||
Hostname: "data-services-01",
|
||||
DisplayName: "Data Services 01",
|
||||
Containers: []demoDockerContainerProfile{
|
||||
{Name: "kafka-broker", Image: "confluentinc/cp-kafka:7.7.1", Tags: []string{"streaming", "platform", "internal"}, Health: "healthy"},
|
||||
{Name: "clickhouse", Image: "clickhouse/clickhouse-server:24.8", Tags: []string{"database", "analytics", "internal"}, Health: "healthy"},
|
||||
{Name: "etl-runner", Image: "ghcr.io/pulse-demo/etl-runner:2026.04", Tags: []string{"batch", "analytics", "worker"}, Health: "healthy"},
|
||||
{Name: "minio", Image: "minio/minio:RELEASE.2026-01-20T00-00-00Z", Tags: []string{"storage", "gateway", "internal"}, Health: "healthy"},
|
||||
{Name: "schema-registry", Image: "confluentinc/cp-schema-registry:7.7.1", Tags: []string{"streaming", "platform", "internal"}, Health: "healthy"},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
// Per-cluster demo identities. Each cluster carries its own node naming
|
||||
|
|
@ -508,6 +530,7 @@ func applyDemoDockerScenario(state *models.StateSnapshot, now time.Time) {
|
|||
// One docker host index is forced offline so the demo Docker page exposes a
|
||||
// disconnected host with its containers shown as exited rather than running.
|
||||
const offlineDockerIndex = 2
|
||||
onlineOrdinal := 0
|
||||
for i := range state.DockerHosts {
|
||||
host := &state.DockerHosts[i]
|
||||
if i == offlineDockerIndex {
|
||||
|
|
@ -518,9 +541,20 @@ func applyDemoDockerScenario(state *models.StateSnapshot, now time.Time) {
|
|||
ensureMockDockerNativeInventory(host, i, now)
|
||||
continue
|
||||
}
|
||||
hostProfile := demoDockerHostProfiles[i%len(demoDockerHostProfiles)]
|
||||
hostProfile := demoDockerHostProfiles[onlineOrdinal%len(demoDockerHostProfiles)]
|
||||
// Hostnames must stay unique across the fleet: unified resources
|
||||
// treats hostname as identity, so two hosts sharing a profile name
|
||||
// collapse into one canonical resource whose status, parent, and
|
||||
// swarm services then flap on every poll as the two real hosts
|
||||
// alternate underneath it.
|
||||
round := onlineOrdinal / len(demoDockerHostProfiles)
|
||||
onlineOrdinal++
|
||||
host.Hostname = hostProfile.Hostname
|
||||
host.DisplayName = hostProfile.DisplayName
|
||||
if round > 0 {
|
||||
host.Hostname = fmt.Sprintf("%s-%d", hostProfile.Hostname, round+1)
|
||||
host.DisplayName = fmt.Sprintf("%s (%d)", hostProfile.DisplayName, round+1)
|
||||
}
|
||||
host.Status = "online"
|
||||
host.Containers = applyDemoDockerContainerProfiles(host.Containers, hostProfile.Containers, now)
|
||||
ensureMockDockerNativeInventory(host, i, now)
|
||||
|
|
@ -918,6 +952,15 @@ func applyDemoHostScenario(state *models.StateSnapshot, now time.Time) {
|
|||
host.Disks[j].Free = host.Disks[j].Total
|
||||
host.Disks[j].Usage = -1
|
||||
}
|
||||
// An offline host stopped reporting, so its sighting must stay
|
||||
// stale. Refreshing it like the online hosts below made the
|
||||
// agent's LastSeen race the 2-minute source staleness threshold
|
||||
// and sawtooth the resource between online and offline every
|
||||
// couple of minutes.
|
||||
if !now.IsZero() {
|
||||
host.LastSeen = now.Add(-10 * time.Minute)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if now.IsZero() {
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -61,6 +61,61 @@ const (
|
|||
hostConnectionPrefix = "host-"
|
||||
)
|
||||
|
||||
// Churn pacing for the mock estate. Restart and state-change events are
|
||||
// expressed as expected events per day per entity and converted to a
|
||||
// per-update probability against the configured update interval, so the
|
||||
// pacing does not depend on how often the mock loop ticks. The default
|
||||
// 2-second tick runs 43,200 times a day: a raw per-tick probability like
|
||||
// 0.03 means thousands of events per container per day, which floods
|
||||
// resource_changes and makes the demo Changes timeline unreadable. These
|
||||
// rates target what a real homelab produces: a few restarts and a few
|
||||
// state changes per day across the whole fleet, while per-tick metric
|
||||
// wiggle keeps the demo feeling alive.
|
||||
const (
|
||||
// One container restart every ~12 days per container; across the
|
||||
// ~56 running demo containers that is 4-5 restarts a day fleet-wide.
|
||||
mockDockerRestartsPerDay = 0.08
|
||||
// Health-check blips (healthy -> unhealthy / starting).
|
||||
mockDockerUnhealthyPerDay = 0.05
|
||||
mockDockerStartingPerDay = 0.05
|
||||
// A stopped container being started again.
|
||||
mockDockerStoppedStartsPerDay = 0.25
|
||||
// An offline docker host coming back.
|
||||
mockDockerHostRecoveriesPerDay = 0.25
|
||||
// Kubernetes background churn per cluster.
|
||||
mockK8sClusterRecoveriesPerDay = 0.5
|
||||
mockK8sNodeReadyFlipsPerDay = 0.3
|
||||
mockK8sPodCrashFlipsPerDay = 0.5
|
||||
// Standalone host status churn per host.
|
||||
mockHostDegradedPerDay = 0.1
|
||||
mockHostRecoveriesPerDay = 0.25
|
||||
mockDiskWearoutStepsPerDay = 0.1
|
||||
)
|
||||
|
||||
// Mean dwell times for transient states, converted to a per-update recovery
|
||||
// probability. Long enough to be seen on the dashboard, short enough that
|
||||
// the estate does not look permanently broken.
|
||||
const (
|
||||
mockDockerUnhealthyMeanRecovery = 12 * time.Minute
|
||||
mockDockerStartingMeanRecovery = 90 * time.Second
|
||||
mockHostDegradedMeanRecovery = 15 * time.Minute
|
||||
)
|
||||
|
||||
// mockEventsPerDayChance converts an expected events-per-day rate for a
|
||||
// single entity into a probability for one mock update tick.
|
||||
func mockEventsPerDayChance(eventsPerDay float64) float64 {
|
||||
return eventsPerDay * currentMockUpdateStepSeconds() / (24 * 60 * 60)
|
||||
}
|
||||
|
||||
// mockRecoveryChance converts a mean time-to-recovery into a probability
|
||||
// for one mock update tick.
|
||||
func mockRecoveryChance(meanRecovery time.Duration) float64 {
|
||||
if meanRecovery <= 0 {
|
||||
return 1
|
||||
}
|
||||
return currentMockUpdateStepSeconds() / meanRecovery.Seconds()
|
||||
}
|
||||
|
||||
// Default fixture sizes target a mature small-to-mid homelab / SMB
|
||||
// environment so platform pages exercise table density, sorting,
|
||||
// grouping, drawer behavior, and responsive layout out of the box:
|
||||
|
|
@ -1507,7 +1562,7 @@ func initializeMockKubernetesClusterUsage(cluster *models.KubernetesCluster, now
|
|||
return
|
||||
}
|
||||
|
||||
reconcileMockKubernetesPodScheduling(cluster, randomize)
|
||||
reconcileMockKubernetesPodScheduling(cluster, now, randomize)
|
||||
|
||||
nodeByName := make(map[string]*models.KubernetesNode, len(cluster.Nodes))
|
||||
for i := range cluster.Nodes {
|
||||
|
|
@ -1529,10 +1584,22 @@ func initializeMockKubernetesClusterUsage(cluster *models.KubernetesCluster, now
|
|||
recomputeMockKubernetesNodeUsage(cluster, now)
|
||||
}
|
||||
|
||||
func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, randomize bool) {
|
||||
// reconcileMockKubernetesPodScheduling keeps pod placement coherent with
|
||||
// node readiness. It runs on every mock update tick, both mid-tick (with
|
||||
// randomize) and again after the demo scenario re-pins pod profiles, so it
|
||||
// must be idempotent: the same inputs must produce the same pod state, and
|
||||
// it must never fabricate a fresh StartTime for a pod that already has one.
|
||||
// A regenerated StartTime moves the pod's derived uptime backwards, which
|
||||
// unified resources records as a restart — an earlier version of this
|
||||
// function re-anchored StartTime with rand on every recovery and flooded
|
||||
// resource_changes with tens of thousands of phantom pod restarts per day.
|
||||
func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, now time.Time, randomize bool) {
|
||||
if cluster == nil {
|
||||
return
|
||||
}
|
||||
if now.IsZero() {
|
||||
now = time.Now()
|
||||
}
|
||||
clusterOffline := strings.EqualFold(strings.TrimSpace(cluster.Status), "offline")
|
||||
|
||||
if clusterOffline {
|
||||
|
|
@ -1590,11 +1657,14 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran
|
|||
pod.StartTime = nil
|
||||
continue
|
||||
}
|
||||
if randomize && rand.Float64() < 0.25 {
|
||||
// A stable per-pod choice, not a fresh roll per tick: the same
|
||||
// pods on a dead node always park as Pending and the rest always
|
||||
// reschedule, so repeated reconciles cannot flap a pod between
|
||||
// the two outcomes.
|
||||
if randomize && mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "notready-park")%4 == 0 {
|
||||
pod.Phase = "Pending"
|
||||
pod.Reason = "NodeNotReady"
|
||||
pod.Message = "Waiting for node recovery"
|
||||
pod.StartTime = nil
|
||||
continue
|
||||
}
|
||||
assignIndex := int(mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "ready-node") % uint64(len(readyNodes)))
|
||||
|
|
@ -1602,14 +1672,19 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran
|
|||
pod.Reason = ""
|
||||
pod.Message = ""
|
||||
if pod.StartTime == nil {
|
||||
start := time.Now().Add(-time.Duration(30+rand.Intn(240)) * time.Second)
|
||||
start := now
|
||||
pod.StartTime = &start
|
||||
}
|
||||
case "pending", "unknown":
|
||||
if len(readyNodes) == 0 {
|
||||
continue
|
||||
}
|
||||
if randomize && rand.Float64() >= 0.22 {
|
||||
// Only un-park pods this reconciler parked itself. Curated
|
||||
// fixture stories (Unschedulable capacity pressure,
|
||||
// ImagePullBackOff, CrashLoopBackOff) must stay put, or the
|
||||
// scenario re-pins them every tick and the recover/re-pin cycle
|
||||
// churns phantom state transitions and restarts.
|
||||
if !mockReconcilerParkedPod(*pod) {
|
||||
continue
|
||||
}
|
||||
assignIndex := int(mockStableHash64(strings.TrimSpace(pod.UID), pod.Name, "recover-node") % uint64(len(readyNodes)))
|
||||
|
|
@ -1617,8 +1692,10 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran
|
|||
pod.Phase = "Running"
|
||||
pod.Reason = ""
|
||||
pod.Message = ""
|
||||
start := time.Now().Add(-time.Duration(20+rand.Intn(180)) * time.Second)
|
||||
pod.StartTime = &start
|
||||
if pod.StartTime == nil {
|
||||
start := now
|
||||
pod.StartTime = &start
|
||||
}
|
||||
for j := range pod.Containers {
|
||||
if strings.EqualFold(pod.Containers[j].State, "terminated") {
|
||||
continue
|
||||
|
|
@ -1632,6 +1709,22 @@ func reconcileMockKubernetesPodScheduling(cluster *models.KubernetesCluster, ran
|
|||
}
|
||||
}
|
||||
|
||||
// mockReconcilerParkedPod reports whether a pending/unknown pod is in a
|
||||
// transient state that the scheduling reconciler (or a node/cluster outage)
|
||||
// produced, as opposed to a curated fixture story that must persist.
|
||||
func mockReconcilerParkedPod(pod models.KubernetesPod) bool {
|
||||
switch strings.TrimSpace(pod.Reason) {
|
||||
case "NodeNotReady", "ClusterOffline", "NodeLost":
|
||||
return true
|
||||
case "Unschedulable":
|
||||
// The reconciler's own capacity parking; the curated Unschedulable
|
||||
// story uses "0/3 nodes available".
|
||||
return strings.TrimSpace(pod.Message) == "No ready nodes available"
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func normalizeMockKubernetesNodeCapacity(node *models.KubernetesNode) {
|
||||
if node == nil {
|
||||
return
|
||||
|
|
@ -2248,6 +2341,7 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
|
||||
now := time.Now()
|
||||
hosts := make([]models.DockerHost, 0, hostCount)
|
||||
swarmLeaderAssigned := false
|
||||
|
||||
for i := 0; i < hostCount; i++ {
|
||||
agentVersion := dockerAgentVersions[rand.Intn(len(dockerAgentVersions))]
|
||||
|
|
@ -2310,8 +2404,7 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
// adapter to project Swarm services as docker-service rows
|
||||
// (`dockerSwarmClusterKey` returns empty without them). Anchor
|
||||
// the mock estate to a single named cluster so all manager and
|
||||
// worker hosts share Swarm identity and their services
|
||||
// deduplicate correctly across managers.
|
||||
// worker hosts share Swarm identity.
|
||||
swarmInfo = &models.DockerSwarmInfo{
|
||||
NodeID: fmt.Sprintf("%s-node", hostID),
|
||||
NodeRole: "worker",
|
||||
|
|
@ -2324,6 +2417,17 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
|
||||
if i%2 == 0 {
|
||||
swarmInfo.NodeRole = "manager"
|
||||
}
|
||||
// Cluster-scoped Swarm objects (services, tasks, secrets,
|
||||
// configs) come from exactly one control plane, mirroring how a
|
||||
// real deployment queries them through a single manager. When
|
||||
// several mock hosts each fabricated their own service list from
|
||||
// their own containers under the shared cluster key, the
|
||||
// registry's cross-host dedupe alternated between the divergent
|
||||
// candidates on every poll and flooded resource_changes with
|
||||
// phantom service renames and status flips.
|
||||
if !swarmLeaderAssigned && strings.EqualFold(swarmInfo.NodeRole, "manager") {
|
||||
swarmLeaderAssigned = true
|
||||
swarmInfo.ControlAvailable = true
|
||||
swarmInfo.Scope = "cluster"
|
||||
services, tasks = generateDockerServicesAndTasks(hostname, containers, now)
|
||||
|
|
@ -2331,8 +2435,6 @@ func generateDockerHosts(config MockConfig) []models.DockerHost {
|
|||
if len(services) == 0 {
|
||||
swarmInfo.Scope = "node"
|
||||
}
|
||||
} else if i%3 == 0 {
|
||||
services, tasks = generateDockerServicesAndTasks(hostname, containers, now)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -2771,7 +2873,6 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time.
|
|||
}
|
||||
|
||||
nodes := make([]models.DockerNode, 0, len(hosts))
|
||||
nodeByID := make(map[string]models.DockerNode, len(hosts))
|
||||
for _, host := range hosts {
|
||||
if host.Swarm == nil || strings.EqualFold(host.Runtime, "podman") {
|
||||
continue
|
||||
|
|
@ -2827,7 +2928,6 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time.
|
|||
CreatedAt: now.Add(-168 * time.Hour),
|
||||
}
|
||||
nodes = append(nodes, node)
|
||||
nodeByID[nodeID] = node
|
||||
}
|
||||
|
||||
for i := range hosts {
|
||||
|
|
@ -2836,15 +2936,16 @@ func populateMockDockerSwarmNodeInventories(hosts []models.DockerHost, now time.
|
|||
host.Nodes = []models.DockerNode{}
|
||||
continue
|
||||
}
|
||||
// Only the control-plane manager carries the node inventory, the
|
||||
// same way `docker node ls` only answers on a manager. When every
|
||||
// host repeated its own node entry, the registry saw one node from
|
||||
// several reporting hosts and re-parented it to whichever host was
|
||||
// polled last, spamming relationship_change rows.
|
||||
if strings.EqualFold(host.Swarm.NodeRole, "manager") && host.Swarm.ControlAvailable {
|
||||
host.Nodes = cloneMockDockerNodes(nodes)
|
||||
continue
|
||||
}
|
||||
if node, ok := nodeByID[host.Swarm.NodeID]; ok {
|
||||
host.Nodes = []models.DockerNode{node}
|
||||
} else {
|
||||
host.Nodes = []models.DockerNode{}
|
||||
}
|
||||
host.Nodes = []models.DockerNode{}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -5642,7 +5743,7 @@ func updateFixtureStateMetricsAt(data *models.StateSnapshot, config MockConfig,
|
|||
disk.Temperature = int(math.Round(SampleMetric("disk", resourceID, "smart_temp", refreshNow)))
|
||||
|
||||
// Occasionally degrade SSD life
|
||||
if disk.Wearout > 0 && rand.Float64() < 0.01 {
|
||||
if disk.Wearout > 0 && rand.Float64() < mockEventsPerDayChance(mockDiskWearoutStepsPerDay) {
|
||||
disk.Wearout = disk.Wearout - 1
|
||||
if disk.Wearout < 0 {
|
||||
disk.Wearout = 0
|
||||
|
|
@ -5735,7 +5836,7 @@ func updateKubernetesClusters(data *models.StateSnapshot, config MockConfig, now
|
|||
|
||||
if cluster.Status != "offline" {
|
||||
cluster.LastSeen = now.Add(-time.Duration(rand.Intn(12)) * time.Second)
|
||||
} else if config.RandomMetrics && rand.Float64() < 0.01 {
|
||||
} else if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockK8sClusterRecoveriesPerDay) {
|
||||
cluster.Status = "online"
|
||||
cluster.LastSeen = now
|
||||
for nodeIdx := range cluster.Nodes {
|
||||
|
|
@ -5745,13 +5846,13 @@ func updateKubernetesClusters(data *models.StateSnapshot, config MockConfig, now
|
|||
|
||||
if config.RandomMetrics {
|
||||
// Small chance to flip a node Ready state.
|
||||
if len(cluster.Nodes) > 0 && rand.Float64() < 0.05 {
|
||||
if len(cluster.Nodes) > 0 && rand.Float64() < mockEventsPerDayChance(mockK8sNodeReadyFlipsPerDay) {
|
||||
idx := rand.Intn(len(cluster.Nodes))
|
||||
cluster.Nodes[idx].Ready = !cluster.Nodes[idx].Ready
|
||||
}
|
||||
|
||||
// Small chance to flip a pod into/out of CrashLoopBackOff.
|
||||
if len(cluster.Pods) > 0 && rand.Float64() < 0.07 {
|
||||
if len(cluster.Pods) > 0 && rand.Float64() < mockEventsPerDayChance(mockK8sPodCrashFlipsPerDay) {
|
||||
idx := rand.Intn(len(cluster.Pods))
|
||||
pod := &cluster.Pods[idx]
|
||||
if kubernetesPodHealthy(*pod) {
|
||||
|
|
@ -5821,7 +5922,7 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T
|
|||
}
|
||||
temperature := SampleMetric("dockerHost", host.ID, "temperature", now)
|
||||
host.Temperature = &temperature
|
||||
} else if config.RandomMetrics && rand.Float64() < 0.01 {
|
||||
} else if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockDockerHostRecoveriesPerDay) {
|
||||
// Occasionally bring an offline host back online
|
||||
host.Status = "online"
|
||||
host.LastSeen = now
|
||||
|
|
@ -5861,7 +5962,7 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T
|
|||
flagged++
|
||||
}
|
||||
|
||||
if config.RandomMetrics && (state == "exited" || state == "paused") && rand.Float64() < 0.02 {
|
||||
if config.RandomMetrics && (state == "exited" || state == "paused") && rand.Float64() < mockEventsPerDayChance(mockDockerStoppedStartsPerDay) {
|
||||
container.State = "running"
|
||||
container.Status = "Up a few seconds"
|
||||
container.Health = "starting"
|
||||
|
|
@ -5906,26 +6007,26 @@ func updateDockerHosts(data *models.StateSnapshot, config MockConfig, now time.T
|
|||
|
||||
switch health {
|
||||
case "unhealthy":
|
||||
if rand.Float64() < 0.3 {
|
||||
if rand.Float64() < mockRecoveryChance(mockDockerUnhealthyMeanRecovery) {
|
||||
container.Health = "healthy"
|
||||
health = "healthy"
|
||||
}
|
||||
case "starting":
|
||||
if rand.Float64() < 0.5 {
|
||||
if rand.Float64() < mockRecoveryChance(mockDockerStartingMeanRecovery) {
|
||||
container.Health = "healthy"
|
||||
health = "healthy"
|
||||
}
|
||||
default:
|
||||
if rand.Float64() < 0.03 {
|
||||
if rand.Float64() < mockEventsPerDayChance(mockDockerUnhealthyPerDay) {
|
||||
container.Health = "unhealthy"
|
||||
health = "unhealthy"
|
||||
} else if rand.Float64() < 0.04 {
|
||||
} else if rand.Float64() < mockEventsPerDayChance(mockDockerStartingPerDay) {
|
||||
container.Health = "starting"
|
||||
health = "starting"
|
||||
}
|
||||
}
|
||||
|
||||
if rand.Float64() < 0.01 {
|
||||
if rand.Float64() < mockEventsPerDayChance(mockDockerRestartsPerDay) {
|
||||
container.RestartCount++
|
||||
}
|
||||
}
|
||||
|
|
@ -6148,7 +6249,7 @@ func updateHosts(data *models.StateSnapshot, config MockConfig, now time.Time) {
|
|||
host.NetOutRate = 0
|
||||
host.DiskReadRate = 0
|
||||
host.DiskWriteRate = 0
|
||||
if config.RandomMetrics && rand.Float64() < 0.02 {
|
||||
if config.RandomMetrics && rand.Float64() < mockEventsPerDayChance(mockHostRecoveriesPerDay) {
|
||||
host.Status = "online"
|
||||
host.LastSeen = now
|
||||
host.UptimeSeconds = int64(120 + rand.Intn(3600))
|
||||
|
|
@ -6191,10 +6292,10 @@ func updateHosts(data *models.StateSnapshot, config MockConfig, now time.Time) {
|
|||
}
|
||||
|
||||
if host.Status == "degraded" {
|
||||
if rand.Float64() < 0.25 {
|
||||
if rand.Float64() < mockRecoveryChance(mockHostDegradedMeanRecovery) {
|
||||
host.Status = "online"
|
||||
}
|
||||
} else if rand.Float64() < 0.05 {
|
||||
} else if rand.Float64() < mockEventsPerDayChance(mockHostDegradedPerDay) {
|
||||
host.Status = "degraded"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue