From ca78a10978184e5e36c25a85c7cb2cf3f2ec4787 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 1 Apr 2026 14:05:26 +0100 Subject: [PATCH] Align mock chart seeding with live timeline --- .../v6/internal/subsystems/monitoring.md | 3 + internal/monitoring/mock_metrics_history.go | 102 ++++-------- .../monitoring/mock_metrics_history_test.go | 148 +++++++++++++++++- 3 files changed, 175 insertions(+), 78 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index e19981d77..fc280e4cc 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -133,6 +133,9 @@ window instead of stitching a second live tail onto the end of seeded sparklines. Monitoring must not let provider-owned mock resources receive a duplicate generic unified-resource writer that appends a divergent recent tail after the canonical mock sampler has already seeded and extended that series. +The seed path must therefore include the canonical terminal `now` sample on +its tiered timeline instead of generating history only up to before `now` and +then appending a separately anchored current-value tail afterward. That same chart boundary also owns role-shaped realism. Seeded history, synthetic summary fallbacks, and runtime mock writes must derive their bounds and curve shape from the same canonical resource-role registry, so database, diff --git a/internal/monitoring/mock_metrics_history.go b/internal/monitoring/mock_metrics_history.go index b34916c6e..c2f165101 100644 --- a/internal/monitoring/mock_metrics_history.go +++ b/internal/monitoring/mock_metrics_history.go @@ -501,14 +501,15 @@ func generateFlatSeries(current float64, points int, min, max, span float64, rng } // buildTieredTimestamps generates a sorted list of timestamps with denser -// intervals for recent data: +// intervals for recent data, including the canonical terminal sample at now: // // Last 2h: 1min intervals (~120 points) // 2h–24h: 2min intervals (~660 points) // 24h–end: ~65min intervals (variable) // // This ensures short time ranges (1h, 4h) have enough data points without -// needing an API-level fallback layer. +// needing an API-level fallback layer, and it keeps seeded history on the same +// timeline the live mock sampler would have produced. func buildTieredTimestamps(now time.Time, totalDuration time.Duration) []time.Time { // Each segment covers [now - startOffset, now - endOffset) and is walked // chronologically from oldest to newest. Segments are defined oldest-first @@ -547,6 +548,10 @@ func buildTieredTimestamps(now time.Time, totalDuration time.Duration) []time.Ti } } + if len(timestamps) == 0 || !timestamps[len(timestamps)-1].Equal(now) { + timestamps = append(timestamps, now) + } + return timestamps } @@ -559,13 +564,11 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi return } - // Build a tiered timestamp list so short time ranges (1h, 4h) have dense - // data without needing an API-level fallback layer. + // Build one canonical timestamp list so seeded history and subsequent live + // mock ticks sample the same runtime timeline model. // Last 2h: 1min intervals (~120 points) // 2h–24h: 2min intervals (~660 points) // 24h–90d: ~65min intervals (~1920 points) - // The current "now" point is appended explicitly by each recorder so the - // seed and live sampler share one canonical terminal timestamp. seedTimestamps := buildTieredTimestamps(now, seedDuration) const seedBatchSize = 5000 numPoints := len(seedTimestamps) @@ -603,34 +606,30 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi return } - series := mockmodel.StorageCapacitySeriesForTimestampsWithRole( - currentUsed, - currentTotal, + currentUsage := clampFloat((currentUsed/currentTotal)*100, 0, 100) + usageSeries := GenerateSeededResourceMetricSeriesForTimestamps( + currentUsage, seedTimestamps, - mock.MetricSeed("storage", storageID, "usage"), - mock.MetricRole("storage", storageID), + "storage", + storageID, + "usage", + styleFlat, ) for i := 0; i < numPoints; i++ { ts := seedTimestamps[i] - mh.AddStorageMetric(storageID, "usage", series.Usage[i], ts) - mh.AddStorageMetric(storageID, "used", series.Used[i], ts) - mh.AddStorageMetric(storageID, "avail", series.Avail[i], ts) - mh.AddStorageMetric(storageID, "total", series.Total[i], ts) - queueMetric("storage", storageID, "usage", series.Usage[i], ts) - queueMetric("storage", storageID, "used", series.Used[i], ts) - queueMetric("storage", storageID, "avail", series.Avail[i], ts) - queueMetric("storage", storageID, "total", series.Total[i], ts) + usage := clampFloat(usageSeries[i], 0, 100) + used := currentTotal * (usage / 100.0) + avail := math.Max(0, currentTotal-used) + mh.AddStorageMetric(storageID, "usage", usage, ts) + mh.AddStorageMetric(storageID, "used", used, ts) + mh.AddStorageMetric(storageID, "avail", avail, ts) + mh.AddStorageMetric(storageID, "total", currentTotal, ts) + queueMetric("storage", storageID, "usage", usage, ts) + queueMetric("storage", storageID, "used", used, ts) + queueMetric("storage", storageID, "avail", avail, ts) + queueMetric("storage", storageID, "total", currentTotal, ts) } - last := numPoints - 1 - mh.AddStorageMetric(storageID, "usage", series.Usage[last], now) - mh.AddStorageMetric(storageID, "used", series.Used[last], now) - mh.AddStorageMetric(storageID, "avail", series.Avail[last], now) - mh.AddStorageMetric(storageID, "total", series.Total[last], now) - queueMetric("storage", storageID, "usage", series.Usage[last], now) - queueMetric("storage", storageID, "used", series.Used[last], now) - queueMetric("storage", storageID, "avail", series.Avail[last], now) - queueMetric("storage", storageID, "total", series.Total[last], now) } recordNode := func(node models.Node) { @@ -656,13 +655,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi queueMetric("node", node.ID, "disk", diskSeries[i], ts) } - // Ensure the latest point lands at "now" for full-range charts. - mh.AddNodeMetric(node.ID, "cpu", node.CPU*100, now) - mh.AddNodeMetric(node.ID, "memory", node.Memory.Usage, now) - mh.AddNodeMetric(node.ID, "disk", node.Disk.Usage, now) - queueMetric("node", node.ID, "cpu", node.CPU*100, now) - queueMetric("node", node.ID, "memory", node.Memory.Usage, now) - queueMetric("node", node.ID, "disk", node.Disk.Usage, now) } recordGuest := func( @@ -748,35 +740,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi } } - // Ensure the latest point lands at "now" for full-range charts. - for _, metricID := range uniqueMetricIDs { - mh.AddGuestMetric(metricID, "cpu", cpuPercent, now) - mh.AddGuestMetric(metricID, "memory", memPercent, now) - } - queueMetric(storeType, storeID, "cpu", cpuPercent, now) - queueMetric(storeType, storeID, "memory", memPercent, now) - if includeDisk { - for _, metricID := range uniqueMetricIDs { - mh.AddGuestMetric(metricID, "disk", diskPercent, now) - } - queueMetric(storeType, storeID, "disk", diskPercent, now) - } - if includeDiskIO { - for _, metricID := range uniqueMetricIDs { - mh.AddGuestMetric(metricID, "diskread", diskRead, now) - mh.AddGuestMetric(metricID, "diskwrite", diskWrite, now) - } - queueMetric(storeType, storeID, "diskread", diskRead, now) - queueMetric(storeType, storeID, "diskwrite", diskWrite, now) - } - if includeNetwork { - for _, metricID := range uniqueMetricIDs { - mh.AddGuestMetric(metricID, "netin", netIn, now) - mh.AddGuestMetric(metricID, "netout", netOut, now) - } - queueMetric(storeType, storeID, "netin", netIn, now) - queueMetric(storeType, storeID, "netout", netOut, now) - } } log.Debug().Int("count", len(state.Nodes)).Msg("mock seeding: processing nodes") @@ -921,9 +884,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts) } - // Ensure the latest point lands at "now" for full-range charts. - mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now) - queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now) } log.Debug().Int("count", len(state.CephClusters)).Msg("mock seeding: processing ceph clusters") @@ -954,8 +914,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi queueMetric("ceph", cephID, "usage", usageSeries[i], ts) } - // Ensure the latest point lands at "now" for full-range charts. - queueMetric("ceph", cephID, "usage", cluster.UsagePercent, now) } log.Debug().Int("count", len(state.DockerHosts)).Msg("mock seeding: processing docker hosts") @@ -1079,8 +1037,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi mh.AddGuestMetric(poolKey, "disk", diskSeries[i], ts) queueMetric("storage", poolKey, "usage", diskSeries[i], ts) } - mh.AddGuestMetric(poolKey, "disk", diskPercent, now) - queueMetric("storage", poolKey, "usage", diskPercent, now) recordStorageTimeline(poolKey, float64(pool.UsedBytes), float64(pool.TotalBytes)) } @@ -1106,8 +1062,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi mh.AddGuestMetric(dsKey, "disk", diskSeries[i], ts) queueMetric("storage", dsKey, "usage", diskSeries[i], ts) } - mh.AddGuestMetric(dsKey, "disk", diskPercent, now) - queueMetric("storage", dsKey, "usage", diskPercent, now) recordStorageTimeline(dsKey, float64(dataset.UsedBytes), float64(totalBytes)) } @@ -1136,8 +1090,6 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi mh.AddDiskMetric(resourceID, "smart_temp", tempSeries[i], ts) queueMetric("disk", resourceID, "smart_temp", tempSeries[i], ts) } - mh.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now) - queueMetric("disk", resourceID, "smart_temp", float64(disk.Temperature), now) } for _, app := range trueNASFixtures.Apps { diff --git a/internal/monitoring/mock_metrics_history_test.go b/internal/monitoring/mock_metrics_history_test.go index d161bab25..bd88093d7 100644 --- a/internal/monitoring/mock_metrics_history_test.go +++ b/internal/monitoring/mock_metrics_history_test.go @@ -19,7 +19,7 @@ func fixtureGraphWithState(state models.StateSnapshot) mock.FixtureGraph { return mock.FixtureGraph{State: state} } -func TestBuildTieredTimestamps_LeavesTerminalNowToRecorders(t *testing.T) { +func TestBuildTieredTimestamps_IncludesCanonicalTerminalNow(t *testing.T) { now := time.Date(2026, time.March, 31, 12, 0, 0, 0, time.UTC) timestamps := buildTieredTimestamps(now, time.Hour) @@ -28,8 +28,8 @@ func TestBuildTieredTimestamps_LeavesTerminalNowToRecorders(t *testing.T) { } last := timestamps[len(timestamps)-1] - if !last.Before(now) { - t.Fatalf("expected seed timestamps to stop before now, got %v with now=%v", last, now) + if !last.Equal(now) { + t.Fatalf("expected seed timestamps to include terminal now, got %v with now=%v", last, now) } } @@ -925,3 +925,145 @@ func TestGenerateSeededMetricSeriesForTimestamps_UsesSameTimelineAsMockRuntime(t } } } + +func TestSeedMockMetricsHistory_StaysContinuousWithSubsequentLiveMockTicks(t *testing.T) { + now := time.Date(2026, time.March, 31, 12, 0, 0, 0, time.UTC) + next := now.Add(time.Minute) + const storageTotal = int64(2 * 1024 * 1024 * 1024 * 1024) + + storageUsageAt := func(at time.Time) (float64, int64, int64) { + usage := mock.SampleMetric("storage", "storage-tail", "usage", at) + used := int64(math.Round((float64(storageTotal) * usage) / 100.0)) + if used < 0 { + used = 0 + } + if used > storageTotal { + used = storageTotal + } + return usage, used, storageTotal - used + } + + vmMemoryNow := mock.SampleMetric("vm", "vm-tail", "memory", now) + vmDiskNow := mock.SampleMetric("vm", "vm-tail", "disk", now) + storageUsageNow, storageUsedNow, storageFreeNow := storageUsageAt(now) + + seedState := models.StateSnapshot{ + VMs: []models.VM{ + { + ID: "vm-tail", + Status: "running", + CPU: mock.SampleMetric("vm", "vm-tail", "cpu", now) / 100.0, + Memory: models.Memory{ + Usage: vmMemoryNow, + Total: 16 * 1024 * 1024 * 1024, + Used: int64(math.Round((16 * 1024 * 1024 * 1024) * (vmMemoryNow / 100.0))), + }, + Disk: models.Disk{ + Usage: vmDiskNow, + Total: 512 * 1024 * 1024 * 1024, + Used: int64(math.Round((512 * 1024 * 1024 * 1024) * (vmDiskNow / 100.0))), + }, + NetworkIn: mock.SampleMetricInt("vm", "vm-tail", "netin", now), + NetworkOut: mock.SampleMetricInt("vm", "vm-tail", "netout", now), + DiskRead: mock.SampleMetricInt("vm", "vm-tail", "diskread", now), + DiskWrite: mock.SampleMetricInt("vm", "vm-tail", "diskwrite", now), + }, + }, + Storage: []models.Storage{ + { + ID: "storage-tail", + Status: "available", + Total: storageTotal, + Used: storageUsedNow, + Free: storageFreeNow, + Usage: storageUsageNow, + }, + }, + } + + mh := NewMetricsHistory(5000, 7*24*time.Hour) + seedMockMetricsHistory(mh, nil, fixtureGraphWithState(seedState), now, 7*24*time.Hour, time.Minute) + + vmCPUSeeded := mh.GetGuestMetrics("vm-tail", "cpu", 7*24*time.Hour) + if len(vmCPUSeeded) == 0 { + t.Fatal("expected seeded vm cpu history") + } + for i, point := range vmCPUSeeded { + want := mock.SampleMetric("vm", "vm-tail", "cpu", point.Timestamp) + if diff := math.Abs(point.Value - want); diff > 1e-9 { + t.Fatalf("expected seeded vm cpu point %d to follow canonical runtime timeline: got=%f want=%f ts=%v", i, point.Value, want, point.Timestamp) + } + } + + storageSeeded := mh.GetAllStorageMetrics("storage-tail", 7*24*time.Hour)["usage"] + if len(storageSeeded) == 0 { + t.Fatal("expected seeded storage usage history") + } + for i, point := range storageSeeded { + want := mock.SampleMetric("storage", "storage-tail", "usage", point.Timestamp) + if diff := math.Abs(point.Value - want); diff > 1e-9 { + t.Fatalf("expected seeded storage usage point %d to follow canonical runtime timeline: got=%f want=%f ts=%v", i, point.Value, want, point.Timestamp) + } + } + + vmMemoryNext := mock.SampleMetric("vm", "vm-tail", "memory", next) + vmDiskNext := mock.SampleMetric("vm", "vm-tail", "disk", next) + storageUsageNext, storageUsedNext, storageFreeNext := storageUsageAt(next) + liveState := models.StateSnapshot{ + VMs: []models.VM{ + { + ID: "vm-tail", + Status: "running", + CPU: mock.SampleMetric("vm", "vm-tail", "cpu", next) / 100.0, + Memory: models.Memory{ + Usage: vmMemoryNext, + Total: 16 * 1024 * 1024 * 1024, + Used: int64(math.Round((16 * 1024 * 1024 * 1024) * (vmMemoryNext / 100.0))), + }, + Disk: models.Disk{ + Usage: vmDiskNext, + Total: 512 * 1024 * 1024 * 1024, + Used: int64(math.Round((512 * 1024 * 1024 * 1024) * (vmDiskNext / 100.0))), + }, + NetworkIn: mock.SampleMetricInt("vm", "vm-tail", "netin", next), + NetworkOut: mock.SampleMetricInt("vm", "vm-tail", "netout", next), + DiskRead: mock.SampleMetricInt("vm", "vm-tail", "diskread", next), + DiskWrite: mock.SampleMetricInt("vm", "vm-tail", "diskwrite", next), + }, + }, + Storage: []models.Storage{ + { + ID: "storage-tail", + Status: "available", + Total: storageTotal, + Used: storageUsedNext, + Free: storageFreeNext, + Usage: storageUsageNext, + }, + }, + } + + recordMockStateToMetricsHistory(mh, nil, fixtureGraphWithState(liveState), next) + + vmCPUAfterTick := mh.GetGuestMetrics("vm-tail", "cpu", 7*24*time.Hour) + if got := vmCPUAfterTick[len(vmCPUAfterTick)-1].Timestamp; !got.Equal(next) { + t.Fatalf("expected latest vm cpu point at %v, got %v", next, got) + } + if got, want := vmCPUAfterTick[len(vmCPUAfterTick)-1].Value, mock.SampleMetric("vm", "vm-tail", "cpu", next); math.Abs(got-want) > 1e-9 { + t.Fatalf("expected live vm cpu tick to continue canonical runtime timeline: got=%f want=%f", got, want) + } + if got := vmCPUAfterTick[len(vmCPUAfterTick)-2].Timestamp; !got.Equal(now) { + t.Fatalf("expected penultimate vm cpu point to remain anchored at seed now %v, got %v", now, got) + } + + storageAfterTick := mh.GetAllStorageMetrics("storage-tail", 7*24*time.Hour)["usage"] + if got := storageAfterTick[len(storageAfterTick)-1].Timestamp; !got.Equal(next) { + t.Fatalf("expected latest storage usage point at %v, got %v", next, got) + } + if got, want := storageAfterTick[len(storageAfterTick)-1].Value, mock.SampleMetric("storage", "storage-tail", "usage", next); math.Abs(got-want) > 1e-9 { + t.Fatalf("expected live storage usage tick to continue canonical runtime timeline: got=%f want=%f", got, want) + } + if got := storageAfterTick[len(storageAfterTick)-2].Timestamp; !got.Equal(now) { + t.Fatalf("expected penultimate storage point to remain anchored at seed now %v, got %v", now, got) + } +}