From 6388bd8b97bccea9cc00d7c828dbc6f79cf92dce Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 1 Apr 2026 19:14:12 +0100 Subject: [PATCH] Stabilize low-trust Proxmox VM memory readings --- .../subsystems/frontend-primitives.md | 9 +- .../v6/internal/subsystems/monitoring.md | 15 +- .../shared/InteractiveSparkline.tsx | 14 +- .../SharedPrimitives.guardrails.test.ts | 14 ++ .../src/components/shared/TooltipPortal.tsx | 3 +- .../monitoring/canonical_guardrails_test.go | 37 ++++ .../monitoring/diagnostic_snapshots_test.go | 30 +++ internal/monitoring/guest_memory_stability.go | 150 +++++++++++++ .../monitoring/guest_memory_stability_test.go | 203 ++++++++++++++++++ .../monitoring/monitor_extra_coverage_test.go | 4 +- internal/monitoring/monitor_polling_vm.go | 20 ++ .../monitoring/monitor_proxmox_pool_test.go | 2 +- .../monitoring/monitor_pve_guest_builders.go | 25 ++- internal/monitoring/monitor_pve_guest_poll.go | 3 +- 14 files changed, 508 insertions(+), 21 deletions(-) create mode 100644 internal/monitoring/guest_memory_stability.go create mode 100644 internal/monitoring/guest_memory_stability_test.go diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 40643f682..2c352e95a 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -230,8 +230,8 @@ connections` visible as the API-backed alternative for Proxmox and `frontend-modern/src/routing/resourceLinks.ts` instead of freezing raw route strings or provider-local link builders inside feature panels. 15. Keep shared summary-card emphasis coherent. When shared summary primitives enter an `inactive` state, `SummaryMetricCard`, `InteractiveSparkline`, and `DensityMap` must all demote background context together so storage, infrastructure, and workloads read as one interaction model instead of mixing page-local opacity, sticky-shell, or highlight rules. -16. Keep density-map summaries overview-first. When a shared summary density map receives row focus or chart-hover emphasis, `frontend-modern/src/components/shared/DensityMap.tsx`, `frontend-modern/src/components/shared/useDensityMapState.ts`, and `frontend-modern/src/components/shared/densityMapModel.ts` must preserve the multi-entity overview rows and keep focused-entity detail in the hover tooltip instead of swapping the card into a single-series chart, dimming the rest of the map into unusable background noise, duplicating cursor-value tooltip copy, or adding persistent card chrome that steals heatmap space. The card body must stay overview-first; the tooltip may carry the active entity identity, current value, peak, and mini sparkline. -17. Keep sparkline scrubbing source-local and sibling-sync timestamp-based. The chart a user is actively scrubbing in `frontend-modern/src/components/shared/InteractiveSparkline.tsx` and `frontend-modern/src/components/shared/useInteractiveSparklineState.ts` must keep its dashed hover cursor on the real local mouse `x`, while sibling cards may map the shared hover timestamp onto their own timelines. Shared cursor sync must not snap the source chart back onto the nearest sample timestamp, the rendered SVG/canvas hover cursor must bind to the actual numeric cursor coordinate rather than a boolean guard state, the time cursor must span the chart viewport instead of collapsing to the series height, and the hover tooltip must track the pointer instead of anchoring to the chart top edge. +16. Keep density-map summaries overview-first. When a shared summary density map receives row focus or chart-hover emphasis, `frontend-modern/src/components/shared/DensityMap.tsx`, `frontend-modern/src/components/shared/useDensityMapState.ts`, and `frontend-modern/src/components/shared/densityMapModel.ts` must preserve the multi-entity overview rows and keep focused-entity detail in the hover tooltip instead of swapping the card into a single-series chart, dimming the rest of the map into unusable background noise, duplicating cursor-value tooltip copy, or adding persistent card chrome that steals heatmap space. The card body must stay overview-first; the tooltip may carry the active entity identity, current value, peak, and mini sparkline, and shared tooltip shells must follow semantic surface tokens instead of forcing a dark palette in light mode. +17. Keep sparkline scrubbing source-local and sibling-sync timestamp-based. The chart a user is actively scrubbing in `frontend-modern/src/components/shared/InteractiveSparkline.tsx` and `frontend-modern/src/components/shared/useInteractiveSparklineState.ts` must keep its dashed hover cursor on the real local mouse `x`, while sibling cards may map the shared hover timestamp onto their own timelines. Shared cursor sync must not snap the source chart back onto the nearest sample timestamp, the rendered SVG/canvas hover cursor must bind to the actual numeric cursor coordinate rather than a boolean guard state, the time cursor must span the chart viewport instead of collapsing to the series height, and the hover tooltip must track the pointer instead of anchoring to the chart top edge while following the active theme rather than a hardcoded dark shell. 18. Keep shared contextual focus canonical after adoption. Once a summary or table surface enters route-backed contextual focus, future additions must extend `frontend-modern/src/components/shared/contextualFocus.ts` and its guardrail tests rather than forking another helper for workload IDs, resource IDs, or scroll-preserving same-route selection. 19. Keep shared infrastructure/resource selectors on the canonical agent-facet truth. Shared primitives and settings-facing selector helpers must treat @@ -854,7 +854,10 @@ owns tooltip positioning lifecycle, RAF scheduling, and singleton visibility state, and `frontend-modern/src/components/shared/tooltipModel.ts` owns tooltip sanitization plus viewport-clamped positioning math. Future tooltip work should extend those owners instead of pushing singleton state, DOM measurement, or -sanitization logic back into the shared shell. +sanitization logic back into the shared shell. Shared portal-mounted tooltip +shells such as `frontend-modern/src/components/shared/TooltipPortal.tsx` must +use the same semantic surface tokens as the canonical tooltip instead of +introducing light-mode-inverted palettes. The shared collapsible search input now follows that same owner split. `frontend-modern/src/components/shared/CollapsibleSearchInput.tsx` stays the render shell, `frontend-modern/src/components/shared/useCollapsibleSearchInputState.ts` diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index cde05f36b..3cb0ececf 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -40,6 +40,11 @@ truth for live infrastructure data. 16. `internal/monitoring/truenas_poller.go` 17. `internal/monitoring/vmware_poller.go` 18. `internal/dockeragent/swarm.go` +19. `internal/monitoring/guest_memory_sources.go` +20. `internal/monitoring/guest_memory_stability.go` +21. `internal/monitoring/monitor_polling_vm.go` +22. `internal/monitoring/monitor_pve_guest_builders.go` +23. `internal/monitoring/monitor_pve_guest_poll.go` ## Shared Boundaries @@ -64,7 +69,7 @@ truth for live infrastructure data. 1. Update this contract when monitoring truth ownership changes 2. Tighten guardrails when `GetState()`-centric paths are removed -3. Keep discovery-provider, metrics-history, Docker Swarm collection, and container bootstrap proof routes explicit in `registry.json` +3. Keep discovery-provider, guest-memory trust, metrics-history, Docker Swarm collection, and container bootstrap proof routes explicit in `registry.json` 4. Update related read-state or monitor tests when new collector paths land 5. Keep platform ingestion semantics aligned with `docs/release-control/v6/internal/PLATFORM_SUPPORT_MODEL.md`: hybrid is a @@ -418,6 +423,14 @@ and guest-agent fallback caches must key on `(instance, node, vmid)` instead of raw `node/vmid`, so separate Proxmox instances cannot leak stale or foreign memory evidence into each other just because they reuse the same node name and VMID. +That same guest-memory boundary also owns stabilization when Proxmox falls +back to low-trust VM full-usage readings. The shared VM polling paths must use +the previous guest diagnostic snapshot, not the resource model, to decide when +one more `previous-snapshot` carry-forward is justified. A live guest-agent +signal is sufficient healthy evidence for that decision even before disk or +network enrichment finishes, and the preserved result must be recorded with an +explicit snapshot note so diagnostics can distinguish deliberate stabilization +from ordinary fallback. That compatibility boundary also applies to historical snapshot labels that may still exist in tests, live in-memory state, or pre-canonical diagnostic paths: legacy aliases such as `rrd-available`, `rrd-data`, `node-status-available`, diff --git a/frontend-modern/src/components/shared/InteractiveSparkline.tsx b/frontend-modern/src/components/shared/InteractiveSparkline.tsx index d14a07184..3879d2c4d 100644 --- a/frontend-modern/src/components/shared/InteractiveSparkline.tsx +++ b/frontend-modern/src/components/shared/InteractiveSparkline.tsx @@ -211,17 +211,15 @@ export const InteractiveSparkline: Component = (props {(hover) => (
-
+
{formatInteractiveSparklineHoverTime(hover().timestamp)}
@@ -240,20 +238,20 @@ export const InteractiveSparkline: Component = (props props.highlightNearestSeriesOnHover && hover().focusedTooltip && hover().highlightedSeriesIndex === entry.seriesIndex - ? { 'background-color': 'rgba(255,255,255,0.1)' } + ? { 'background-color': 'rgba(148, 163, 184, 0.14)' } : {} } > - {entry.name} - + {entry.name} + {sparkline.formatValue(entry.value)}
)} hover().values.length}> -
+
+{hover().totalValues - hover().values.length} more series
diff --git a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts index c28949bec..28ad27ede 100644 --- a/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts +++ b/frontend-modern/src/components/shared/SharedPrimitives.guardrails.test.ts @@ -52,6 +52,7 @@ import toggleModelSource from '@/components/shared/toggleModel.ts?raw'; import searchTipsPopoverSource from '@/components/shared/SearchTipsPopover.tsx?raw'; import searchTipsPopoverModelSource from '@/components/shared/searchTipsPopoverModel.ts?raw'; import tooltipSource from '@/components/shared/Tooltip.tsx?raw'; +import tooltipPortalSource from '@/components/shared/TooltipPortal.tsx?raw'; import tooltipModelSource from '@/components/shared/tooltipModel.ts?raw'; import trialBannerSource from '@/components/shared/TrialBanner.tsx?raw'; import trialBannerModelSource from '@/components/shared/trialBannerModel.ts?raw'; @@ -221,6 +222,19 @@ describe('shared primitive guardrails', () => { expect(commandPaletteModelSource).toContain('filterCommandPaletteCommands'); }); + it('keeps shared tooltip shells on semantic theme tokens', () => { + expect(tooltipSource).toContain('bg-surface text-base-content border-border'); + expect(tooltipSource).not.toContain("'background-color': 'rgb(15, 23, 42)'"); + expect(tooltipPortalSource).toContain('bg-surface'); + expect(tooltipPortalSource).toContain('text-base-content'); + expect(tooltipPortalSource).toContain('border-border'); + expect(tooltipPortalSource).not.toContain("'background-color': 'rgb(15, 23, 42)'"); + expect(interactiveSparklineSource).toContain('bg-surface'); + expect(interactiveSparklineSource).toContain('text-base-content'); + expect(interactiveSparklineSource).toContain('border-border'); + expect(interactiveSparklineSource).not.toContain("'background-color': 'rgb(15, 23, 42)'"); + }); + it('keeps active use trial nudge on shell, runtime, and model owners', () => { expect(activeUseTrialNudgeSource).toContain('useActiveUseTrialNudgeState'); expect(activeUseTrialNudgeSource).toContain('ACTIVE_USE_TRIAL_NUDGE_TITLE'); diff --git a/frontend-modern/src/components/shared/TooltipPortal.tsx b/frontend-modern/src/components/shared/TooltipPortal.tsx index c566b234e..7382123ee 100644 --- a/frontend-modern/src/components/shared/TooltipPortal.tsx +++ b/frontend-modern/src/components/shared/TooltipPortal.tsx @@ -26,8 +26,7 @@ export function TooltipPortal(props: TooltipPortalProps) { }} >
{props.children}
diff --git a/internal/monitoring/canonical_guardrails_test.go b/internal/monitoring/canonical_guardrails_test.go index ec9eb45ad..921fc9736 100644 --- a/internal/monitoring/canonical_guardrails_test.go +++ b/internal/monitoring/canonical_guardrails_test.go @@ -151,6 +151,43 @@ func TestProxmoxGuestMemoryFallbackUsesInstanceScopedCachesAndAgentMeminfo(t *te } } +func TestProxmoxGuestMemoryCarryForwardUsesCanonicalSnapshotStability(t *testing.T) { + requiredSnippets := map[string][]string{ + "guest_memory_stability.go": { + "func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {", + "func stabilizeGuestLowTrustMemory(", + "switch CanonicalMemorySource(source) {", + `"preserved-previous-memory-after-repeated-low-trust-pattern"`, + `"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"`, + }, + "monitor_pve_guest_builders.go": { + `prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)`, + "state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(", + }, + "monitor_pve_guest_poll.go": { + "Notes: snapshotNotes,", + }, + "monitor_polling_vm.go": { + `prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", n.Node, vm.VMID)`, + "memUsed, memorySource, snapshotNotes := stabilizeGuestLowTrustMemory(", + "Notes: snapshotNotes,", + }, + } + + for file, snippets := range requiredSnippets { + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("failed to read %s: %v", file, err) + } + source := string(data) + for _, snippet := range snippets { + if !strings.Contains(source, snippet) { + t.Fatalf("%s must contain %q", file, snippet) + } + } + } +} + func TestMonitoringRuntimeAvoidsLegacyMockPartialHelpers(t *testing.T) { forbiddenSnippets := []string{ "mock.GetMockState(", diff --git a/internal/monitoring/diagnostic_snapshots_test.go b/internal/monitoring/diagnostic_snapshots_test.go index af8049528..e7ab36a19 100644 --- a/internal/monitoring/diagnostic_snapshots_test.go +++ b/internal/monitoring/diagnostic_snapshots_test.go @@ -1146,6 +1146,36 @@ func TestGetDiagnosticSnapshots(t *testing.T) { } }) + t.Run("guest carry-forward notes remain visible", func(t *testing.T) { + m := &Monitor{ + nodeSnapshots: make(map[string]NodeMemorySnapshot), + guestSnapshots: make(map[string]GuestMemorySnapshot), + } + + m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{ + Instance: "pve1", + Node: "node1", + GuestType: "qemu", + VMID: 100, + MemorySource: "previous-snapshot", + Notes: []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"}, + }) + + result := m.GetDiagnosticSnapshots() + if len(result.Guests) != 1 { + t.Fatalf("Guests length = %d, want 1", len(result.Guests)) + } + if result.Guests[0].MemorySource != "previous-snapshot" { + t.Fatalf("MemorySource = %q, want previous-snapshot", result.Guests[0].MemorySource) + } + if result.Guests[0].FallbackReason != "preserved-previous-snapshot" { + t.Fatalf("FallbackReason = %q, want preserved-previous-snapshot", result.Guests[0].FallbackReason) + } + if len(result.Guests[0].Notes) != 1 || result.Guests[0].Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" { + t.Fatalf("Notes = %#v, want carry-forward note", result.Guests[0].Notes) + } + }) + t.Run("empty Monitor returns empty set", func(t *testing.T) { m := &Monitor{ nodeSnapshots: make(map[string]NodeMemorySnapshot), diff --git a/internal/monitoring/guest_memory_stability.go b/internal/monitoring/guest_memory_stability.go new file mode 100644 index 000000000..db1d4e637 --- /dev/null +++ b/internal/monitoring/guest_memory_stability.go @@ -0,0 +1,150 @@ +package monitoring + +import ( + "math" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +const ( + guestMemoryCarryForwardMaxAge = 2 * time.Minute + guestMemoryHealthyGuestMaxAge = 10 * time.Minute + guestMemoryCarryForwardMinUsageDelta = 5.0 + guestMemoryReliabilityLow = 0 + guestMemoryReliabilityFallback = 1 + guestMemoryReliabilityTrusted = 2 +) + +func guestMemorySourceReliability(source string) int { + switch CanonicalMemorySource(source) { + case "available-field", "derived-free-buffers-cached", + "guest-agent-meminfo", "rrd-memavailable", "rrd-memused", "agent": + return guestMemoryReliabilityTrusted + case "derived-total-minus-used", "previous-snapshot": + return guestMemoryReliabilityFallback + case "unknown", "cluster-resources", "status-mem", "status-freemem", "status-unavailable": + return guestMemoryReliabilityLow + default: + return guestMemoryReliabilityFallback + } +} + +func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot { + if m == nil { + return nil + } + + key := makeGuestSnapshotKey(instance, guestType, node, vmid) + + m.diagMu.RLock() + snapshot, ok := m.guestSnapshots[key] + m.diagMu.RUnlock() + if !ok { + return nil + } + + normalized := normalizeGuestMemorySnapshot(snapshot) + return &normalized +} + +func guestAgentSignalsHealthy( + detailedStatus *proxmox.VMStatus, + diskFromAgent bool, + ipAddresses []string, + networkInterfaces []models.GuestNetworkInterface, + osName, osVersion, agentVersion string, +) bool { + if detailedStatus != nil && detailedStatus.Agent.Value > 0 { + return true + } + return diskFromAgent || + len(ipAddresses) > 0 || + len(networkInterfaces) > 0 || + osName != "" || + osVersion != "" || + agentVersion != "" +} + +func stabilizeGuestLowTrustMemory( + prev *GuestMemorySnapshot, + currentStatus string, + currentSource string, + currentTotal uint64, + currentUsed uint64, + now time.Time, + guestAgentHealthy bool, +) (uint64, string, []string) { + if shouldCarryForwardPreviousGuestMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now) { + return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-after-repeated-low-trust-pattern"} + } + if shouldCarryForwardHealthyGuestLowTrustMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now, guestAgentHealthy) { + return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"} + } + return currentUsed, currentSource, nil +} + +func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time) bool { + if prev == nil || currentStatus != "running" || prev.Status != "running" { + return false + } + if prev.Memory.Total <= 0 || prev.Memory.Used < 0 { + return false + } + if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryCarryForwardMaxAge { + return false + } + + prevReliability := guestMemorySourceReliability(prev.MemorySource) + currentReliability := guestMemorySourceReliability(currentSource) + if prevReliability < guestMemoryReliabilityTrusted || currentReliability > guestMemoryReliabilityFallback { + return false + } + if prevReliability <= currentReliability { + return false + } + if currentTotal > 0 && prev.Memory.Total > 0 && prev.Memory.Total != int64(currentTotal) { + return false + } + + currentUsage := safePercentage(float64(currentUsed), float64(currentTotal)) + if prev.Memory.Usage > 0 && math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta { + return false + } + + return true +} + +func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time, guestAgentHealthy bool) bool { + if prev == nil || !guestAgentHealthy || currentStatus != "running" || prev.Status != "running" { + return false + } + if prev.Memory.Total <= 0 || prev.Memory.Used < 0 { + return false + } + if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryHealthyGuestMaxAge { + return false + } + if currentTotal == 0 || prev.Memory.Total != int64(currentTotal) { + return false + } + if guestMemorySourceReliability(currentSource) != guestMemoryReliabilityLow { + return false + } + + currentUsage := safePercentage(float64(currentUsed), float64(currentTotal)) + if currentUsage < 99 { + return false + } + if prev.Memory.Usage >= 90 || math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta { + return false + } + + prevReliability := guestMemorySourceReliability(prev.MemorySource) + if CanonicalMemorySource(prev.MemorySource) != "previous-snapshot" && prevReliability < guestMemoryReliabilityTrusted { + return false + } + + return true +} diff --git a/internal/monitoring/guest_memory_stability_test.go b/internal/monitoring/guest_memory_stability_test.go new file mode 100644 index 000000000..6c5e9e129 --- /dev/null +++ b/internal/monitoring/guest_memory_stability_test.go @@ -0,0 +1,203 @@ +package monitoring + +import ( + "context" + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/models" + "github.com/rcourtman/pulse-go-rewrite/pkg/proxmox" +) + +func TestStabilizeGuestLowTrustMemoryCarriesForwardTrustedSnapshot(t *testing.T) { + const gib = uint64(1024 * 1024 * 1024) + + now := time.Now() + prev := &GuestMemorySnapshot{ + Status: "running", + RetrievedAt: now.Add(-time.Minute), + MemorySource: "rrd-memavailable", + Memory: models.Memory{ + Total: 8 * int64(gib), + Used: 3 * int64(gib), + Usage: safePercentage(float64(3*gib), float64(8*gib)), + }, + } + + used, source, notes := stabilizeGuestLowTrustMemory(prev, "running", "status-freemem", 8*gib, 8*gib, now, false) + if used != 3*gib { + t.Fatalf("used = %d, want %d", used, 3*gib) + } + if source != "previous-snapshot" { + t.Fatalf("source = %q, want previous-snapshot", source) + } + if len(notes) != 1 || notes[0] != "preserved-previous-memory-after-repeated-low-trust-pattern" { + t.Fatalf("notes = %#v, want repeated low-trust carry-forward note", notes) + } +} + +func TestStabilizeGuestLowTrustMemoryUsesHealthyGuestAgentEvidence(t *testing.T) { + const gib = uint64(1024 * 1024 * 1024) + + now := time.Now() + prev := &GuestMemorySnapshot{ + Status: "running", + RetrievedAt: now.Add(-2 * time.Minute), + MemorySource: "previous-snapshot", + Memory: models.Memory{ + Total: 8 * int64(gib), + Used: 3 * int64(gib), + Usage: safePercentage(float64(3*gib), float64(8*gib)), + }, + } + + used, source, notes := stabilizeGuestLowTrustMemory(prev, "running", "cluster-resources", 8*gib, 8*gib, now, true) + if used != 3*gib { + t.Fatalf("used = %d, want %d", used, 3*gib) + } + if source != "previous-snapshot" { + t.Fatalf("source = %q, want previous-snapshot", source) + } + if len(notes) != 1 || notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" { + t.Fatalf("notes = %#v, want healthy guest carry-forward note", notes) + } +} + +func TestHandleClusterVMResourcePreservesHealthyGuestMemoryFromPreviousSnapshot(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + + const gib = uint64(1024 * 1024 * 1024) + + mon := newTestPVEMonitor("test") + defer mon.alertManager.Stop() + defer mon.notificationMgr.Stop() + + previousSample := time.Now().Add(-time.Minute) + mon.recordGuestSnapshot("test", "qemu", "node1", 101, GuestMemorySnapshot{ + Name: "vm-101", + Status: "running", + RetrievedAt: previousSample, + MemorySource: "previous-snapshot", + Memory: models.Memory{ + Total: 8 * int64(gib), + Used: 3 * int64(gib), + Free: 5 * int64(gib), + Usage: safePercentage(float64(3*gib), float64(8*gib)), + }, + }) + + client := &vmMemoryTrustStubClient{ + stubPVEClient: &stubPVEClient{}, + vmStatus: &proxmox.VMStatus{ + Status: "running", + MaxMem: 8 * gib, + Mem: 8 * gib, + Agent: proxmox.VMAgentField{Value: 1}, + }, + } + + vm, ok := mon.handleClusterVMResource( + context.Background(), + "test", + proxmox.ClusterResource{ + ID: "qemu/101", + Type: "qemu", + Node: "node1", + Name: "vm-101", + Status: "running", + VMID: 101, + MaxMem: 8 * gib, + Mem: 8 * gib, + MaxCPU: 2, + }, + makeGuestID("test", "node1", 101), + client, + nil, + nil, + ) + if !ok { + t.Fatal("handleClusterVMResource() returned ok=false") + } + if got := uint64(vm.Memory.Used); got != 3*gib { + t.Fatalf("vm.Memory.Used = %d, want %d", got, 3*gib) + } + + key := makeGuestSnapshotKey("test", "qemu", "node1", 101) + snap, ok := mon.guestSnapshots[key] + if !ok { + t.Fatalf("expected guest snapshot %q to be recorded", key) + } + if snap.MemorySource != "previous-snapshot" { + t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource) + } + if len(snap.Notes) != 1 || snap.Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" { + t.Fatalf("snapshot.Notes = %#v, want healthy guest carry-forward note", snap.Notes) + } +} + +func TestPollVMsWithNodesPreservesHealthyGuestMemoryFromPreviousSnapshot(t *testing.T) { + t.Setenv("PULSE_DATA_DIR", t.TempDir()) + + const gib = uint64(1024 * 1024 * 1024) + + mon := newTestPVEMonitor("test") + defer mon.alertManager.Stop() + defer mon.notificationMgr.Stop() + + previousSample := time.Now().Add(-time.Minute) + mon.recordGuestSnapshot("test", "qemu", "node1", 101, GuestMemorySnapshot{ + Name: "vm-101", + Status: "running", + RetrievedAt: previousSample, + MemorySource: "previous-snapshot", + Memory: models.Memory{ + Total: 8 * int64(gib), + Used: 3 * int64(gib), + Free: 5 * int64(gib), + Usage: safePercentage(float64(3*gib), float64(8*gib)), + }, + }) + + client := &vmMemoryTrustStubClient{ + stubPVEClient: &stubPVEClient{}, + vms: []proxmox.VM{{ + VMID: 101, + Name: "vm-101", + Node: "node1", + Status: "running", + MaxMem: 8 * gib, + Mem: 8 * gib, + CPUs: 2, + }}, + vmStatus: &proxmox.VMStatus{ + Status: "running", + MaxMem: 8 * gib, + Mem: 8 * gib, + Agent: proxmox.VMAgentField{Value: 1}, + }, + } + + nodes := []proxmox.Node{{Node: "node1", Status: "online"}} + nodeEffectiveStatus := map[string]string{"node1": "online"} + mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus) + + vms := mon.state.GetSnapshot().VMs + if len(vms) != 1 { + t.Fatalf("expected 1 VM, got %d", len(vms)) + } + if got := uint64(vms[0].Memory.Used); got != 3*gib { + t.Fatalf("vms[0].Memory.Used = %d, want %d", got, 3*gib) + } + + key := makeGuestSnapshotKey("test", "qemu", "node1", 101) + snap, ok := mon.guestSnapshots[key] + if !ok { + t.Fatalf("expected guest snapshot %q to be recorded", key) + } + if snap.MemorySource != "previous-snapshot" { + t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource) + } + if len(snap.Notes) != 1 || snap.Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" { + t.Fatalf("snapshot.Notes = %#v, want healthy guest carry-forward note", snap.Notes) + } +} diff --git a/internal/monitoring/monitor_extra_coverage_test.go b/internal/monitoring/monitor_extra_coverage_test.go index ad3c805e2..6988a4228 100644 --- a/internal/monitoring/monitor_extra_coverage_test.go +++ b/internal/monitoring/monitor_extra_coverage_test.go @@ -1145,7 +1145,7 @@ func TestBuildVMFromClusterResource_ContinuesGuestAgentQueriesAfterTransientStat LastSeen: time.Now(), } - vm, _, _, _, ok := monitor.buildVMFromClusterResource( + vm, _, _, _, _, ok := monitor.buildVMFromClusterResource( context.Background(), "cluster-a", proxmox.ClusterResource{ @@ -1473,7 +1473,7 @@ func TestBuildVMFromClusterResource_UsesLinkedHostAgentDiskFallback(t *testing.T } guestID := makeGuestID("cluster-a", "node-a", 101) - vm, _, _, _, ok := monitor.buildVMFromClusterResource( + vm, _, _, _, _, ok := monitor.buildVMFromClusterResource( context.Background(), "cluster-a", proxmox.ClusterResource{ diff --git a/internal/monitoring/monitor_polling_vm.go b/internal/monitoring/monitor_polling_vm.go index 79823e23c..e3f816caa 100644 --- a/internal/monitoring/monitor_polling_vm.go +++ b/internal/monitoring/monitor_polling_vm.go @@ -496,6 +496,25 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu } } + prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", n.Node, vm.VMID) + memUsed, memorySource, snapshotNotes := stabilizeGuestLowTrustMemory( + prevSnapshot, + vm.Status, + memorySource, + memTotal, + memUsed, + sampleTime, + guestAgentSignalsHealthy( + vmStatus, + diskFromAgent, + ipAddresses, + networkInterfaces, + osName, + osVersion, + guestAgentVersion, + ), + ) + memTotalBytes := clampToInt64(memTotal) memUsedBytes := clampToInt64(memUsed) if memTotalBytes > 0 && memUsedBytes > memTotalBytes { @@ -577,6 +596,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu FallbackReason: guestMemoryFallbackReason(memorySource), Memory: modelVM.Memory, Raw: guestRaw, + Notes: snapshotNotes, }) // Check alerts diff --git a/internal/monitoring/monitor_proxmox_pool_test.go b/internal/monitoring/monitor_proxmox_pool_test.go index c6f5918fd..5e0ccca8c 100644 --- a/internal/monitoring/monitor_proxmox_pool_test.go +++ b/internal/monitoring/monitor_proxmox_pool_test.go @@ -10,7 +10,7 @@ import ( func TestBuildVMFromClusterResource_PreservesProxmoxPool(t *testing.T) { monitor := &Monitor{rateTracker: NewRateTracker()} - vm, _, _, _, ok := monitor.buildVMFromClusterResource( + vm, _, _, _, _, ok := monitor.buildVMFromClusterResource( context.Background(), "cluster-a", proxmox.ClusterResource{ diff --git a/internal/monitoring/monitor_pve_guest_builders.go b/internal/monitoring/monitor_pve_guest_builders.go index 9af6563ae..dee03134f 100644 --- a/internal/monitoring/monitor_pve_guest_builders.go +++ b/internal/monitoring/monitor_pve_guest_builders.go @@ -143,15 +143,16 @@ func (m *Monitor) buildVMFromClusterResource( guestID string, vmIDToHostAgent map[string]models.Host, prevVM *models.VM, -) (models.VM, VMMemoryRaw, string, time.Time, bool) { +) (models.VM, VMMemoryRaw, string, []string, time.Time, bool) { // Skip templates if configured if res.Template == 1 { - return models.VM{}, VMMemoryRaw{}, "", time.Time{}, false + return models.VM{}, VMMemoryRaw{}, "", nil, time.Time{}, false } if guestID == "" { guestID = makeGuestID(instanceName, res.Node, res.VMID) } + prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID) state := vmBuildState{ memTotal: res.MaxMem, @@ -255,6 +256,24 @@ func (m *Monitor) buildVMFromClusterResource( } sampleTime := time.Now() + var snapshotNotes []string + state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory( + prevSnapshot, + res.Status, + state.memorySource, + state.memTotal, + state.memUsed, + sampleTime, + guestAgentSignalsHealthy( + state.detailedStatus, + state.diskFromAgent, + state.ipAddresses, + state.networkInterfaces, + state.osName, + state.osVersion, + state.agentVersion, + ), + ) currentMetrics := IOMetrics{ DiskRead: state.diskReadBytes, DiskWrite: state.diskWriteBytes, @@ -331,7 +350,7 @@ func (m *Monitor) buildVMFromClusterResource( } } - return vm, state.guestRaw, state.memorySource, sampleTime, true + return vm, state.guestRaw, state.memorySource, snapshotNotes, sampleTime, true } type vmFSInfoSummary struct { diff --git a/internal/monitoring/monitor_pve_guest_poll.go b/internal/monitoring/monitor_pve_guest_poll.go index 4b1bdeed2..75fe1bd57 100644 --- a/internal/monitoring/monitor_pve_guest_poll.go +++ b/internal/monitoring/monitor_pve_guest_poll.go @@ -120,7 +120,7 @@ func (m *Monitor) handleClusterVMResource( prevVM *models.VM, vmIDToHostAgent map[string]models.Host, ) (models.VM, bool) { - vm, guestRaw, memorySource, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM) + vm, guestRaw, memorySource, snapshotNotes, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM) if !ok { return models.VM{}, false } @@ -138,6 +138,7 @@ func (m *Monitor) handleClusterVMResource( FallbackReason: guestMemoryFallbackReason(memorySource), Memory: vm.Memory, Raw: guestRaw, + Notes: snapshotNotes, }) m.checkGuestAlertsForVM(instanceName, vm)