mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-10 00:14:38 +00:00
Stabilize low-trust Proxmox VM memory readings
This commit is contained in:
rcourtman
parent
e4a89ac6c5
commit
6388bd8b97
14 changed files with 508 additions and 21 deletions
|
|
@ -230,8 +230,8 @@ connections` visible as the API-backed alternative for Proxmox and
|
|||
`frontend-modern/src/routing/resourceLinks.ts` instead of freezing raw
|
||||
route strings or provider-local link builders inside feature panels.
|
||||
15. Keep shared summary-card emphasis coherent. When shared summary primitives enter an `inactive` state, `SummaryMetricCard`, `InteractiveSparkline`, and `DensityMap` must all demote background context together so storage, infrastructure, and workloads read as one interaction model instead of mixing page-local opacity, sticky-shell, or highlight rules.
|
||||
16. Keep density-map summaries overview-first. When a shared summary density map receives row focus or chart-hover emphasis, `frontend-modern/src/components/shared/DensityMap.tsx`, `frontend-modern/src/components/shared/useDensityMapState.ts`, and `frontend-modern/src/components/shared/densityMapModel.ts` must preserve the multi-entity overview rows and keep focused-entity detail in the hover tooltip instead of swapping the card into a single-series chart, dimming the rest of the map into unusable background noise, duplicating cursor-value tooltip copy, or adding persistent card chrome that steals heatmap space. The card body must stay overview-first; the tooltip may carry the active entity identity, current value, peak, and mini sparkline.
|
||||
17. Keep sparkline scrubbing source-local and sibling-sync timestamp-based. The chart a user is actively scrubbing in `frontend-modern/src/components/shared/InteractiveSparkline.tsx` and `frontend-modern/src/components/shared/useInteractiveSparklineState.ts` must keep its dashed hover cursor on the real local mouse `x`, while sibling cards may map the shared hover timestamp onto their own timelines. Shared cursor sync must not snap the source chart back onto the nearest sample timestamp, the rendered SVG/canvas hover cursor must bind to the actual numeric cursor coordinate rather than a boolean guard state, the time cursor must span the chart viewport instead of collapsing to the series height, and the hover tooltip must track the pointer instead of anchoring to the chart top edge.
|
||||
16. Keep density-map summaries overview-first. When a shared summary density map receives row focus or chart-hover emphasis, `frontend-modern/src/components/shared/DensityMap.tsx`, `frontend-modern/src/components/shared/useDensityMapState.ts`, and `frontend-modern/src/components/shared/densityMapModel.ts` must preserve the multi-entity overview rows and keep focused-entity detail in the hover tooltip instead of swapping the card into a single-series chart, dimming the rest of the map into unusable background noise, duplicating cursor-value tooltip copy, or adding persistent card chrome that steals heatmap space. The card body must stay overview-first; the tooltip may carry the active entity identity, current value, peak, and mini sparkline, and shared tooltip shells must follow semantic surface tokens instead of forcing a dark palette in light mode.
|
||||
17. Keep sparkline scrubbing source-local and sibling-sync timestamp-based. The chart a user is actively scrubbing in `frontend-modern/src/components/shared/InteractiveSparkline.tsx` and `frontend-modern/src/components/shared/useInteractiveSparklineState.ts` must keep its dashed hover cursor on the real local mouse `x`, while sibling cards may map the shared hover timestamp onto their own timelines. Shared cursor sync must not snap the source chart back onto the nearest sample timestamp, the rendered SVG/canvas hover cursor must bind to the actual numeric cursor coordinate rather than a boolean guard state, the time cursor must span the chart viewport instead of collapsing to the series height, and the hover tooltip must track the pointer instead of anchoring to the chart top edge while following the active theme rather than a hardcoded dark shell.
|
||||
18. Keep shared contextual focus canonical after adoption. Once a summary or table surface enters route-backed contextual focus, future additions must extend `frontend-modern/src/components/shared/contextualFocus.ts` and its guardrail tests rather than forking another helper for workload IDs, resource IDs, or scroll-preserving same-route selection.
|
||||
19. Keep shared infrastructure/resource selectors on the canonical agent-facet
|
||||
truth. Shared primitives and settings-facing selector helpers must treat
|
||||
|
|
@ -854,7 +854,10 @@ owns tooltip positioning lifecycle, RAF scheduling, and singleton visibility
|
|||
state, and `frontend-modern/src/components/shared/tooltipModel.ts` owns tooltip
|
||||
sanitization plus viewport-clamped positioning math. Future tooltip work should
|
||||
extend those owners instead of pushing singleton state, DOM measurement, or
|
||||
sanitization logic back into the shared shell.
|
||||
sanitization logic back into the shared shell. Shared portal-mounted tooltip
|
||||
shells such as `frontend-modern/src/components/shared/TooltipPortal.tsx` must
|
||||
use the same semantic surface tokens as the canonical tooltip instead of
|
||||
introducing light-mode-inverted palettes.
|
||||
The shared collapsible search input now follows that same owner split.
|
||||
`frontend-modern/src/components/shared/CollapsibleSearchInput.tsx` stays the
|
||||
render shell, `frontend-modern/src/components/shared/useCollapsibleSearchInputState.ts`
|
||||
|
|
|
|||
|
|
@ -40,6 +40,11 @@ truth for live infrastructure data.
|
|||
16. `internal/monitoring/truenas_poller.go`
|
||||
17. `internal/monitoring/vmware_poller.go`
|
||||
18. `internal/dockeragent/swarm.go`
|
||||
19. `internal/monitoring/guest_memory_sources.go`
|
||||
20. `internal/monitoring/guest_memory_stability.go`
|
||||
21. `internal/monitoring/monitor_polling_vm.go`
|
||||
22. `internal/monitoring/monitor_pve_guest_builders.go`
|
||||
23. `internal/monitoring/monitor_pve_guest_poll.go`
|
||||
|
||||
## Shared Boundaries
|
||||
|
||||
|
|
@ -64,7 +69,7 @@ truth for live infrastructure data.
|
|||
|
||||
1. Update this contract when monitoring truth ownership changes
|
||||
2. Tighten guardrails when `GetState()`-centric paths are removed
|
||||
3. Keep discovery-provider, metrics-history, Docker Swarm collection, and container bootstrap proof routes explicit in `registry.json`
|
||||
3. Keep discovery-provider, guest-memory trust, metrics-history, Docker Swarm collection, and container bootstrap proof routes explicit in `registry.json`
|
||||
4. Update related read-state or monitor tests when new collector paths land
|
||||
5. Keep platform ingestion semantics aligned with
|
||||
`docs/release-control/v6/internal/PLATFORM_SUPPORT_MODEL.md`: hybrid is a
|
||||
|
|
@ -418,6 +423,14 @@ and guest-agent fallback caches must key on `(instance, node, vmid)` instead
|
|||
of raw `node/vmid`, so separate Proxmox instances cannot leak stale or foreign
|
||||
memory evidence into each other just because they reuse the same node name and
|
||||
VMID.
|
||||
That same guest-memory boundary also owns stabilization when Proxmox falls
|
||||
back to low-trust VM full-usage readings. The shared VM polling paths must use
|
||||
the previous guest diagnostic snapshot, not the resource model, to decide when
|
||||
one more `previous-snapshot` carry-forward is justified. A live guest-agent
|
||||
signal is sufficient healthy evidence for that decision even before disk or
|
||||
network enrichment finishes, and the preserved result must be recorded with an
|
||||
explicit snapshot note so diagnostics can distinguish deliberate stabilization
|
||||
from ordinary fallback.
|
||||
That compatibility boundary also applies to historical snapshot labels that may
|
||||
still exist in tests, live in-memory state, or pre-canonical diagnostic paths:
|
||||
legacy aliases such as `rrd-available`, `rrd-data`, `node-status-available`,
|
||||
|
|
|
|||
|
|
@ -211,17 +211,15 @@ export const InteractiveSparkline: Component<InteractiveSparklineProps> = (props
|
|||
{(hover) => (
|
||||
<div
|
||||
data-sparkline-tooltip="true"
|
||||
class="fixed pointer-events-none text-xs rounded px-2 py-1.5 shadow-lg border border-slate-600"
|
||||
class="fixed pointer-events-none rounded-md border border-border bg-surface px-2 py-1.5 text-xs text-base-content shadow-lg"
|
||||
style={{
|
||||
left: `${hover().tooltipX}px`,
|
||||
top: `${hover().tooltipY}px`,
|
||||
transform: 'translate(-50%, -100%)',
|
||||
'z-index': '9999',
|
||||
'background-color': 'rgb(15, 23, 42)',
|
||||
color: 'rgb(248, 250, 252)',
|
||||
}}
|
||||
>
|
||||
<div class="font-medium text-center mb-1">
|
||||
<div class="mb-1 text-center font-medium text-base-content">
|
||||
{formatInteractiveSparklineHoverTime(hover().timestamp)}
|
||||
</div>
|
||||
<For each={hover().values}>
|
||||
|
|
@ -240,20 +238,20 @@ export const InteractiveSparkline: Component<InteractiveSparklineProps> = (props
|
|||
props.highlightNearestSeriesOnHover &&
|
||||
hover().focusedTooltip &&
|
||||
hover().highlightedSeriesIndex === entry.seriesIndex
|
||||
? { 'background-color': 'rgba(255,255,255,0.1)' }
|
||||
? { 'background-color': 'rgba(148, 163, 184, 0.14)' }
|
||||
: {}
|
||||
}
|
||||
>
|
||||
<span class="w-1.5 h-1.5 rounded-full" style={{ background: entry.color }} />
|
||||
<span style={{ color: 'rgb(203, 213, 225)' }}>{entry.name}</span>
|
||||
<span class="ml-auto font-medium" style={{ color: 'rgb(248, 250, 252)' }}>
|
||||
<span class="text-muted">{entry.name}</span>
|
||||
<span class="ml-auto font-medium text-base-content">
|
||||
{sparkline.formatValue(entry.value)}
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
</For>
|
||||
<Show when={hover().totalValues > hover().values.length}>
|
||||
<div class="text-[10px] mt-0.5" style={{ color: 'rgb(148, 163, 184)' }}>
|
||||
<div class="mt-0.5 text-[10px] text-muted">
|
||||
+{hover().totalValues - hover().values.length} more series
|
||||
</div>
|
||||
</Show>
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ import toggleModelSource from '@/components/shared/toggleModel.ts?raw';
|
|||
import searchTipsPopoverSource from '@/components/shared/SearchTipsPopover.tsx?raw';
|
||||
import searchTipsPopoverModelSource from '@/components/shared/searchTipsPopoverModel.ts?raw';
|
||||
import tooltipSource from '@/components/shared/Tooltip.tsx?raw';
|
||||
import tooltipPortalSource from '@/components/shared/TooltipPortal.tsx?raw';
|
||||
import tooltipModelSource from '@/components/shared/tooltipModel.ts?raw';
|
||||
import trialBannerSource from '@/components/shared/TrialBanner.tsx?raw';
|
||||
import trialBannerModelSource from '@/components/shared/trialBannerModel.ts?raw';
|
||||
|
|
@ -221,6 +222,19 @@ describe('shared primitive guardrails', () => {
|
|||
expect(commandPaletteModelSource).toContain('filterCommandPaletteCommands');
|
||||
});
|
||||
|
||||
it('keeps shared tooltip shells on semantic theme tokens', () => {
|
||||
expect(tooltipSource).toContain('bg-surface text-base-content border-border');
|
||||
expect(tooltipSource).not.toContain("'background-color': 'rgb(15, 23, 42)'");
|
||||
expect(tooltipPortalSource).toContain('bg-surface');
|
||||
expect(tooltipPortalSource).toContain('text-base-content');
|
||||
expect(tooltipPortalSource).toContain('border-border');
|
||||
expect(tooltipPortalSource).not.toContain("'background-color': 'rgb(15, 23, 42)'");
|
||||
expect(interactiveSparklineSource).toContain('bg-surface');
|
||||
expect(interactiveSparklineSource).toContain('text-base-content');
|
||||
expect(interactiveSparklineSource).toContain('border-border');
|
||||
expect(interactiveSparklineSource).not.toContain("'background-color': 'rgb(15, 23, 42)'");
|
||||
});
|
||||
|
||||
it('keeps active use trial nudge on shell, runtime, and model owners', () => {
|
||||
expect(activeUseTrialNudgeSource).toContain('useActiveUseTrialNudgeState');
|
||||
expect(activeUseTrialNudgeSource).toContain('ACTIVE_USE_TRIAL_NUDGE_TITLE');
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@ export function TooltipPortal(props: TooltipPortalProps) {
|
|||
}}
|
||||
>
|
||||
<div
|
||||
class="text-[10px] rounded-md shadow-lg px-2 py-1.5 border border-slate-600"
|
||||
style={{ 'background-color': 'rgb(15, 23, 42)', color: 'rgb(248, 250, 252)' }}
|
||||
class="rounded-md border border-border bg-surface px-2 py-1.5 text-[10px] text-base-content shadow-lg"
|
||||
>
|
||||
{props.children}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -151,6 +151,43 @@ func TestProxmoxGuestMemoryFallbackUsesInstanceScopedCachesAndAgentMeminfo(t *te
|
|||
}
|
||||
}
|
||||
|
||||
func TestProxmoxGuestMemoryCarryForwardUsesCanonicalSnapshotStability(t *testing.T) {
|
||||
requiredSnippets := map[string][]string{
|
||||
"guest_memory_stability.go": {
|
||||
"func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {",
|
||||
"func stabilizeGuestLowTrustMemory(",
|
||||
"switch CanonicalMemorySource(source) {",
|
||||
`"preserved-previous-memory-after-repeated-low-trust-pattern"`,
|
||||
`"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"`,
|
||||
},
|
||||
"monitor_pve_guest_builders.go": {
|
||||
`prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)`,
|
||||
"state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(",
|
||||
},
|
||||
"monitor_pve_guest_poll.go": {
|
||||
"Notes: snapshotNotes,",
|
||||
},
|
||||
"monitor_polling_vm.go": {
|
||||
`prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", n.Node, vm.VMID)`,
|
||||
"memUsed, memorySource, snapshotNotes := stabilizeGuestLowTrustMemory(",
|
||||
"Notes: snapshotNotes,",
|
||||
},
|
||||
}
|
||||
|
||||
for file, snippets := range requiredSnippets {
|
||||
data, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read %s: %v", file, err)
|
||||
}
|
||||
source := string(data)
|
||||
for _, snippet := range snippets {
|
||||
if !strings.Contains(source, snippet) {
|
||||
t.Fatalf("%s must contain %q", file, snippet)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMonitoringRuntimeAvoidsLegacyMockPartialHelpers(t *testing.T) {
|
||||
forbiddenSnippets := []string{
|
||||
"mock.GetMockState(",
|
||||
|
|
|
|||
|
|
@ -1146,6 +1146,36 @@ func TestGetDiagnosticSnapshots(t *testing.T) {
|
|||
}
|
||||
})
|
||||
|
||||
t.Run("guest carry-forward notes remain visible", func(t *testing.T) {
|
||||
m := &Monitor{
|
||||
nodeSnapshots: make(map[string]NodeMemorySnapshot),
|
||||
guestSnapshots: make(map[string]GuestMemorySnapshot),
|
||||
}
|
||||
|
||||
m.recordGuestSnapshot("pve1", "qemu", "node1", 100, GuestMemorySnapshot{
|
||||
Instance: "pve1",
|
||||
Node: "node1",
|
||||
GuestType: "qemu",
|
||||
VMID: 100,
|
||||
MemorySource: "previous-snapshot",
|
||||
Notes: []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"},
|
||||
})
|
||||
|
||||
result := m.GetDiagnosticSnapshots()
|
||||
if len(result.Guests) != 1 {
|
||||
t.Fatalf("Guests length = %d, want 1", len(result.Guests))
|
||||
}
|
||||
if result.Guests[0].MemorySource != "previous-snapshot" {
|
||||
t.Fatalf("MemorySource = %q, want previous-snapshot", result.Guests[0].MemorySource)
|
||||
}
|
||||
if result.Guests[0].FallbackReason != "preserved-previous-snapshot" {
|
||||
t.Fatalf("FallbackReason = %q, want preserved-previous-snapshot", result.Guests[0].FallbackReason)
|
||||
}
|
||||
if len(result.Guests[0].Notes) != 1 || result.Guests[0].Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" {
|
||||
t.Fatalf("Notes = %#v, want carry-forward note", result.Guests[0].Notes)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty Monitor returns empty set", func(t *testing.T) {
|
||||
m := &Monitor{
|
||||
nodeSnapshots: make(map[string]NodeMemorySnapshot),
|
||||
|
|
|
|||
150
internal/monitoring/guest_memory_stability.go
Normal file
150
internal/monitoring/guest_memory_stability.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
package monitoring
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
)
|
||||
|
||||
const (
|
||||
guestMemoryCarryForwardMaxAge = 2 * time.Minute
|
||||
guestMemoryHealthyGuestMaxAge = 10 * time.Minute
|
||||
guestMemoryCarryForwardMinUsageDelta = 5.0
|
||||
guestMemoryReliabilityLow = 0
|
||||
guestMemoryReliabilityFallback = 1
|
||||
guestMemoryReliabilityTrusted = 2
|
||||
)
|
||||
|
||||
func guestMemorySourceReliability(source string) int {
|
||||
switch CanonicalMemorySource(source) {
|
||||
case "available-field", "derived-free-buffers-cached",
|
||||
"guest-agent-meminfo", "rrd-memavailable", "rrd-memused", "agent":
|
||||
return guestMemoryReliabilityTrusted
|
||||
case "derived-total-minus-used", "previous-snapshot":
|
||||
return guestMemoryReliabilityFallback
|
||||
case "unknown", "cluster-resources", "status-mem", "status-freemem", "status-unavailable":
|
||||
return guestMemoryReliabilityLow
|
||||
default:
|
||||
return guestMemoryReliabilityFallback
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Monitor) previousGuestSnapshot(instance, guestType, node string, vmid int) *GuestMemorySnapshot {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
key := makeGuestSnapshotKey(instance, guestType, node, vmid)
|
||||
|
||||
m.diagMu.RLock()
|
||||
snapshot, ok := m.guestSnapshots[key]
|
||||
m.diagMu.RUnlock()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
normalized := normalizeGuestMemorySnapshot(snapshot)
|
||||
return &normalized
|
||||
}
|
||||
|
||||
func guestAgentSignalsHealthy(
|
||||
detailedStatus *proxmox.VMStatus,
|
||||
diskFromAgent bool,
|
||||
ipAddresses []string,
|
||||
networkInterfaces []models.GuestNetworkInterface,
|
||||
osName, osVersion, agentVersion string,
|
||||
) bool {
|
||||
if detailedStatus != nil && detailedStatus.Agent.Value > 0 {
|
||||
return true
|
||||
}
|
||||
return diskFromAgent ||
|
||||
len(ipAddresses) > 0 ||
|
||||
len(networkInterfaces) > 0 ||
|
||||
osName != "" ||
|
||||
osVersion != "" ||
|
||||
agentVersion != ""
|
||||
}
|
||||
|
||||
func stabilizeGuestLowTrustMemory(
|
||||
prev *GuestMemorySnapshot,
|
||||
currentStatus string,
|
||||
currentSource string,
|
||||
currentTotal uint64,
|
||||
currentUsed uint64,
|
||||
now time.Time,
|
||||
guestAgentHealthy bool,
|
||||
) (uint64, string, []string) {
|
||||
if shouldCarryForwardPreviousGuestMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now) {
|
||||
return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-after-repeated-low-trust-pattern"}
|
||||
}
|
||||
if shouldCarryForwardHealthyGuestLowTrustMemory(prev, currentStatus, currentSource, currentTotal, currentUsed, now, guestAgentHealthy) {
|
||||
return uint64(prev.Memory.Used), "previous-snapshot", []string{"preserved-previous-memory-for-healthy-guest-low-trust-full-usage"}
|
||||
}
|
||||
return currentUsed, currentSource, nil
|
||||
}
|
||||
|
||||
func shouldCarryForwardPreviousGuestMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time) bool {
|
||||
if prev == nil || currentStatus != "running" || prev.Status != "running" {
|
||||
return false
|
||||
}
|
||||
if prev.Memory.Total <= 0 || prev.Memory.Used < 0 {
|
||||
return false
|
||||
}
|
||||
if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryCarryForwardMaxAge {
|
||||
return false
|
||||
}
|
||||
|
||||
prevReliability := guestMemorySourceReliability(prev.MemorySource)
|
||||
currentReliability := guestMemorySourceReliability(currentSource)
|
||||
if prevReliability < guestMemoryReliabilityTrusted || currentReliability > guestMemoryReliabilityFallback {
|
||||
return false
|
||||
}
|
||||
if prevReliability <= currentReliability {
|
||||
return false
|
||||
}
|
||||
if currentTotal > 0 && prev.Memory.Total > 0 && prev.Memory.Total != int64(currentTotal) {
|
||||
return false
|
||||
}
|
||||
|
||||
currentUsage := safePercentage(float64(currentUsed), float64(currentTotal))
|
||||
if prev.Memory.Usage > 0 && math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func shouldCarryForwardHealthyGuestLowTrustMemory(prev *GuestMemorySnapshot, currentStatus, currentSource string, currentTotal, currentUsed uint64, now time.Time, guestAgentHealthy bool) bool {
|
||||
if prev == nil || !guestAgentHealthy || currentStatus != "running" || prev.Status != "running" {
|
||||
return false
|
||||
}
|
||||
if prev.Memory.Total <= 0 || prev.Memory.Used < 0 {
|
||||
return false
|
||||
}
|
||||
if prev.RetrievedAt.IsZero() || now.Sub(prev.RetrievedAt) > guestMemoryHealthyGuestMaxAge {
|
||||
return false
|
||||
}
|
||||
if currentTotal == 0 || prev.Memory.Total != int64(currentTotal) {
|
||||
return false
|
||||
}
|
||||
if guestMemorySourceReliability(currentSource) != guestMemoryReliabilityLow {
|
||||
return false
|
||||
}
|
||||
|
||||
currentUsage := safePercentage(float64(currentUsed), float64(currentTotal))
|
||||
if currentUsage < 99 {
|
||||
return false
|
||||
}
|
||||
if prev.Memory.Usage >= 90 || math.Abs(prev.Memory.Usage-currentUsage) < guestMemoryCarryForwardMinUsageDelta {
|
||||
return false
|
||||
}
|
||||
|
||||
prevReliability := guestMemorySourceReliability(prev.MemorySource)
|
||||
if CanonicalMemorySource(prev.MemorySource) != "previous-snapshot" && prevReliability < guestMemoryReliabilityTrusted {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
203
internal/monitoring/guest_memory_stability_test.go
Normal file
203
internal/monitoring/guest_memory_stability_test.go
Normal file
|
|
@ -0,0 +1,203 @@
|
|||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
||||
)
|
||||
|
||||
func TestStabilizeGuestLowTrustMemoryCarriesForwardTrustedSnapshot(t *testing.T) {
|
||||
const gib = uint64(1024 * 1024 * 1024)
|
||||
|
||||
now := time.Now()
|
||||
prev := &GuestMemorySnapshot{
|
||||
Status: "running",
|
||||
RetrievedAt: now.Add(-time.Minute),
|
||||
MemorySource: "rrd-memavailable",
|
||||
Memory: models.Memory{
|
||||
Total: 8 * int64(gib),
|
||||
Used: 3 * int64(gib),
|
||||
Usage: safePercentage(float64(3*gib), float64(8*gib)),
|
||||
},
|
||||
}
|
||||
|
||||
used, source, notes := stabilizeGuestLowTrustMemory(prev, "running", "status-freemem", 8*gib, 8*gib, now, false)
|
||||
if used != 3*gib {
|
||||
t.Fatalf("used = %d, want %d", used, 3*gib)
|
||||
}
|
||||
if source != "previous-snapshot" {
|
||||
t.Fatalf("source = %q, want previous-snapshot", source)
|
||||
}
|
||||
if len(notes) != 1 || notes[0] != "preserved-previous-memory-after-repeated-low-trust-pattern" {
|
||||
t.Fatalf("notes = %#v, want repeated low-trust carry-forward note", notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStabilizeGuestLowTrustMemoryUsesHealthyGuestAgentEvidence(t *testing.T) {
|
||||
const gib = uint64(1024 * 1024 * 1024)
|
||||
|
||||
now := time.Now()
|
||||
prev := &GuestMemorySnapshot{
|
||||
Status: "running",
|
||||
RetrievedAt: now.Add(-2 * time.Minute),
|
||||
MemorySource: "previous-snapshot",
|
||||
Memory: models.Memory{
|
||||
Total: 8 * int64(gib),
|
||||
Used: 3 * int64(gib),
|
||||
Usage: safePercentage(float64(3*gib), float64(8*gib)),
|
||||
},
|
||||
}
|
||||
|
||||
used, source, notes := stabilizeGuestLowTrustMemory(prev, "running", "cluster-resources", 8*gib, 8*gib, now, true)
|
||||
if used != 3*gib {
|
||||
t.Fatalf("used = %d, want %d", used, 3*gib)
|
||||
}
|
||||
if source != "previous-snapshot" {
|
||||
t.Fatalf("source = %q, want previous-snapshot", source)
|
||||
}
|
||||
if len(notes) != 1 || notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" {
|
||||
t.Fatalf("notes = %#v, want healthy guest carry-forward note", notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleClusterVMResourcePreservesHealthyGuestMemoryFromPreviousSnapshot(t *testing.T) {
|
||||
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
||||
|
||||
const gib = uint64(1024 * 1024 * 1024)
|
||||
|
||||
mon := newTestPVEMonitor("test")
|
||||
defer mon.alertManager.Stop()
|
||||
defer mon.notificationMgr.Stop()
|
||||
|
||||
previousSample := time.Now().Add(-time.Minute)
|
||||
mon.recordGuestSnapshot("test", "qemu", "node1", 101, GuestMemorySnapshot{
|
||||
Name: "vm-101",
|
||||
Status: "running",
|
||||
RetrievedAt: previousSample,
|
||||
MemorySource: "previous-snapshot",
|
||||
Memory: models.Memory{
|
||||
Total: 8 * int64(gib),
|
||||
Used: 3 * int64(gib),
|
||||
Free: 5 * int64(gib),
|
||||
Usage: safePercentage(float64(3*gib), float64(8*gib)),
|
||||
},
|
||||
})
|
||||
|
||||
client := &vmMemoryTrustStubClient{
|
||||
stubPVEClient: &stubPVEClient{},
|
||||
vmStatus: &proxmox.VMStatus{
|
||||
Status: "running",
|
||||
MaxMem: 8 * gib,
|
||||
Mem: 8 * gib,
|
||||
Agent: proxmox.VMAgentField{Value: 1},
|
||||
},
|
||||
}
|
||||
|
||||
vm, ok := mon.handleClusterVMResource(
|
||||
context.Background(),
|
||||
"test",
|
||||
proxmox.ClusterResource{
|
||||
ID: "qemu/101",
|
||||
Type: "qemu",
|
||||
Node: "node1",
|
||||
Name: "vm-101",
|
||||
Status: "running",
|
||||
VMID: 101,
|
||||
MaxMem: 8 * gib,
|
||||
Mem: 8 * gib,
|
||||
MaxCPU: 2,
|
||||
},
|
||||
makeGuestID("test", "node1", 101),
|
||||
client,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
if !ok {
|
||||
t.Fatal("handleClusterVMResource() returned ok=false")
|
||||
}
|
||||
if got := uint64(vm.Memory.Used); got != 3*gib {
|
||||
t.Fatalf("vm.Memory.Used = %d, want %d", got, 3*gib)
|
||||
}
|
||||
|
||||
key := makeGuestSnapshotKey("test", "qemu", "node1", 101)
|
||||
snap, ok := mon.guestSnapshots[key]
|
||||
if !ok {
|
||||
t.Fatalf("expected guest snapshot %q to be recorded", key)
|
||||
}
|
||||
if snap.MemorySource != "previous-snapshot" {
|
||||
t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource)
|
||||
}
|
||||
if len(snap.Notes) != 1 || snap.Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" {
|
||||
t.Fatalf("snapshot.Notes = %#v, want healthy guest carry-forward note", snap.Notes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollVMsWithNodesPreservesHealthyGuestMemoryFromPreviousSnapshot(t *testing.T) {
|
||||
t.Setenv("PULSE_DATA_DIR", t.TempDir())
|
||||
|
||||
const gib = uint64(1024 * 1024 * 1024)
|
||||
|
||||
mon := newTestPVEMonitor("test")
|
||||
defer mon.alertManager.Stop()
|
||||
defer mon.notificationMgr.Stop()
|
||||
|
||||
previousSample := time.Now().Add(-time.Minute)
|
||||
mon.recordGuestSnapshot("test", "qemu", "node1", 101, GuestMemorySnapshot{
|
||||
Name: "vm-101",
|
||||
Status: "running",
|
||||
RetrievedAt: previousSample,
|
||||
MemorySource: "previous-snapshot",
|
||||
Memory: models.Memory{
|
||||
Total: 8 * int64(gib),
|
||||
Used: 3 * int64(gib),
|
||||
Free: 5 * int64(gib),
|
||||
Usage: safePercentage(float64(3*gib), float64(8*gib)),
|
||||
},
|
||||
})
|
||||
|
||||
client := &vmMemoryTrustStubClient{
|
||||
stubPVEClient: &stubPVEClient{},
|
||||
vms: []proxmox.VM{{
|
||||
VMID: 101,
|
||||
Name: "vm-101",
|
||||
Node: "node1",
|
||||
Status: "running",
|
||||
MaxMem: 8 * gib,
|
||||
Mem: 8 * gib,
|
||||
CPUs: 2,
|
||||
}},
|
||||
vmStatus: &proxmox.VMStatus{
|
||||
Status: "running",
|
||||
MaxMem: 8 * gib,
|
||||
Mem: 8 * gib,
|
||||
Agent: proxmox.VMAgentField{Value: 1},
|
||||
},
|
||||
}
|
||||
|
||||
nodes := []proxmox.Node{{Node: "node1", Status: "online"}}
|
||||
nodeEffectiveStatus := map[string]string{"node1": "online"}
|
||||
mon.pollVMsWithNodes(context.Background(), "test", "", false, client, nodes, nodeEffectiveStatus)
|
||||
|
||||
vms := mon.state.GetSnapshot().VMs
|
||||
if len(vms) != 1 {
|
||||
t.Fatalf("expected 1 VM, got %d", len(vms))
|
||||
}
|
||||
if got := uint64(vms[0].Memory.Used); got != 3*gib {
|
||||
t.Fatalf("vms[0].Memory.Used = %d, want %d", got, 3*gib)
|
||||
}
|
||||
|
||||
key := makeGuestSnapshotKey("test", "qemu", "node1", 101)
|
||||
snap, ok := mon.guestSnapshots[key]
|
||||
if !ok {
|
||||
t.Fatalf("expected guest snapshot %q to be recorded", key)
|
||||
}
|
||||
if snap.MemorySource != "previous-snapshot" {
|
||||
t.Fatalf("snapshot.MemorySource = %q, want previous-snapshot", snap.MemorySource)
|
||||
}
|
||||
if len(snap.Notes) != 1 || snap.Notes[0] != "preserved-previous-memory-for-healthy-guest-low-trust-full-usage" {
|
||||
t.Fatalf("snapshot.Notes = %#v, want healthy guest carry-forward note", snap.Notes)
|
||||
}
|
||||
}
|
||||
|
|
@ -1145,7 +1145,7 @@ func TestBuildVMFromClusterResource_ContinuesGuestAgentQueriesAfterTransientStat
|
|||
LastSeen: time.Now(),
|
||||
}
|
||||
|
||||
vm, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
vm, _, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
context.Background(),
|
||||
"cluster-a",
|
||||
proxmox.ClusterResource{
|
||||
|
|
@ -1473,7 +1473,7 @@ func TestBuildVMFromClusterResource_UsesLinkedHostAgentDiskFallback(t *testing.T
|
|||
}
|
||||
guestID := makeGuestID("cluster-a", "node-a", 101)
|
||||
|
||||
vm, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
vm, _, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
context.Background(),
|
||||
"cluster-a",
|
||||
proxmox.ClusterResource{
|
||||
|
|
|
|||
|
|
@ -496,6 +496,25 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
|
|||
}
|
||||
}
|
||||
|
||||
prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", n.Node, vm.VMID)
|
||||
memUsed, memorySource, snapshotNotes := stabilizeGuestLowTrustMemory(
|
||||
prevSnapshot,
|
||||
vm.Status,
|
||||
memorySource,
|
||||
memTotal,
|
||||
memUsed,
|
||||
sampleTime,
|
||||
guestAgentSignalsHealthy(
|
||||
vmStatus,
|
||||
diskFromAgent,
|
||||
ipAddresses,
|
||||
networkInterfaces,
|
||||
osName,
|
||||
osVersion,
|
||||
guestAgentVersion,
|
||||
),
|
||||
)
|
||||
|
||||
memTotalBytes := clampToInt64(memTotal)
|
||||
memUsedBytes := clampToInt64(memUsed)
|
||||
if memTotalBytes > 0 && memUsedBytes > memTotalBytes {
|
||||
|
|
@ -577,6 +596,7 @@ func (m *Monitor) pollVMsWithNodes(ctx context.Context, instanceName string, clu
|
|||
FallbackReason: guestMemoryFallbackReason(memorySource),
|
||||
Memory: modelVM.Memory,
|
||||
Raw: guestRaw,
|
||||
Notes: snapshotNotes,
|
||||
})
|
||||
|
||||
// Check alerts
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import (
|
|||
func TestBuildVMFromClusterResource_PreservesProxmoxPool(t *testing.T) {
|
||||
monitor := &Monitor{rateTracker: NewRateTracker()}
|
||||
|
||||
vm, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
vm, _, _, _, _, ok := monitor.buildVMFromClusterResource(
|
||||
context.Background(),
|
||||
"cluster-a",
|
||||
proxmox.ClusterResource{
|
||||
|
|
|
|||
|
|
@ -143,15 +143,16 @@ func (m *Monitor) buildVMFromClusterResource(
|
|||
guestID string,
|
||||
vmIDToHostAgent map[string]models.Host,
|
||||
prevVM *models.VM,
|
||||
) (models.VM, VMMemoryRaw, string, time.Time, bool) {
|
||||
) (models.VM, VMMemoryRaw, string, []string, time.Time, bool) {
|
||||
// Skip templates if configured
|
||||
if res.Template == 1 {
|
||||
return models.VM{}, VMMemoryRaw{}, "", time.Time{}, false
|
||||
return models.VM{}, VMMemoryRaw{}, "", nil, time.Time{}, false
|
||||
}
|
||||
|
||||
if guestID == "" {
|
||||
guestID = makeGuestID(instanceName, res.Node, res.VMID)
|
||||
}
|
||||
prevSnapshot := m.previousGuestSnapshot(instanceName, "qemu", res.Node, res.VMID)
|
||||
|
||||
state := vmBuildState{
|
||||
memTotal: res.MaxMem,
|
||||
|
|
@ -255,6 +256,24 @@ func (m *Monitor) buildVMFromClusterResource(
|
|||
}
|
||||
|
||||
sampleTime := time.Now()
|
||||
var snapshotNotes []string
|
||||
state.memUsed, state.memorySource, snapshotNotes = stabilizeGuestLowTrustMemory(
|
||||
prevSnapshot,
|
||||
res.Status,
|
||||
state.memorySource,
|
||||
state.memTotal,
|
||||
state.memUsed,
|
||||
sampleTime,
|
||||
guestAgentSignalsHealthy(
|
||||
state.detailedStatus,
|
||||
state.diskFromAgent,
|
||||
state.ipAddresses,
|
||||
state.networkInterfaces,
|
||||
state.osName,
|
||||
state.osVersion,
|
||||
state.agentVersion,
|
||||
),
|
||||
)
|
||||
currentMetrics := IOMetrics{
|
||||
DiskRead: state.diskReadBytes,
|
||||
DiskWrite: state.diskWriteBytes,
|
||||
|
|
@ -331,7 +350,7 @@ func (m *Monitor) buildVMFromClusterResource(
|
|||
}
|
||||
}
|
||||
|
||||
return vm, state.guestRaw, state.memorySource, sampleTime, true
|
||||
return vm, state.guestRaw, state.memorySource, snapshotNotes, sampleTime, true
|
||||
}
|
||||
|
||||
type vmFSInfoSummary struct {
|
||||
|
|
|
|||
|
|
@ -120,7 +120,7 @@ func (m *Monitor) handleClusterVMResource(
|
|||
prevVM *models.VM,
|
||||
vmIDToHostAgent map[string]models.Host,
|
||||
) (models.VM, bool) {
|
||||
vm, guestRaw, memorySource, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM)
|
||||
vm, guestRaw, memorySource, snapshotNotes, sampleTime, ok := m.buildVMFromClusterResource(ctx, instanceName, res, client, guestID, vmIDToHostAgent, prevVM)
|
||||
if !ok {
|
||||
return models.VM{}, false
|
||||
}
|
||||
|
|
@ -138,6 +138,7 @@ func (m *Monitor) handleClusterVMResource(
|
|||
FallbackReason: guestMemoryFallbackReason(memorySource),
|
||||
Memory: vm.Memory,
|
||||
Raw: guestRaw,
|
||||
Notes: snapshotNotes,
|
||||
})
|
||||
|
||||
m.checkGuestAlertsForVM(instanceName, vm)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue