Pulse/internal/monitoring/vm_memory_telemetry_guard_test.go
rcourtman c1fa084100
Some checks are pending
Build and Test / Secret Scan (push) Waiting to run
Build and Test / Frontend & Backend (push) Waiting to run
Helm CI / Lint and Render Chart (push) Waiting to run
Core E2E Tests / Playwright Core E2E (push) Waiting to run
Stabilize low-trust 100 percent VM memory across mixed sizes (#1319)
2026-03-27 11:50:51 +00:00

297 lines
12 KiB
Go

package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
)
func TestDetectRepeatedVMMemoryUsage(t *testing.T) {
tests := []struct {
name string
vms []models.VM
wantSuspicious bool
wantRepeated int
wantRunning int
wantRepeatedMem int64
}{
{
name: "detects suspicious repeated memory used values",
vms: []models.VM{
{Name: "vm1", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm2", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm3", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm4", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm5", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: 8 << 30, Used: 2 << 30}},
},
wantSuspicious: true,
wantRepeated: 4,
wantRunning: 5,
wantRepeatedMem: 3 << 30,
},
{
name: "ignores patterns under share threshold",
vms: []models.VM{
{Name: "vm1", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm2", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm3", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: 8 << 30, Used: 2 << 30}},
{Name: "vm4", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: 8 << 30, Used: 1 << 30}},
},
wantSuspicious: false,
wantRepeated: 0,
wantRunning: 4,
},
{
name: "ignores when not enough running qemu guests",
vms: []models.VM{
{Name: "vm1", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm2", Type: "qemu", Status: "running", MemorySource: "status-mem", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30}},
{Name: "vm3", Type: "qemu", Status: "stopped", MemorySource: "powered-off", Memory: models.Memory{Total: 8 << 30, Used: 0}},
},
wantSuspicious: false,
wantRepeated: 0,
wantRunning: 2,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := detectRepeatedVMMemoryUsage(tt.vms)
if got.suspicious != tt.wantSuspicious {
t.Fatalf("suspicious = %v, want %v", got.suspicious, tt.wantSuspicious)
}
if got.repeatedCount != tt.wantRepeated {
t.Fatalf("repeatedCount = %d, want %d", got.repeatedCount, tt.wantRepeated)
}
if got.runningCount != tt.wantRunning {
t.Fatalf("runningCount = %d, want %d", got.runningCount, tt.wantRunning)
}
if got.repeatedMemUsed != tt.wantRepeatedMem {
t.Fatalf("repeatedMemUsed = %d, want %d", got.repeatedMemUsed, tt.wantRepeatedMem)
}
})
}
}
func TestFilterVMsByInstance(t *testing.T) {
vms := []models.VM{
{ID: "a", Instance: "pve1"},
{ID: "b", Instance: "pve2"},
{ID: "c", Instance: "pve1"},
}
filtered := filterVMsByInstance(vms, "pve1")
if len(filtered) != 2 {
t.Fatalf("len(filtered) = %d, want 2", len(filtered))
}
if filtered[0].ID != "a" || filtered[1].ID != "c" {
t.Fatalf("unexpected filtered result: %#v", filtered)
}
}
func TestStabilizeSuspiciousRepeatedVMMemory(t *testing.T) {
now := time.Now()
total := int64(8 << 30)
current := []models.VM{
{
ID: "vm1",
Name: "vm1",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: total, Used: total, Usage: 100, Balloon: 2 << 30},
LastSeen: now,
},
{
ID: "vm2",
Name: "vm2",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: total, Used: total, Usage: 100},
LastSeen: now,
},
{
ID: "vm3",
Name: "vm3",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: total, Used: total, Usage: 100},
LastSeen: now,
},
{
ID: "vm4",
Name: "vm4",
Type: "qemu",
Status: "running",
MemorySource: "rrd-memavailable",
Memory: models.Memory{Total: total, Used: 2 << 30, Free: 6 << 30, Usage: 25},
LastSeen: now,
},
}
alertVMs := append([]models.VM(nil), current...)
snapshots := []GuestMemorySnapshot{
{Name: "vm1", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[0].Memory},
{Name: "vm2", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[1].Memory},
{Name: "vm3", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[2].Memory},
{Name: "vm4", Status: "running", RetrievedAt: now, MemorySource: "rrd-memavailable", Memory: current[3].Memory},
}
previous := []models.VM{
{
ID: "vm1",
Name: "vm1",
Type: "qemu",
Status: "running",
MemorySource: "guest-agent-meminfo",
Memory: models.Memory{Total: total, Used: 3 << 30, Free: 5 << 30, Usage: 37.5, Balloon: 1 << 30},
LastSeen: now,
},
{
ID: "vm2",
Name: "vm2",
Type: "qemu",
Status: "running",
MemorySource: "previous-snapshot",
Memory: models.Memory{Total: total, Used: 4 << 30, Free: 4 << 30, Usage: 50},
LastSeen: now,
},
{
ID: "vm3",
Name: "vm3",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: total, Used: 7 << 30, Free: 1 << 30, Usage: 87.5},
LastSeen: now,
},
}
applied := stabilizeSuspiciousRepeatedVMMemory(current, alertVMs, snapshots, previous, now)
if applied != 2 {
t.Fatalf("applied = %d, want 2", applied)
}
if current[0].MemorySource != "previous-snapshot" || current[0].Memory.Used != 3<<30 {
t.Fatalf("vm1 memory = %#v source=%q, want previous-snapshot with preserved used", current[0].Memory, current[0].MemorySource)
}
if current[0].Memory.Balloon != 2<<30 {
t.Fatalf("vm1 balloon = %d, want current balloon preserved", current[0].Memory.Balloon)
}
if current[1].MemorySource != "previous-snapshot" || current[1].Memory.Used != 4<<30 {
t.Fatalf("vm2 memory = %#v source=%q, want previous-snapshot with preserved used", current[1].Memory, current[1].MemorySource)
}
if current[2].Memory.Used != total || current[2].MemorySource != "status-mem" {
t.Fatalf("vm3 should remain low-trust current reading, got %#v source=%q", current[2].Memory, current[2].MemorySource)
}
if alertVMs[0].Memory.Used != current[0].Memory.Used || alertVMs[0].MemorySource != "previous-snapshot" {
t.Fatalf("alert VM not stabilized: %#v source=%q", alertVMs[0].Memory, alertVMs[0].MemorySource)
}
if snapshots[0].Memory.Used != current[0].Memory.Used || snapshots[0].MemorySource != "previous-snapshot" {
t.Fatalf("snapshot not stabilized: %#v source=%q", snapshots[0].Memory, snapshots[0].MemorySource)
}
if len(snapshots[0].Notes) != 1 || snapshots[0].Notes[0] != "preserved-previous-memory-after-repeated-low-trust-pattern" {
t.Fatalf("snapshot notes = %#v, want stabilization note", snapshots[0].Notes)
}
}
func TestStabilizeSuspiciousRepeatedVMMemory_IgnoresTrustedPattern(t *testing.T) {
now := time.Now()
total := int64(8 << 30)
current := []models.VM{
{ID: "vm1", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: total, Used: 3 << 30, Usage: 37.5}, LastSeen: now},
{ID: "vm2", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: total, Used: 3 << 30, Usage: 37.5}, LastSeen: now},
{ID: "vm3", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: total, Used: 3 << 30, Usage: 37.5}, LastSeen: now},
{ID: "vm4", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: total, Used: 2 << 30, Usage: 25}, LastSeen: now},
}
applied := stabilizeSuspiciousRepeatedVMMemory(current, nil, nil, current, now)
if applied != 0 {
t.Fatalf("applied = %d, want 0", applied)
}
if current[0].MemorySource != "rrd-memavailable" {
t.Fatalf("memory source changed unexpectedly to %q", current[0].MemorySource)
}
}
func TestStabilizeSuspiciousRepeatedVMMemory_PreservesLowTrustFullUsageAcrossDifferentTotals(t *testing.T) {
now := time.Now()
current := []models.VM{
{
ID: "vm1",
Name: "vm1",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: 8 << 30, Used: 8 << 30, Usage: 100},
LastSeen: now,
},
{
ID: "vm2",
Name: "vm2",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: 12 << 30, Used: 12 << 30, Usage: 100},
LastSeen: now,
},
{
ID: "vm3",
Name: "vm3",
Type: "qemu",
Status: "running",
MemorySource: "status-mem",
Memory: models.Memory{Total: 16 << 30, Used: 16 << 30, Usage: 100},
LastSeen: now,
},
{
ID: "vm4",
Name: "vm4",
Type: "qemu",
Status: "running",
MemorySource: "rrd-memavailable",
Memory: models.Memory{Total: 16 << 30, Used: 4 << 30, Free: 12 << 30, Usage: 25},
LastSeen: now,
},
}
alertVMs := append([]models.VM(nil), current...)
snapshots := []GuestMemorySnapshot{
{Name: "vm1", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[0].Memory, Raw: VMMemoryRaw{StatusMem: 8 << 30}},
{Name: "vm2", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[1].Memory, Raw: VMMemoryRaw{StatusMem: 12 << 30}},
{Name: "vm3", Status: "running", RetrievedAt: now, MemorySource: "status-mem", Memory: current[2].Memory, Raw: VMMemoryRaw{StatusMem: 16 << 30}},
{Name: "vm4", Status: "running", RetrievedAt: now, MemorySource: "rrd-memavailable", Memory: current[3].Memory},
}
previous := []models.VM{
{ID: "vm1", Name: "vm1", Type: "qemu", Status: "running", MemorySource: "guest-agent-meminfo", Memory: models.Memory{Total: 8 << 30, Used: 3 << 30, Free: 5 << 30, Usage: 37.5}, LastSeen: now},
{ID: "vm2", Name: "vm2", Type: "qemu", Status: "running", MemorySource: "rrd-memavailable", Memory: models.Memory{Total: 12 << 30, Used: 5 << 30, Free: 7 << 30, Usage: 41.6667}, LastSeen: now},
{ID: "vm3", Name: "vm3", Type: "qemu", Status: "running", MemorySource: "previous-snapshot", Memory: models.Memory{Total: 16 << 30, Used: 6 << 30, Free: 10 << 30, Usage: 37.5}, LastSeen: now},
}
applied := stabilizeSuspiciousRepeatedVMMemory(current, alertVMs, snapshots, previous, now)
if applied != 3 {
t.Fatalf("applied = %d, want 3", applied)
}
if current[0].MemorySource != "previous-snapshot" || current[0].Memory.Used != 3<<30 {
t.Fatalf("vm1 memory = %#v source=%q, want preserved trusted reading", current[0].Memory, current[0].MemorySource)
}
if current[1].MemorySource != "previous-snapshot" || current[1].Memory.Used != 5<<30 {
t.Fatalf("vm2 memory = %#v source=%q, want preserved trusted reading", current[1].Memory, current[1].MemorySource)
}
if current[2].MemorySource != "previous-snapshot" || current[2].Memory.Used != 6<<30 {
t.Fatalf("vm3 memory = %#v source=%q, want preserved previous snapshot", current[2].Memory, current[2].MemorySource)
}
if current[3].MemorySource != "rrd-memavailable" {
t.Fatalf("vm4 should stay trusted, got source=%q", current[3].MemorySource)
}
if !snapshotHasNote(snapshots[0].Notes, "preserved-previous-memory-after-repeated-low-trust-pattern") {
t.Fatalf("expected stabilization note on vm1 snapshot, got %#v", snapshots[0].Notes)
}
if alertVMs[1].Memory.Used != current[1].Memory.Used || alertVMs[1].MemorySource != "previous-snapshot" {
t.Fatalf("alert VM not stabilized: %#v source=%q", alertVMs[1].Memory, alertVMs[1].MemorySource)
}
}