diff --git a/docs/release-control/v6/internal/subsystems/monitoring.md b/docs/release-control/v6/internal/subsystems/monitoring.md index e2f31669be..adb023c2e4 100644 --- a/docs/release-control/v6/internal/subsystems/monitoring.md +++ b/docs/release-control/v6/internal/subsystems/monitoring.md @@ -1877,8 +1877,15 @@ being mistaken for an absent field. Node RRD fallback caches must key on `(instance, node)`, just as guest RRD and guest-agent caches key on `(instance, node, vmid)`, so identically named nodes in different Proxmox instances cannot exchange memory evidence. -Running LXC cluster-resource memory is cache-inclusive and therefore cannot be -used as live usage when both RRD `memavailable` and `memused` are absent. +Running LXC memory acknowledges a Proxmox API reality: guest RRD responses +carry only cache-inclusive `mem`/`maxmem` columns — the cache-aware +`memavailable`/`memused` columns exist only in node RRD — so cache-aware LXC +evidence is structurally absent from the guest RRD endpoint. When the +cache-aware RRD branches do not match, a running container with a non-zero +cluster-resource listing value must fall back to that cache-inclusive value +under the low-trust `cluster-resources` source rather than reporting the +guest unavailable; `unavailable` is reserved for running containers with no +listing evidence at all (#1634). Unified Linux and Docker agent ingest likewise must not repair a missing used value from total minus free alone; it may use an explicit used/percentage or complete free-plus-cache evidence. In every collector, known capacity with no diff --git a/internal/monitoring/memory_trust_characterization_test.go b/internal/monitoring/memory_trust_characterization_test.go index 1d54a76657..46cedc101d 100644 --- a/internal/monitoring/memory_trust_characterization_test.go +++ b/internal/monitoring/memory_trust_characterization_test.go @@ -1053,7 +1053,7 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) wantStatus: "running", }, { - name: "running LXC without RRD memory stays unknown", + name: "running LXC without RRD memory falls back to cluster resources (#1634)", res: proxmox.ClusterResource{ ID: "lxc/204", Type: "lxc", @@ -1065,6 +1065,22 @@ func TestHandleClusterContainerResourceMemoryTrustCharacterization(t *testing.T) Mem: 7 * gib, MaxCPU: 4, }, + wantSource: "cluster-resources", + wantUsed: 7 * gib, + wantStatus: "running", + }, + { + name: "running LXC without RRD memory or listing value stays unknown", + res: proxmox.ClusterResource{ + ID: "lxc/206", + Type: "lxc", + Node: "node1", + Name: "ct-206", + Status: "running", + VMID: 206, + MaxMem: 8 * gib, + MaxCPU: 4, + }, wantSource: "unavailable", wantStatus: "running", wantUnknown: true, diff --git a/internal/monitoring/monitor_pve_guest_lxc.go b/internal/monitoring/monitor_pve_guest_lxc.go index d9c531d228..feb4b0d7d0 100644 --- a/internal/monitoring/monitor_pve_guest_lxc.go +++ b/internal/monitoring/monitor_pve_guest_lxc.go @@ -79,7 +79,17 @@ func (m *Monitor) calculateLXCMemory( Str("instance", instanceName). Str("container", res.Name). Int("vmid", res.VMID). - Msg("RRD memory data unavailable for LXC; memory usage remains unavailable") + Msg("RRD memory data unavailable for LXC; falling back to cluster resources value") + } + + // PVE guest RRD carries only mem/maxmem columns (cgroup values, cache + // included) — the cache-aware memused/memavailable columns exist only + // in node RRD, so the branches above cannot match a real response. + // Without this fallback every running container reports unavailable + // memory (issue #1634). + if memorySource == "unavailable" && res.Mem > 0 { + memUsed = res.Mem + memorySource = "cluster-resources" } } diff --git a/internal/monitoring/monitor_pve_guest_lxc_test.go b/internal/monitoring/monitor_pve_guest_lxc_test.go index 764002ace9..7678d8f15f 100644 --- a/internal/monitoring/monitor_pve_guest_lxc_test.go +++ b/internal/monitoring/monitor_pve_guest_lxc_test.go @@ -175,3 +175,133 @@ func TestBuildContainerFromClusterResource_UsesContainerStatusCountersForRates(t t.Fatalf("expected NetworkOut rate from container status counters, got %d", container.NetworkOut) } } + +type stubPVEClientLXCRRD struct { + stubPVEClient + + lxcRRDPoints []proxmox.GuestRRDPoint + lxcRRDErr error +} + +func (s *stubPVEClientLXCRRD) GetLXCRRDData(ctx context.Context, node string, vmid int, timeframe, cf string, ds []string) ([]proxmox.GuestRRDPoint, error) { + return s.lxcRRDPoints, s.lxcRRDErr +} + +// Issue #1634: real PVE guest RRD responses carry only mem/maxmem, never the +// cache-aware memused/memavailable columns, so running containers must fall +// back to the cluster-resources listing value instead of reporting memory as +// unavailable (rendered as 0%). +func TestIssue1634LXCMemoryFallsBackToClusterResourcesOnRealRRDShape(t *testing.T) { + t.Parallel() + + maxMem := float64(8 * 1024 * 1024 * 1024) + client := &stubPVEClientLXCRRD{ + lxcRRDPoints: []proxmox.GuestRRDPoint{ + {Time: 1785164640, MaxMem: &maxMem}, + }, + } + + monitor := &Monitor{rateTracker: NewRateTracker()} + resource := proxmox.ClusterResource{ + Type: "lxc", + Node: "pve-a", + Name: "issue1634-ct", + Status: "running", + VMID: 108, + MaxMem: 8 * 1024 * 1024 * 1024, + Mem: 335716352, + } + + container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource( + context.Background(), + "cluster-a", + resource, + client, + map[int]bool{}, + ) + if !ok { + t.Fatal("expected container sample to be built") + } + if CanonicalMemorySource(memorySource) != "cluster-resources" { + t.Fatalf("memory source = %q, want cluster-resources fallback", memorySource) + } + if container.Memory.UsageUnavailable { + t.Fatal("running LXC memory marked unavailable despite listing value") + } + if container.Memory.Used != int64(resource.Mem) { + t.Fatalf("memory used = %d, want listing value %d", container.Memory.Used, resource.Mem) + } + if container.Memory.Usage <= 0 { + t.Fatalf("memory usage = %f, want > 0", container.Memory.Usage) + } + if !container.Memory.HasKnownUsage() { + t.Fatal("expected fallback memory to be usable for projections") + } +} + +func TestIssue1634LXCMemoryFallsBackWhenRRDErrors(t *testing.T) { + t.Parallel() + + client := &stubPVEClientLXCRRD{lxcRRDErr: context.DeadlineExceeded} + + monitor := &Monitor{rateTracker: NewRateTracker()} + resource := proxmox.ClusterResource{ + Type: "lxc", + Node: "pve-a", + Name: "issue1634-rrd-err", + Status: "running", + VMID: 109, + MaxMem: 512 * 1024 * 1024, + Mem: 113344512, + } + + container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource( + context.Background(), + "cluster-a", + resource, + client, + map[int]bool{}, + ) + if !ok { + t.Fatal("expected container sample to be built") + } + if CanonicalMemorySource(memorySource) != "cluster-resources" { + t.Fatalf("memory source = %q, want cluster-resources fallback", memorySource) + } + if container.Memory.Used != int64(resource.Mem) || container.Memory.UsageUnavailable { + t.Fatalf("memory = %+v, want listing fallback", container.Memory) + } +} + +func TestIssue1634LXCMemoryStaysUnavailableWithoutListingValue(t *testing.T) { + t.Parallel() + + client := &stubPVEClientLXCRRD{} + + monitor := &Monitor{rateTracker: NewRateTracker()} + resource := proxmox.ClusterResource{ + Type: "lxc", + Node: "pve-a", + Name: "issue1634-no-mem", + Status: "running", + VMID: 110, + MaxMem: 512 * 1024 * 1024, + } + + container, _, memorySource, _, ok := monitor.buildContainerFromClusterResource( + context.Background(), + "cluster-a", + resource, + client, + map[int]bool{}, + ) + if !ok { + t.Fatal("expected container sample to be built") + } + if CanonicalMemorySource(memorySource) != "unavailable" { + t.Fatalf("memory source = %q, want unavailable when no evidence exists", memorySource) + } + if !container.Memory.UsageUnavailable { + t.Fatal("expected memory to stay marked unavailable without any usage evidence") + } +}