fix(unifiedresources): carry real poll timestamps for storage and docker container sightings

resourceFromStorage and resourceFromDockerContainer stamped LastSeen with
time.Now() at conversion because their source models carried no poll
timestamp. The registry rebuilds from the retained state snapshot every
cycle, so those resources re-reported a fresh sighting each rebuild even
after their upstream source (PVE instance, docker host agent) stopped
delivering, and their per-source SourceStatus could never go stale via
markStaleLocked. 53faa4e46 fixed this fabrication at the ingest layer but
left these two adapter-level stamps.

- models.Storage gains LastSeen (omitzero), stamped where entries are
  built: the PVE storage poll (poll start time, including synthesized
  cluster-shared entries; preserved entries for unpolled nodes keep their
  old stamp), the PBS datastore conversion (PBS instance sighting), Ceph
  pool projection (cluster LastUpdated), and the mock generator (offline
  mock nodes get a backdated stamp so the stale path renders).
- resourceFromStorage passes storage.LastSeen through; zero stays zero
  ("never seen") instead of becoming conversion time.
- resourceFromDockerContainer uses host.LastSeen: containers are delivered
  wholesale with each host report, so the host report timestamp is the
  container sighting. This matches every other docker sub-resource adapter
  (services, tasks, volumes, networks, images already use host.LastSeen).
- ingestStorage routes PBS-poller datastore entries (instance "pbs-<name>",
  type pbs) to SourcePBS, parented to the PBS instance. Keying them
  SourceProxmox would judge their freshness against the 60s Proxmox stale
  threshold while PBS polls every 60s by default, flapping healthy
  datastores stale between polls; SourcePBS carries the cadence-matched
  120s threshold. PVE-reported pbs-typed storage.cfg backends stay
  SourceProxmox. Side effect: syncUnifiedStorageMetrics no longer skips
  PBS datastore storage, so those entries gain usage history.
- storageFromReadStateView round-trips LastSeen so the legacy storage API
  reports the honest sighting; mock refresh re-stamps available storage on
  each simulated poll.

The parent host/node staleness was already honest, so platform pages
reflected outages at the parent level; this makes the per-resource source
freshness honest too.
This commit is contained in:
rcourtman 2026-06-11 10:57:22 +01:00
parent 53faa4e46a
commit e2a036ce2e
10 changed files with 207 additions and 23 deletions

View file

@ -685,6 +685,8 @@ export interface Storage {
shared: boolean;
enabled: boolean;
active: boolean;
// RFC3339 timestamp of the last poll that delivered this entry; absent = never seen
lastSeen?: string;
// Added for deduplication in storage view
nodes?: string[];
nodeIds?: string[];

View file

@ -4391,6 +4391,12 @@ func generateStorage(nodes []models.Node) []models.Storage {
for _, node := range nodes {
isOffline := node.Status != "online" || node.ConnectionHealth == "offline" || node.Uptime <= 0
// Storage on an offline node hasn't been freshly observed; backdate
// its sighting so the stale path renders instead of fake freshness.
nodeStorageSeen := now
if isOffline {
nodeStorageSeen = now.Add(-10 * time.Minute)
}
// Local storage (always present)
localTotal := int64(500 * 1024 * 1024 * 1024) // 500GB
localID := fmt.Sprintf("%s-%s-local", node.Instance, node.Name)
@ -4427,6 +4433,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: false,
Enabled: localEnabled,
Active: localActive,
LastSeen: nodeStorageSeen,
})
// Local-zfs (common)
@ -4492,6 +4499,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: false,
Enabled: zfsEnabled,
Active: zfsActive,
LastSeen: nodeStorageSeen,
ZFSPool: zfsPool,
})
@ -4533,6 +4541,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: true, // PBS storage is shared cluster-wide (all nodes can access it)
Enabled: true,
Active: true,
LastSeen: now,
})
}
}
@ -4557,6 +4566,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: true,
Enabled: true,
Active: true,
LastSeen: now,
})
}
@ -4590,6 +4600,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: true,
Enabled: true,
Active: true,
LastSeen: now,
})
cephFsTotal := int64(24 * 1024 * 1024 * 1024 * 1024) // 24TB
@ -4610,6 +4621,7 @@ func generateStorage(nodes []models.Node) []models.Storage {
Shared: true,
Enabled: true,
Active: true,
LastSeen: now,
})
}
@ -5554,6 +5566,9 @@ func updateFixtureStateMetricsAt(data *models.StateSnapshot, config MockConfig,
continue
}
ApplyStorageUsage(storage, SampleMetric("storage", storage.ID, "usage", refreshNow))
// Refreshing usage simulates a fresh poll; stamp the sighting so
// available mock storage doesn't age past the stale thresholds.
storage.LastSeen = refreshNow
}
// Update disk metrics.

View file

@ -2046,27 +2046,28 @@ type DockerHostCommandStatus struct {
// Storage represents a storage resource
type Storage struct {
ID string `json:"id"`
Name string `json:"name"`
Node string `json:"node"`
Instance string `json:"instance"`
Nodes []string `json:"nodes,omitempty"`
NodeIDs []string `json:"nodeIds,omitempty"`
AliasIDs []string `json:"aliasIds,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Type string `json:"type"`
Status string `json:"status"`
Pool string `json:"pool,omitempty"`
Path string `json:"path,omitempty"`
Total int64 `json:"total"`
Used int64 `json:"used"`
Free int64 `json:"free"`
Usage float64 `json:"usage"`
Content string `json:"content"`
Shared bool `json:"shared"`
Enabled bool `json:"enabled"`
Active bool `json:"active"`
ZFSPool *ZFSPool `json:"zfsPool,omitempty"` // ZFS pool details if this is ZFS storage
ID string `json:"id"`
Name string `json:"name"`
Node string `json:"node"`
Instance string `json:"instance"`
Nodes []string `json:"nodes,omitempty"`
NodeIDs []string `json:"nodeIds,omitempty"`
AliasIDs []string `json:"aliasIds,omitempty"`
NodeCount int `json:"nodeCount,omitempty"`
Type string `json:"type"`
Status string `json:"status"`
Pool string `json:"pool,omitempty"`
Path string `json:"path,omitempty"`
Total int64 `json:"total"`
Used int64 `json:"used"`
Free int64 `json:"free"`
Usage float64 `json:"usage"`
Content string `json:"content"`
Shared bool `json:"shared"`
Enabled bool `json:"enabled"`
Active bool `json:"active"`
LastSeen time.Time `json:"lastSeen,omitzero"` // when the owning poller last delivered this entry; zero = never seen
ZFSPool *ZFSPool `json:"zfsPool,omitempty"` // ZFS pool details if this is ZFS storage
}
func CephPoolStorageID(instanceName, poolName string) string {
@ -2146,6 +2147,7 @@ func StorageFromCephPool(cluster CephCluster, pool CephPool) Storage {
Shared: true,
Enabled: true,
Active: status == "available",
LastSeen: cluster.LastUpdated,
}
}

View file

@ -3014,6 +3014,7 @@ func storageFromReadStateView(view *unifiedresources.StoragePoolView) models.Sto
Shared: view.Shared(),
Enabled: view.Enabled(),
Active: view.Active(),
LastSeen: view.LastSeen(),
ZFSPool: storageZFSPoolFromReadStateView(view),
}
}

View file

@ -471,6 +471,7 @@ func (m *Monitor) pollPBSInstance(ctx context.Context, instanceName string, clie
Shared: true, // PBS datastores are typically shared/network storage
Enabled: true,
Active: pbsInst.Status == "online",
LastSeen: pbsInst.LastSeen,
}
pbsStorages = append(pbsStorages, pbsStorage)
}

View file

@ -375,6 +375,7 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
Shared: shared,
Enabled: storage.Enabled == 1,
Active: storage.Active == 1,
LastSeen: startTime.UTC(),
}
if hasClusterConfig {
@ -605,6 +606,7 @@ func (m *Monitor) pollStorageWithNodes(ctx context.Context, instanceName string,
Shared: true,
Enabled: true,
Active: true,
LastSeen: startTime.UTC(),
}
allStorage = append(allStorage, synthetic)

View file

@ -1621,7 +1621,7 @@ func resourceFromStorage(storage models.Storage) (Resource, ResourceIdentity) {
Type: ResourceTypeStorage,
Name: name,
Status: statusFromStorage(storage),
LastSeen: now,
LastSeen: storage.LastSeen,
UpdatedAt: now,
Metrics: metricsFromStorage(storage),
Proxmox: &ProxmoxData{
@ -1973,7 +1973,7 @@ func resourceFromDockerContainer(ct models.DockerContainer, host models.DockerHo
Technology: runtime,
Name: ct.Name,
Status: statusFromDockerState(ct.State),
LastSeen: time.Now().UTC(),
LastSeen: host.LastSeen,
UpdatedAt: time.Now().UTC(),
Metrics: metrics,
}

View file

@ -853,3 +853,61 @@ func TestNamespacedKubernetesResourceScaffold(t *testing.T) {
t.Fatalf("identity clusterName = %q, want prod", identity.ClusterName)
}
}
// Storage models carry the timestamp of the poll that delivered them. The
// adapter must pass that sighting through — and preserve zero for never-seen
// entries — instead of fabricating a fresh one at conversion time, because
// the registry is rebuilt from the retained snapshot every cycle and a
// conversion-time stamp would report a dead source as freshly delivering.
func TestResourceFromStorageUsesModelSighting(t *testing.T) {
seen := time.Now().UTC().Add(-45 * time.Second).Truncate(time.Millisecond)
storage := models.Storage{
ID: "cluster-a-pve-1-local",
Name: "local",
Node: "pve-1",
Instance: "cluster-a",
Type: "dir",
Status: "available",
LastSeen: seen,
}
resource, _ := resourceFromStorage(storage)
if !resource.LastSeen.Equal(seen) {
t.Fatalf("resource.LastSeen = %s, want the model sighting %s", resource.LastSeen, seen)
}
storage.LastSeen = time.Time{}
resource, _ = resourceFromStorage(storage)
if !resource.LastSeen.IsZero() {
t.Fatalf("resource.LastSeen = %s, want zero for a never-seen storage entry", resource.LastSeen)
}
}
// Docker containers are delivered wholesale with each host report, so the
// host's report timestamp is the container's sighting. Stamping conversion
// time instead would keep a container "fresh" forever after its host agent
// stopped reporting.
func TestResourceFromDockerContainerUsesHostSighting(t *testing.T) {
seen := time.Now().UTC().Add(-45 * time.Second).Truncate(time.Millisecond)
container := models.DockerContainer{
ID: "abcdef123456",
Name: "web",
State: "running",
}
host := models.DockerHost{
ID: "docker-1",
Hostname: "docker-1",
LastSeen: seen,
}
resource, _ := resourceFromDockerContainer(container, host)
if !resource.LastSeen.Equal(seen) {
t.Fatalf("resource.LastSeen = %s, want the host report sighting %s", resource.LastSeen, seen)
}
host.LastSeen = time.Time{}
resource, _ = resourceFromDockerContainer(container, host)
if !resource.LastSeen.IsZero() {
t.Fatalf("resource.LastSeen = %s, want zero when the host has never reported", resource.LastSeen)
}
}

View file

@ -1513,6 +1513,20 @@ func attachLinkedAgentID(proxmox *ProxmoxData, linkedAgentID string) {
func (rr *ResourceRegistry) ingestStorage(storage models.Storage) {
resource, identity := resourceFromStorage(storage)
if pbsStorageInstance(storage) {
// Datastore-derived entries from the PBS poller (monitor_pbs_pmg)
// are delivered by the PBS source on the PBS cadence, not by a PVE
// poll. Keying them SourceProxmox would judge their freshness
// against the (faster) Proxmox stale threshold and flap healthy
// datastores stale between PBS polls. storage.Instance carries the
// PBS instance source ID ("pbs-<name>"), so parent them to the PBS
// instance, which IngestSnapshot ingests before storage.
if parentID, ok := rr.bySource[SourcePBS][storage.Instance]; ok {
resource.ParentID = &parentID
}
rr.ingest(SourcePBS, storage.ID, resource, identity)
return
}
parentSourceID := proxmoxNodeSourceID(storage.Instance, storage.Node)
if parentID, ok := rr.bySource[SourceProxmox][parentSourceID]; ok {
resource.ParentID = &parentID
@ -1520,6 +1534,15 @@ func (rr *ResourceRegistry) ingestStorage(storage models.Storage) {
rr.ingest(SourceProxmox, storage.ID, resource, identity)
}
// pbsStorageInstance reports whether a storage entry was produced by the PBS
// poller's datastore conversion rather than a PVE storage poll. PVE itself
// reports pbs-typed storage.cfg backends too, but those carry the PVE
// instance name; only the PBS poller writes the "pbs-" instance prefix.
func pbsStorageInstance(storage models.Storage) bool {
return strings.EqualFold(strings.TrimSpace(storage.Type), "pbs") &&
strings.HasPrefix(strings.TrimSpace(storage.Instance), "pbs-")
}
func (rr *ResourceRegistry) ingestPhysicalDisk(disk models.PhysicalDisk) {
resource, identity := resourceFromPhysicalDisk(disk)
parentSourceID := proxmoxNodeSourceID(disk.Instance, disk.Node)

View file

@ -4521,3 +4521,83 @@ func TestIngestStampsOnlineSourceStatusForRealSighting(t *testing.T) {
t.Fatalf("SourceStatus[proxmox].LastSeen = %s, want %s", status.LastSeen, seen)
}
}
// Datastore-derived storage entries written by the PBS poller (instance
// "pbs-<name>", type "pbs") are delivered on the PBS cadence, so they must be
// keyed to SourcePBS — judging them against the faster Proxmox stale
// threshold would flap healthy datastores stale between PBS polls. PVE also
// reports pbs-typed storage.cfg backends, and those stay SourceProxmox: they
// really are delivered by the PVE storage poll.
func TestIngestStorageRoutesPBSPollerEntriesToPBSSource(t *testing.T) {
seen := time.Now().UTC().Add(-30 * time.Second).Truncate(time.Millisecond)
snapshot := models.StateSnapshot{
PBSInstances: []models.PBSInstance{{
ID: "pbs-backup",
Name: "backup",
Host: "https://pbs.local:8007",
Status: "online",
LastSeen: seen,
}},
Storage: []models.Storage{
{
ID: "pbs-backup-main",
Name: "main",
Node: "backup",
Instance: "pbs-backup",
Type: "pbs",
Status: "available",
LastSeen: seen,
},
{
ID: "home-pve1-pbs-remote",
Name: "pbs-remote",
Node: "pve1",
Instance: "home",
Type: "pbs",
Status: "available",
LastSeen: seen,
},
},
}
rr := NewRegistry(nil)
rr.IngestSnapshot(snapshot)
var pbsInstanceID string
var pollerStorage, pveStorage *Resource
for _, resource := range rr.List() {
resource := resource
switch {
case resource.Type == ResourceTypePBS:
pbsInstanceID = resource.ID
case resource.Type == ResourceTypeStorage && resource.Proxmox != nil && resource.Proxmox.SourceID == "pbs-backup-main":
pollerStorage = &resource
case resource.Type == ResourceTypeStorage && resource.Proxmox != nil && resource.Proxmox.SourceID == "home-pve1-pbs-remote":
pveStorage = &resource
}
}
if pbsInstanceID == "" {
t.Fatal("expected a PBS instance resource")
}
if pollerStorage == nil || pveStorage == nil {
t.Fatalf("expected both storage resources (pollerStorage=%v, pveStorage=%v)", pollerStorage != nil, pveStorage != nil)
}
if len(pollerStorage.Sources) != 1 || pollerStorage.Sources[0] != SourcePBS {
t.Fatalf("PBS poller storage Sources = %v, want [%s]", pollerStorage.Sources, SourcePBS)
}
status, ok := pollerStorage.SourceStatus[SourcePBS]
if !ok {
t.Fatal("expected a pbs source status entry on the PBS poller storage")
}
if !status.LastSeen.Equal(seen) {
t.Fatalf("SourceStatus[pbs].LastSeen = %s, want %s", status.LastSeen, seen)
}
if pollerStorage.ParentID == nil || *pollerStorage.ParentID != pbsInstanceID {
t.Fatalf("PBS poller storage ParentID = %v, want the PBS instance %q", pollerStorage.ParentID, pbsInstanceID)
}
if len(pveStorage.Sources) != 1 || pveStorage.Sources[0] != SourceProxmox {
t.Fatalf("PVE-reported pbs-typed storage Sources = %v, want [%s]", pveStorage.Sources, SourceProxmox)
}
}