From 61cd902dedaa8cc219bd54b5ef1646c4d18434f8 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 1 May 2026 19:17:05 +0100 Subject: [PATCH] Fix Docker update alert disable cleanup Refs #1355 --- ...-docker-update-alert-disable-2026-05-01.md | 46 ++++++ docs/release-control/v6/internal/status.json | 6 + .../v6/internal/subsystems/alerts.md | 8 + internal/alerts/alerts.go | 110 +++++++++++++ internal/alerts/update_alerts_test.go | 154 ++++++++++++++++++ 5 files changed, 324 insertions(+) create mode 100644 docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md diff --git a/docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md b/docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md new file mode 100644 index 0000000000..56533bc857 --- /dev/null +++ b/docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md @@ -0,0 +1,46 @@ +# Known RC Issue Closure For GA Docker Update Alert Disable Record + +- Date: `2026-05-01` +- Gate: `known-rc-issue-closure-for-ga` +- Result: `passed` + +## Context + +The v5 maintenance delta audit found that `#1355` had not been fully carried +into v6. Pulse v5 cleared Docker image-update alerts when Docker update alerts +were disabled globally or for a specific container, but the v6 alerts runtime +could preserve the active alert and first-seen tracking after the setting no +longer permitted that alert to fire. + +That meant an operator could disable Docker update alerts, or disable alerting +for a single Docker container, and still see stale image-update alert state +survive into the v6 candidate. + +## Disposition + +The fix is in the alerts runtime Docker update-alert lifecycle: + +- `UpdateConfig` now clears active Docker container update alerts and both + resource and identity first-seen tracking when Docker update alerts are + disabled. +- Active-alert reevaluation now treats Docker update alerts as lifecycle alerts, + not generic threshold alerts, so disabled Docker containers, ignored prefixes, + and disabled update-alert settings resolve stale update incidents. +- Per-container disabled overrides now clear the canonical image-update alert, + the legacy update alias, and update tracking before skipping evaluation. +- The enabled path preserves existing update alerts and first-seen tracking + when unrelated Docker threshold settings change. + +## Proof + +- `go test ./internal/alerts -run 'Test(UpdateConfig(Clears|Keeps)DockerContainerUpdateAlerts|EvaluateDockerContainerClearsUpdateAlert|DockerUpdateTracking)' -count=1` +- `go test ./internal/alerts -run 'TestCheckDockerContainerImageUpdate' -count=1` +- `go test ./internal/alerts -run 'Test.*Docker.*Update|TestCheckDockerContainerImageUpdate|TestUpdateConfig' -count=1` +- `go test ./internal/alerts -count=1` + +## Outcome + +The v6 candidate no longer knowingly regresses v5 `#1355`. Docker image-update +alerts are removed when the owning configuration disables them, and the runtime +does not leave stale first-seen tracking behind that could resurrect an already +disabled update alert. diff --git a/docs/release-control/v6/internal/status.json b/docs/release-control/v6/internal/status.json index 00904a5eb6..a1f06271a8 100644 --- a/docs/release-control/v6/internal/status.json +++ b/docs/release-control/v6/internal/status.json @@ -4421,6 +4421,12 @@ "kind": "file", "evidence_tier": "managed-runtime-exercise" }, + { + "repo": "pulse", + "path": "docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md", + "kind": "file", + "evidence_tier": "managed-runtime-exercise" + }, { "repo": "pulse", "path": "docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-duplicate-metrics-2026-05-01.md", diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index 98c7774677..cc060fafa4 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -148,6 +148,14 @@ override identity candidates. Static metric-color defaults are only fallback presentation behavior for callers that do not have alert configuration in scope. +Docker container image-update alerts are lifecycle-governed by the alerts +runtime. Disabling Docker update alerts globally, disabling alerts for a +specific Docker container, ignoring a Docker container prefix, or disabling all +Docker container alerts must clear the active image-update alert plus resource +and identity first-seen tracking. Generic threshold reevaluation must not keep +or resurrect image-update alerts after their owning Docker alert configuration +has disabled them. + Alert history persistence is also part of that canonical boundary. The history manager may choose the owned runtime data directory, but it must normalize that directory once and then resolve only the fixed `alert-history.json` and diff --git a/internal/alerts/alerts.go b/internal/alerts/alerts.go index 4ad7b36168..a54c96bfbe 100644 --- a/internal/alerts/alerts.go +++ b/internal/alerts/alerts.go @@ -1980,6 +1980,11 @@ func (m *Manager) applyGlobalOfflineSettingsLocked() { m.dockerUpdateFirstSeen = make(map[string]time.Time) m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time) } + if m.config.DockerDefaults.UpdateAlertDelayHours < 0 && !m.config.DisableAllDockerContainers { + m.clearDockerContainerUpdateAlertsLocked() + m.dockerUpdateFirstSeen = make(map[string]time.Time) + m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time) + } if m.config.DisableAllDockerServices { var serviceAlerts []string for storageKey, alert := range m.activeAlerts { @@ -2007,6 +2012,12 @@ func (m *Manager) reevaluateActiveAlertsLocked() { for storageKey, alert := range m.activeAlerts { alertID := effectiveAlertID(alert, storageKey) backfillCanonicalIdentity(alert) + if alert.Type == "docker-container-update" || strings.HasPrefix(alertID, "docker-container-update-") { + if m.shouldResolveDockerContainerUpdateAlertLocked(alert) { + alertsToResolve = append(alertsToResolve, alertID) + } + continue + } resourceID := alert.ResourceID metricType := alert.Type if resourceID == "" || metricType == "" { @@ -4915,6 +4926,9 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode m.clearDockerContainerStateAlert(resourceID) m.clearDockerContainerHealthAlert(resourceID) m.clearDockerContainerMetricAlerts(resourceID) + m.clearAlert(fmt.Sprintf("docker-container-update-%s", resourceID)) + m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-image-update")) + m.clearDockerContainerUpdateTracking(resourceID, dockerUpdateTrackingKey(host, container)) return } @@ -5777,6 +5791,102 @@ func (m *Manager) clearDockerContainerUpdateTracking(resourceID, trackingKey str m.mu.Unlock() } +func dockerUpdateTrackingKeyFromAlert(alert *Alert) string { + if alert == nil || alert.Metadata == nil { + return "" + } + + metadataString := func(key string) string { + value, ok := alert.Metadata[key] + if !ok || value == nil { + return "" + } + return strings.TrimSpace(fmt.Sprint(value)) + } + + host := models.DockerHost{ + ID: metadataString("hostId"), + DisplayName: metadataString("hostName"), + Hostname: metadataString("hostHostname"), + } + container := models.DockerContainer{ + ID: metadataString("containerId"), + Name: metadataString("containerName"), + Image: metadataString("image"), + } + + if host.ID == "" && host.DisplayName == "" && host.Hostname == "" && + container.ID == "" && container.Name == "" && container.Image == "" { + return "" + } + + return dockerUpdateTrackingKey(host, container) +} + +func (m *Manager) clearDockerContainerUpdateStateLocked(alert *Alert) { + if alert == nil { + return + } + + if alert.ResourceID != "" { + delete(m.dockerUpdateFirstSeen, alert.ResourceID) + } + if trackingKey := dockerUpdateTrackingKeyFromAlert(alert); trackingKey != "" { + delete(m.dockerUpdateFirstSeenByIdentity, trackingKey) + } +} + +func (m *Manager) clearDockerContainerUpdateAlertsLocked() { + toClear := make([]string, 0) + for storageKey, alert := range m.activeAlerts { + if alert == nil { + continue + } + alertID := effectiveAlertID(alert, storageKey) + if alert.Type != "docker-container-update" && !strings.HasPrefix(alertID, "docker-container-update-") { + continue + } + m.clearDockerContainerUpdateStateLocked(alert) + toClear = append(toClear, alertID) + } + for _, alertID := range toClear { + m.clearAlertNoLock(alertID) + } +} + +func (m *Manager) shouldResolveDockerContainerUpdateAlertLocked(alert *Alert) bool { + if alert == nil { + return false + } + + if m.config.DisableAllDockerContainers || m.config.DockerDefaults.UpdateAlertDelayHours < 0 { + m.clearDockerContainerUpdateStateLocked(alert) + return true + } + + if override, exists := m.config.Overrides[alert.ResourceID]; exists && override.Disabled { + m.clearDockerContainerUpdateStateLocked(alert) + return true + } + + containerName := strings.TrimSpace(alert.ResourceName) + containerID := "" + if alert.Metadata != nil { + if value, ok := alert.Metadata["containerName"].(string); ok && containerName == "" { + containerName = value + } + if value, ok := alert.Metadata["containerId"].(string); ok { + containerID = value + } + } + if matchesDockerIgnoredPrefix(containerName, containerID, m.config.DockerIgnoredContainerPrefixes) { + m.clearDockerContainerUpdateStateLocked(alert) + return true + } + + return false +} + func (m *Manager) touchDockerContainerUpdateAlert(alertID string) { m.mu.Lock() defer m.mu.Unlock() diff --git a/internal/alerts/update_alerts_test.go b/internal/alerts/update_alerts_test.go index 3ddbd37728..964f645ee4 100644 --- a/internal/alerts/update_alerts_test.go +++ b/internal/alerts/update_alerts_test.go @@ -526,6 +526,160 @@ func TestDockerUpdateTrackingCleanup(t *testing.T) { } } +func seedDockerUpdateAlert(t *testing.T, m *Manager, host models.DockerHost, container models.DockerContainer, resourceID string, firstSeen time.Time) (string, string, string) { + t.Helper() + + containerName := dockerContainerDisplayName(container) + alertID := "docker-container-update-" + resourceID + canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update") + trackingKey := dockerUpdateTrackingKey(host, container) + alert := &Alert{ + ID: alertID, + Type: "docker-container-update", + Level: AlertLevelWarning, + ResourceID: resourceID, + ResourceName: containerName, + Instance: dockerInstanceName(host), + CanonicalState: canonicalAlertID, + Metadata: dockerContainerAlertMetadata(host, container, containerName), + StartTime: firstSeen, + LastSeen: firstSeen, + } + + m.mu.Lock() + m.setActiveAlertNoLock(canonicalAlertID, alert) + m.dockerUpdateFirstSeen[resourceID] = firstSeen + m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen + m.mu.Unlock() + + return alertID, canonicalAlertID, trackingKey +} + +func TestUpdateConfigClearsDockerContainerUpdateAlertsWhenDisabled(t *testing.T) { + m := NewManager() + + host := models.DockerHost{ + ID: "docker-host-1", + DisplayName: "Docker Host", + Hostname: "docker.local", + } + container := models.DockerContainer{ + ID: "container-1", + Name: "/frontend", + Image: "nginx:latest", + } + resourceID := dockerResourceID(host.ID, container.ID) + firstSeen := time.Now().Add(-48 * time.Hour) + _, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) + + config := m.GetConfig() + config.DockerDefaults.UpdateAlertDelayHours = -1 + m.UpdateConfig(config) + + m.mu.RLock() + _, hasAlert := m.activeAlerts[canonicalAlertID] + _, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID] + _, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey] + m.mu.RUnlock() + + if hasAlert { + t.Fatalf("expected docker update alert to be cleared when update alerts are disabled") + } + if hasResourceTracking { + t.Fatalf("expected docker update resource tracking to be cleared when update alerts are disabled") + } + if hasIdentityTracking { + t.Fatalf("expected docker update identity tracking to be cleared when update alerts are disabled") + } +} + +func TestUpdateConfigKeepsDockerContainerUpdateAlertsWhenStillEnabled(t *testing.T) { + m := NewManager() + + host := models.DockerHost{ + ID: "docker-host-2", + DisplayName: "Docker Host", + Hostname: "docker-2.local", + } + container := models.DockerContainer{ + ID: "container-2", + Name: "/api", + Image: "ghcr.io/example/api:latest", + } + resourceID := dockerResourceID(host.ID, container.ID) + firstSeen := time.Now().Add(-30 * time.Hour) + _, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) + + config := m.GetConfig() + config.DockerDefaults.UpdateAlertDelayHours = 24 + config.DockerDefaults.CPU.Trigger = 95 + config.DockerDefaults.CPU.Clear = 90 + m.UpdateConfig(config) + + m.mu.RLock() + _, hasAlert := m.activeAlerts[canonicalAlertID] + resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID] + identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey] + m.mu.RUnlock() + + if !hasAlert { + t.Fatalf("expected docker update alert to remain active when update alerts are still enabled") + } + if !hasResourceTracking { + t.Fatalf("expected docker update resource tracking to remain when update alerts stay enabled") + } + if !hasIdentityTracking { + t.Fatalf("expected docker update identity tracking to remain when update alerts stay enabled") + } + if !resourceFirstSeen.Equal(firstSeen) { + t.Fatalf("expected docker update resource tracking to preserve firstSeen, got %s want %s", resourceFirstSeen, firstSeen) + } + if !identityFirstSeen.Equal(firstSeen) { + t.Fatalf("expected docker update identity tracking to preserve firstSeen, got %s want %s", identityFirstSeen, firstSeen) + } +} + +func TestEvaluateDockerContainerClearsUpdateAlertWhenOverrideDisabled(t *testing.T) { + m := NewManager() + + host := models.DockerHost{ + ID: "docker-host-3", + DisplayName: "Docker Host", + Hostname: "docker-3.local", + } + container := models.DockerContainer{ + ID: "container-3", + Name: "/worker", + Image: "ghcr.io/example/worker:latest", + } + resourceID := dockerResourceID(host.ID, container.ID) + firstSeen := time.Now().Add(-36 * time.Hour) + alertID, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen) + + m.mu.Lock() + m.config.Overrides[resourceID] = ThresholdConfig{Disabled: true} + m.mu.Unlock() + + m.evaluateDockerContainer(host, container, resourceID) + + m.mu.RLock() + _, hasCanonicalAlert := m.activeAlerts[canonicalAlertID] + _, hasLegacyAlias := m.activeAlertAlias[alertID] + _, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID] + _, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey] + m.mu.RUnlock() + + if hasCanonicalAlert || hasLegacyAlias { + t.Fatalf("expected docker update alert to be cleared when the container override disables alerts") + } + if hasResourceTracking { + t.Fatalf("expected docker update resource tracking to be cleared when the container override disables alerts") + } + if hasIdentityTracking { + t.Fatalf("expected docker update identity tracking to be cleared when the container override disables alerts") + } +} + func TestDockerUpdateTrackingHostKeyFallbackOrder(t *testing.T) { tests := []struct { name string