mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-19 18:44:53 +00:00
parent
361f289485
commit
61cd902ded
5 changed files with 324 additions and 0 deletions
|
|
@ -0,0 +1,46 @@
|
|||
# Known RC Issue Closure For GA Docker Update Alert Disable Record
|
||||
|
||||
- Date: `2026-05-01`
|
||||
- Gate: `known-rc-issue-closure-for-ga`
|
||||
- Result: `passed`
|
||||
|
||||
## Context
|
||||
|
||||
The v5 maintenance delta audit found that `#1355` had not been fully carried
|
||||
into v6. Pulse v5 cleared Docker image-update alerts when Docker update alerts
|
||||
were disabled globally or for a specific container, but the v6 alerts runtime
|
||||
could preserve the active alert and first-seen tracking after the setting no
|
||||
longer permitted that alert to fire.
|
||||
|
||||
That meant an operator could disable Docker update alerts, or disable alerting
|
||||
for a single Docker container, and still see stale image-update alert state
|
||||
survive into the v6 candidate.
|
||||
|
||||
## Disposition
|
||||
|
||||
The fix is in the alerts runtime Docker update-alert lifecycle:
|
||||
|
||||
- `UpdateConfig` now clears active Docker container update alerts and both
|
||||
resource and identity first-seen tracking when Docker update alerts are
|
||||
disabled.
|
||||
- Active-alert reevaluation now treats Docker update alerts as lifecycle alerts,
|
||||
not generic threshold alerts, so disabled Docker containers, ignored prefixes,
|
||||
and disabled update-alert settings resolve stale update incidents.
|
||||
- Per-container disabled overrides now clear the canonical image-update alert,
|
||||
the legacy update alias, and update tracking before skipping evaluation.
|
||||
- The enabled path preserves existing update alerts and first-seen tracking
|
||||
when unrelated Docker threshold settings change.
|
||||
|
||||
## Proof
|
||||
|
||||
- `go test ./internal/alerts -run 'Test(UpdateConfig(Clears|Keeps)DockerContainerUpdateAlerts|EvaluateDockerContainerClearsUpdateAlert|DockerUpdateTracking)' -count=1`
|
||||
- `go test ./internal/alerts -run 'TestCheckDockerContainerImageUpdate' -count=1`
|
||||
- `go test ./internal/alerts -run 'Test.*Docker.*Update|TestCheckDockerContainerImageUpdate|TestUpdateConfig' -count=1`
|
||||
- `go test ./internal/alerts -count=1`
|
||||
|
||||
## Outcome
|
||||
|
||||
The v6 candidate no longer knowingly regresses v5 `#1355`. Docker image-update
|
||||
alerts are removed when the owning configuration disables them, and the runtime
|
||||
does not leave stale first-seen tracking behind that could resurrect an already
|
||||
disabled update alert.
|
||||
|
|
@ -4421,6 +4421,12 @@
|
|||
"kind": "file",
|
||||
"evidence_tier": "managed-runtime-exercise"
|
||||
},
|
||||
{
|
||||
"repo": "pulse",
|
||||
"path": "docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-docker-update-alert-disable-2026-05-01.md",
|
||||
"kind": "file",
|
||||
"evidence_tier": "managed-runtime-exercise"
|
||||
},
|
||||
{
|
||||
"repo": "pulse",
|
||||
"path": "docs/release-control/v6/internal/records/known-rc-issue-closure-for-ga-duplicate-metrics-2026-05-01.md",
|
||||
|
|
|
|||
|
|
@ -148,6 +148,14 @@ override identity candidates. Static metric-color defaults are only fallback
|
|||
presentation behavior for callers that do not have alert configuration in
|
||||
scope.
|
||||
|
||||
Docker container image-update alerts are lifecycle-governed by the alerts
|
||||
runtime. Disabling Docker update alerts globally, disabling alerts for a
|
||||
specific Docker container, ignoring a Docker container prefix, or disabling all
|
||||
Docker container alerts must clear the active image-update alert plus resource
|
||||
and identity first-seen tracking. Generic threshold reevaluation must not keep
|
||||
or resurrect image-update alerts after their owning Docker alert configuration
|
||||
has disabled them.
|
||||
|
||||
Alert history persistence is also part of that canonical boundary. The history
|
||||
manager may choose the owned runtime data directory, but it must normalize that
|
||||
directory once and then resolve only the fixed `alert-history.json` and
|
||||
|
|
|
|||
|
|
@ -1980,6 +1980,11 @@ func (m *Manager) applyGlobalOfflineSettingsLocked() {
|
|||
m.dockerUpdateFirstSeen = make(map[string]time.Time)
|
||||
m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time)
|
||||
}
|
||||
if m.config.DockerDefaults.UpdateAlertDelayHours < 0 && !m.config.DisableAllDockerContainers {
|
||||
m.clearDockerContainerUpdateAlertsLocked()
|
||||
m.dockerUpdateFirstSeen = make(map[string]time.Time)
|
||||
m.dockerUpdateFirstSeenByIdentity = make(map[string]time.Time)
|
||||
}
|
||||
if m.config.DisableAllDockerServices {
|
||||
var serviceAlerts []string
|
||||
for storageKey, alert := range m.activeAlerts {
|
||||
|
|
@ -2007,6 +2012,12 @@ func (m *Manager) reevaluateActiveAlertsLocked() {
|
|||
for storageKey, alert := range m.activeAlerts {
|
||||
alertID := effectiveAlertID(alert, storageKey)
|
||||
backfillCanonicalIdentity(alert)
|
||||
if alert.Type == "docker-container-update" || strings.HasPrefix(alertID, "docker-container-update-") {
|
||||
if m.shouldResolveDockerContainerUpdateAlertLocked(alert) {
|
||||
alertsToResolve = append(alertsToResolve, alertID)
|
||||
}
|
||||
continue
|
||||
}
|
||||
resourceID := alert.ResourceID
|
||||
metricType := alert.Type
|
||||
if resourceID == "" || metricType == "" {
|
||||
|
|
@ -4915,6 +4926,9 @@ func (m *Manager) evaluateDockerContainer(host models.DockerHost, container mode
|
|||
m.clearDockerContainerStateAlert(resourceID)
|
||||
m.clearDockerContainerHealthAlert(resourceID)
|
||||
m.clearDockerContainerMetricAlerts(resourceID)
|
||||
m.clearAlert(fmt.Sprintf("docker-container-update-%s", resourceID))
|
||||
m.clearAlert(buildCanonicalStateID(resourceID, resourceID+"-image-update"))
|
||||
m.clearDockerContainerUpdateTracking(resourceID, dockerUpdateTrackingKey(host, container))
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -5777,6 +5791,102 @@ func (m *Manager) clearDockerContainerUpdateTracking(resourceID, trackingKey str
|
|||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
func dockerUpdateTrackingKeyFromAlert(alert *Alert) string {
|
||||
if alert == nil || alert.Metadata == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
metadataString := func(key string) string {
|
||||
value, ok := alert.Metadata[key]
|
||||
if !ok || value == nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(fmt.Sprint(value))
|
||||
}
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: metadataString("hostId"),
|
||||
DisplayName: metadataString("hostName"),
|
||||
Hostname: metadataString("hostHostname"),
|
||||
}
|
||||
container := models.DockerContainer{
|
||||
ID: metadataString("containerId"),
|
||||
Name: metadataString("containerName"),
|
||||
Image: metadataString("image"),
|
||||
}
|
||||
|
||||
if host.ID == "" && host.DisplayName == "" && host.Hostname == "" &&
|
||||
container.ID == "" && container.Name == "" && container.Image == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
return dockerUpdateTrackingKey(host, container)
|
||||
}
|
||||
|
||||
func (m *Manager) clearDockerContainerUpdateStateLocked(alert *Alert) {
|
||||
if alert == nil {
|
||||
return
|
||||
}
|
||||
|
||||
if alert.ResourceID != "" {
|
||||
delete(m.dockerUpdateFirstSeen, alert.ResourceID)
|
||||
}
|
||||
if trackingKey := dockerUpdateTrackingKeyFromAlert(alert); trackingKey != "" {
|
||||
delete(m.dockerUpdateFirstSeenByIdentity, trackingKey)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) clearDockerContainerUpdateAlertsLocked() {
|
||||
toClear := make([]string, 0)
|
||||
for storageKey, alert := range m.activeAlerts {
|
||||
if alert == nil {
|
||||
continue
|
||||
}
|
||||
alertID := effectiveAlertID(alert, storageKey)
|
||||
if alert.Type != "docker-container-update" && !strings.HasPrefix(alertID, "docker-container-update-") {
|
||||
continue
|
||||
}
|
||||
m.clearDockerContainerUpdateStateLocked(alert)
|
||||
toClear = append(toClear, alertID)
|
||||
}
|
||||
for _, alertID := range toClear {
|
||||
m.clearAlertNoLock(alertID)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) shouldResolveDockerContainerUpdateAlertLocked(alert *Alert) bool {
|
||||
if alert == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if m.config.DisableAllDockerContainers || m.config.DockerDefaults.UpdateAlertDelayHours < 0 {
|
||||
m.clearDockerContainerUpdateStateLocked(alert)
|
||||
return true
|
||||
}
|
||||
|
||||
if override, exists := m.config.Overrides[alert.ResourceID]; exists && override.Disabled {
|
||||
m.clearDockerContainerUpdateStateLocked(alert)
|
||||
return true
|
||||
}
|
||||
|
||||
containerName := strings.TrimSpace(alert.ResourceName)
|
||||
containerID := ""
|
||||
if alert.Metadata != nil {
|
||||
if value, ok := alert.Metadata["containerName"].(string); ok && containerName == "" {
|
||||
containerName = value
|
||||
}
|
||||
if value, ok := alert.Metadata["containerId"].(string); ok {
|
||||
containerID = value
|
||||
}
|
||||
}
|
||||
if matchesDockerIgnoredPrefix(containerName, containerID, m.config.DockerIgnoredContainerPrefixes) {
|
||||
m.clearDockerContainerUpdateStateLocked(alert)
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *Manager) touchDockerContainerUpdateAlert(alertID string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -526,6 +526,160 @@ func TestDockerUpdateTrackingCleanup(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func seedDockerUpdateAlert(t *testing.T, m *Manager, host models.DockerHost, container models.DockerContainer, resourceID string, firstSeen time.Time) (string, string, string) {
|
||||
t.Helper()
|
||||
|
||||
containerName := dockerContainerDisplayName(container)
|
||||
alertID := "docker-container-update-" + resourceID
|
||||
canonicalAlertID := buildCanonicalStateID(resourceID, resourceID+"-image-update")
|
||||
trackingKey := dockerUpdateTrackingKey(host, container)
|
||||
alert := &Alert{
|
||||
ID: alertID,
|
||||
Type: "docker-container-update",
|
||||
Level: AlertLevelWarning,
|
||||
ResourceID: resourceID,
|
||||
ResourceName: containerName,
|
||||
Instance: dockerInstanceName(host),
|
||||
CanonicalState: canonicalAlertID,
|
||||
Metadata: dockerContainerAlertMetadata(host, container, containerName),
|
||||
StartTime: firstSeen,
|
||||
LastSeen: firstSeen,
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.setActiveAlertNoLock(canonicalAlertID, alert)
|
||||
m.dockerUpdateFirstSeen[resourceID] = firstSeen
|
||||
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
|
||||
m.mu.Unlock()
|
||||
|
||||
return alertID, canonicalAlertID, trackingKey
|
||||
}
|
||||
|
||||
func TestUpdateConfigClearsDockerContainerUpdateAlertsWhenDisabled(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "docker-host-1",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker.local",
|
||||
}
|
||||
container := models.DockerContainer{
|
||||
ID: "container-1",
|
||||
Name: "/frontend",
|
||||
Image: "nginx:latest",
|
||||
}
|
||||
resourceID := dockerResourceID(host.ID, container.ID)
|
||||
firstSeen := time.Now().Add(-48 * time.Hour)
|
||||
_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
|
||||
|
||||
config := m.GetConfig()
|
||||
config.DockerDefaults.UpdateAlertDelayHours = -1
|
||||
m.UpdateConfig(config)
|
||||
|
||||
m.mu.RLock()
|
||||
_, hasAlert := m.activeAlerts[canonicalAlertID]
|
||||
_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
|
||||
_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if hasAlert {
|
||||
t.Fatalf("expected docker update alert to be cleared when update alerts are disabled")
|
||||
}
|
||||
if hasResourceTracking {
|
||||
t.Fatalf("expected docker update resource tracking to be cleared when update alerts are disabled")
|
||||
}
|
||||
if hasIdentityTracking {
|
||||
t.Fatalf("expected docker update identity tracking to be cleared when update alerts are disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateConfigKeepsDockerContainerUpdateAlertsWhenStillEnabled(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "docker-host-2",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker-2.local",
|
||||
}
|
||||
container := models.DockerContainer{
|
||||
ID: "container-2",
|
||||
Name: "/api",
|
||||
Image: "ghcr.io/example/api:latest",
|
||||
}
|
||||
resourceID := dockerResourceID(host.ID, container.ID)
|
||||
firstSeen := time.Now().Add(-30 * time.Hour)
|
||||
_, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
|
||||
|
||||
config := m.GetConfig()
|
||||
config.DockerDefaults.UpdateAlertDelayHours = 24
|
||||
config.DockerDefaults.CPU.Trigger = 95
|
||||
config.DockerDefaults.CPU.Clear = 90
|
||||
m.UpdateConfig(config)
|
||||
|
||||
m.mu.RLock()
|
||||
_, hasAlert := m.activeAlerts[canonicalAlertID]
|
||||
resourceFirstSeen, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
|
||||
identityFirstSeen, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if !hasAlert {
|
||||
t.Fatalf("expected docker update alert to remain active when update alerts are still enabled")
|
||||
}
|
||||
if !hasResourceTracking {
|
||||
t.Fatalf("expected docker update resource tracking to remain when update alerts stay enabled")
|
||||
}
|
||||
if !hasIdentityTracking {
|
||||
t.Fatalf("expected docker update identity tracking to remain when update alerts stay enabled")
|
||||
}
|
||||
if !resourceFirstSeen.Equal(firstSeen) {
|
||||
t.Fatalf("expected docker update resource tracking to preserve firstSeen, got %s want %s", resourceFirstSeen, firstSeen)
|
||||
}
|
||||
if !identityFirstSeen.Equal(firstSeen) {
|
||||
t.Fatalf("expected docker update identity tracking to preserve firstSeen, got %s want %s", identityFirstSeen, firstSeen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateDockerContainerClearsUpdateAlertWhenOverrideDisabled(t *testing.T) {
|
||||
m := NewManager()
|
||||
|
||||
host := models.DockerHost{
|
||||
ID: "docker-host-3",
|
||||
DisplayName: "Docker Host",
|
||||
Hostname: "docker-3.local",
|
||||
}
|
||||
container := models.DockerContainer{
|
||||
ID: "container-3",
|
||||
Name: "/worker",
|
||||
Image: "ghcr.io/example/worker:latest",
|
||||
}
|
||||
resourceID := dockerResourceID(host.ID, container.ID)
|
||||
firstSeen := time.Now().Add(-36 * time.Hour)
|
||||
alertID, canonicalAlertID, trackingKey := seedDockerUpdateAlert(t, m, host, container, resourceID, firstSeen)
|
||||
|
||||
m.mu.Lock()
|
||||
m.config.Overrides[resourceID] = ThresholdConfig{Disabled: true}
|
||||
m.mu.Unlock()
|
||||
|
||||
m.evaluateDockerContainer(host, container, resourceID)
|
||||
|
||||
m.mu.RLock()
|
||||
_, hasCanonicalAlert := m.activeAlerts[canonicalAlertID]
|
||||
_, hasLegacyAlias := m.activeAlertAlias[alertID]
|
||||
_, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
|
||||
_, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
|
||||
m.mu.RUnlock()
|
||||
|
||||
if hasCanonicalAlert || hasLegacyAlias {
|
||||
t.Fatalf("expected docker update alert to be cleared when the container override disables alerts")
|
||||
}
|
||||
if hasResourceTracking {
|
||||
t.Fatalf("expected docker update resource tracking to be cleared when the container override disables alerts")
|
||||
}
|
||||
if hasIdentityTracking {
|
||||
t.Fatalf("expected docker update identity tracking to be cleared when the container override disables alerts")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDockerUpdateTrackingHostKeyFallbackOrder(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue