Fix repeated Docker update recovery alerts

Preserve Docker container update alerts and first-seen tracking when update status is temporarily unavailable or the registry check fails.

Fixes #1394.
This commit is contained in:
rcourtman 2026-04-09 15:59:15 +01:00
parent d8986a0285
commit 30eb9d7847
2 changed files with 192 additions and 5 deletions

View file

@ -4934,6 +4934,15 @@ func (m *Manager) clearDockerContainerUpdateTracking(resourceID, trackingKey str
m.mu.Unlock()
}
func (m *Manager) touchDockerContainerUpdateAlert(alertID string) {
m.mu.Lock()
defer m.mu.Unlock()
if alert, exists := m.activeAlerts[alertID]; exists && alert != nil {
alert.LastSeen = time.Now()
}
}
func dockerUpdateTrackingKeyFromAlert(alert *Alert) string {
if alert == nil || alert.Metadata == nil {
return ""
@ -5040,16 +5049,17 @@ func (m *Manager) checkDockerContainerImageUpdate(host models.DockerHost, contai
// Check if this container has an update status reported
if container.UpdateStatus == nil {
// No update status - clear any tracking and alerts
m.clearAlert(alertID)
m.clearDockerContainerUpdateTracking(resourceID, updateTrackingKey)
// Missing update status means the condition is currently unknown, not resolved.
// Preserve any pending update alert and tracking until we see an affirmative clear.
m.touchDockerContainerUpdateAlert(alertID)
return
}
// Check for errors in update detection (don't alert on errors)
if container.UpdateStatus.Error != "" {
// Update check failed - clear alert but keep tracking
m.clearAlert(alertID)
// A failed update check cannot confirm the update has been resolved.
// Keep the existing alert active and preserve first-seen tracking.
m.touchDockerContainerUpdateAlert(alertID)
return
}

View file

@ -318,6 +318,183 @@ func TestCheckDockerContainerImageUpdatePreservesDelayAcrossHostIDChange(t *test
}
}
func TestCheckDockerContainerImageUpdatePreservesTrackingWhenStatusUnknown(t *testing.T) {
tests := []struct {
name string
updateStatus *models.DockerContainerUpdateStatus
}{
{
name: "missing update status",
updateStatus: nil,
},
{
name: "errored update status",
updateStatus: &models.DockerContainerUpdateStatus{
Error: "rate limited",
LastChecked: time.Now(),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newTestManager(t)
m.mu.Lock()
m.config.DockerDefaults.UpdateAlertDelayHours = 24
m.mu.Unlock()
host := models.DockerHost{
ID: "docker-host-unknown-status",
DisplayName: "Docker Host",
Hostname: "docker.local",
}
container := models.DockerContainer{
ID: "container-unknown-status",
Name: "/frontend",
Image: "nginx:latest",
UpdateStatus: tt.updateStatus,
}
resourceID := dockerResourceID(host.ID, container.ID)
trackingKey := dockerUpdateTrackingKey(host, container)
firstSeen := time.Now().Add(-6 * time.Hour)
m.mu.Lock()
m.dockerUpdateFirstSeen[resourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
m.checkDockerContainerImageUpdate(host, container, resourceID, "frontend", "docker-instance", "docker.local")
m.mu.RLock()
resourceTrackedAt, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
identityTrackedAt, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
_, hasAlert := m.activeAlerts["docker-container-update-"+resourceID]
m.mu.RUnlock()
if !hasResourceTracking {
t.Fatalf("expected resource tracking to remain when update status is unknown")
}
if !hasIdentityTracking {
t.Fatalf("expected identity tracking to remain when update status is unknown")
}
if !resourceTrackedAt.Equal(firstSeen) {
t.Fatalf("expected resource tracking firstSeen to remain %s, got %s", firstSeen, resourceTrackedAt)
}
if !identityTrackedAt.Equal(firstSeen) {
t.Fatalf("expected identity tracking firstSeen to remain %s, got %s", firstSeen, identityTrackedAt)
}
if hasAlert {
t.Fatalf("did not expect an alert to be created while update status is unknown")
}
})
}
}
func TestCheckDockerContainerImageUpdateKeepsActiveAlertWhenStatusUnknown(t *testing.T) {
tests := []struct {
name string
updateStatus *models.DockerContainerUpdateStatus
}{
{
name: "missing update status",
updateStatus: nil,
},
{
name: "errored update status",
updateStatus: &models.DockerContainerUpdateStatus{
Error: "rate limited",
LastChecked: time.Now(),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
m := newTestManager(t)
m.mu.Lock()
m.config.DockerDefaults.UpdateAlertDelayHours = 24
m.mu.Unlock()
host := models.DockerHost{
ID: "docker-host-active-alert",
DisplayName: "Docker Host",
Hostname: "docker.local",
}
container := models.DockerContainer{
ID: "container-active-alert",
Name: "/frontend",
Image: "nginx:latest",
UpdateStatus: tt.updateStatus,
}
resourceID := dockerResourceID(host.ID, container.ID)
alertID := "docker-container-update-" + resourceID
trackingKey := dockerUpdateTrackingKey(host, container)
firstSeen := time.Now().Add(-48 * time.Hour)
previousLastSeen := time.Now().Add(-30 * time.Minute)
m.mu.Lock()
m.activeAlerts[alertID] = &Alert{
ID: alertID,
Type: "docker-container-update",
ResourceID: resourceID,
ResourceName: "frontend",
Instance: "Docker",
StartTime: firstSeen,
LastSeen: previousLastSeen,
Metadata: map[string]interface{}{
"resourceType": "Docker Container",
"hostId": host.ID,
"hostName": host.DisplayName,
"hostHostname": host.Hostname,
"containerId": container.ID,
"containerName": "frontend",
"image": container.Image,
},
}
m.dockerUpdateFirstSeen[resourceID] = firstSeen
m.dockerUpdateFirstSeenByIdentity[trackingKey] = firstSeen
m.mu.Unlock()
m.checkDockerContainerImageUpdate(host, container, resourceID, "frontend", "docker-instance", "docker.local")
m.mu.RLock()
alert, hasAlert := m.activeAlerts[alertID]
resourceTrackedAt, hasResourceTracking := m.dockerUpdateFirstSeen[resourceID]
identityTrackedAt, hasIdentityTracking := m.dockerUpdateFirstSeenByIdentity[trackingKey]
m.mu.RUnlock()
m.resolvedMutex.RLock()
_, wasResolved := m.recentlyResolved[alertID]
m.resolvedMutex.RUnlock()
if !hasAlert {
t.Fatalf("expected active docker update alert to remain when update status is unknown")
}
if !hasResourceTracking {
t.Fatalf("expected resource tracking to remain when update status is unknown")
}
if !hasIdentityTracking {
t.Fatalf("expected identity tracking to remain when update status is unknown")
}
if wasResolved {
t.Fatalf("did not expect docker update alert to be marked resolved when update status is unknown")
}
if !resourceTrackedAt.Equal(firstSeen) {
t.Fatalf("expected resource tracking firstSeen to remain %s, got %s", firstSeen, resourceTrackedAt)
}
if !identityTrackedAt.Equal(firstSeen) {
t.Fatalf("expected identity tracking firstSeen to remain %s, got %s", firstSeen, identityTrackedAt)
}
if !alert.StartTime.Equal(firstSeen) {
t.Fatalf("expected alert StartTime to remain %s, got %s", firstSeen, alert.StartTime)
}
if !alert.LastSeen.After(previousLastSeen) {
t.Fatalf("expected alert LastSeen to advance beyond %s, got %s", previousLastSeen, alert.LastSeen)
}
})
}
}
// Note: Update alerts are now a free feature (no license gating).
// The Pro license gating tests were removed when update alerts became free.