handle repeated update-safety digest changes

This commit is contained in:
rcourtman 2026-05-13 04:18:27 +01:00
parent 5821b6c810
commit 4097ddcb9c
2 changed files with 69 additions and 11 deletions

View file

@ -105,13 +105,13 @@ func (w *UpdateSafetyWatcher) Observe(hosts []models.DockerHost, now time.Time)
continue
}
// Digest changed -- transition to state B.
snap.priorDigest = snap.digest
snap.changeDigest = c.ImageDigest
snap.baseRestarts = snap.restartCount
snap.priorDigest = snap.digest
snap.changeDigest = c.ImageDigest
snap.baseRestarts = snap.restartCount
snap.lastEmittedRestarts = snap.restartCount
snap.detectedAt = now
snap.digest = c.ImageDigest
snap.restartCount = c.RestartCount
snap.detectedAt = now
snap.digest = c.ImageDigest
snap.restartCount = c.RestartCount
severity := FindingSeverityInfo
if c.RestartCount > snap.baseRestarts {
@ -123,7 +123,28 @@ func (w *UpdateSafetyWatcher) Observe(hosts []models.DockerHost, now time.Time)
}
// State B: change already detected, verifying stability.
snap.digest = c.ImageDigest
if c.ImageDigest != snap.digest {
// Another image change landed before the prior verification
// window completed. Treat it as a fresh update and restart the
// window instead of resolving against stale evidence.
snap.priorDigest = snap.digest
snap.changeDigest = c.ImageDigest
snap.baseRestarts = snap.restartCount
snap.lastEmittedRestarts = snap.restartCount
snap.detectedAt = now
snap.digest = c.ImageDigest
snap.restartCount = c.RestartCount
severity := FindingSeverityInfo
if c.RestartCount > snap.baseRestarts {
severity = FindingSeverityWarning
snap.lastEmittedRestarts = c.RestartCount
}
emit = append(emit, buildUpdateSafetyFinding(key, host, c, snap, severity, now, now))
continue
}
snap.digest = c.ImageDigest
snap.restartCount = c.RestartCount
restartsAfterChange := c.RestartCount - snap.baseRestarts
@ -138,10 +159,10 @@ func (w *UpdateSafetyWatcher) Observe(hosts []models.DockerHost, now time.Time)
// Stable for the full window -- emit resolve sentinel and reset.
dedupKey := UpdateSafetyFindingPrefix + ":" + key
resolve = append(resolve, resolveSentinel{DedupKey: dedupKey, Reason: updateSafetyResolveReason})
snap.detectedAt = time.Time{}
snap.priorDigest = ""
snap.changeDigest = ""
snap.baseRestarts = 0
snap.detectedAt = time.Time{}
snap.priorDigest = ""
snap.changeDigest = ""
snap.baseRestarts = 0
snap.lastEmittedRestarts = 0
}
// Otherwise: still in window, no new restarts -- do nothing.

View file

@ -138,6 +138,43 @@ func TestUpdateSafety_StableWindowEmitsResolveSentinel(t *testing.T) {
}
}
// TestUpdateSafety_SecondDigestChangeResetsVerificationWindow verifies that a
// second image update during the verification window restarts the window and
// updates the existing finding evidence rather than resolving the old update.
func TestUpdateSafety_SecondDigestChangeResetsVerificationWindow(t *testing.T) {
w := newUpdateSafetyWatcher()
t0 := time.Now()
w.Observe([]models.DockerHost{makeHost("h1", "c1", "sha256:aaa", 0)}, t0)
emit1, resolve1 := w.Observe([]models.DockerHost{makeHost("h1", "c1", "sha256:bbb", 0)}, t0.Add(5*time.Second))
if len(emit1) != 1 || len(resolve1) != 0 {
t.Fatalf("first update: want 1 emit and 0 resolves, got emit=%d resolve=%d", len(emit1), len(resolve1))
}
secondUpdateAt := t0.Add(20 * time.Second)
emit2, resolve2 := w.Observe([]models.DockerHost{makeHost("h1", "c1", "sha256:ccc", 0)}, secondUpdateAt)
if len(emit2) != 1 || len(resolve2) != 0 {
t.Fatalf("second update: want 1 emit and 0 resolves, got emit=%d resolve=%d", len(emit2), len(resolve2))
}
if emit2[0].Evidence != "prior_digest=sha256:bbb new_digest=sha256:ccc restart_count=0" {
t.Fatalf("second update evidence = %q", emit2[0].Evidence)
}
// The first update's window has elapsed, but the second update's window has
// not. Resolving now would close against stale evidence.
oldWindowElapsed := t0.Add(5*time.Second + updateSafetyVerifyWindow + time.Second)
emit3, resolve3 := w.Observe([]models.DockerHost{makeHost("h1", "c1", "sha256:ccc", 0)}, oldWindowElapsed)
if len(emit3) != 0 || len(resolve3) != 0 {
t.Fatalf("old window elapsed: want silent, got emit=%d resolve=%d", len(emit3), len(resolve3))
}
emit4, resolve4 := w.Observe([]models.DockerHost{makeHost("h1", "c1", "sha256:ccc", 0)}, secondUpdateAt.Add(updateSafetyVerifyWindow+time.Second))
if len(emit4) != 0 || len(resolve4) != 1 {
t.Fatalf("second window elapsed: want 0 emit and 1 resolve, got emit=%d resolve=%d", len(emit4), len(resolve4))
}
}
// TestUpdateSafety_EmptyDigestEmitsNothing verifies that containers with an
// empty ImageDigest (agent not yet reporting one) are silently skipped.
func TestUpdateSafety_EmptyDigestEmitsNothing(t *testing.T) {