fix(monitoring): clear poll lastError on success so healthy connections drop stale error banners

recordTaskResult reset the failure counters on a successful poll but left
LastErrorAt/LastErrorMessage/LastErrorCategory in place forever. The
connections aggregator surfaces that field as a live error banner on
Settings -> Infrastructure and derives the Unauthorized state from it, so
one transient cluster outage (or startup blip) pinned a red "no healthy
nodes available" banner onto a connection that polls fine, and a single
past 401 could pin a healthy connection to Unauthorized.

LastError now means "current outstanding failure since the last success":
recorded on failure, cleared on success. Connections that are genuinely
failing (no successful poll) keep their banner.

Refs #1493
This commit is contained in:
rcourtman 2026-06-10 16:19:24 +01:00
parent 02955619f5
commit 6eaea2fc3a
2 changed files with 20 additions and 0 deletions

View file

@ -2368,6 +2368,14 @@ func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, p
status.LastSuccess = now
status.ConsecutiveFailures = 0
status.FirstFailureAt = time.Time{}
// A recorded error means "current outstanding failure", not "last error
// ever". Clearing it on success keeps downstream consumers honest: the
// connections aggregator surfaces LastError as a live error banner and
// derives Unauthorized state from it, so a stale entry would show a
// red error on a healthy connection forever (#1493).
status.LastErrorAt = time.Time{}
status.LastErrorMessage = ""
status.LastErrorCategory = ""
m.mu.Unlock()
if breaker != nil {
breaker.recordSuccess()

View file

@ -1308,6 +1308,18 @@ func TestRecordTaskResult_SuccessResetsFailures(t *testing.T) {
if !m.pollStatusMap[key].FirstFailureAt.IsZero() {
t.Error("expected FirstFailureAt to be reset to zero")
}
// The recorded error must clear too: it means "current outstanding
// failure", and the connections UI renders it as a live error banner.
// A stale entry would show a red error on a healthy connection (#1493).
if m.pollStatusMap[key].LastErrorMessage != "" {
t.Errorf("expected LastErrorMessage to be cleared, got %q", m.pollStatusMap[key].LastErrorMessage)
}
if !m.pollStatusMap[key].LastErrorAt.IsZero() {
t.Error("expected LastErrorAt to be reset to zero")
}
if m.pollStatusMap[key].LastErrorCategory != "" {
t.Errorf("expected LastErrorCategory to be cleared, got %q", m.pollStatusMap[key].LastErrorCategory)
}
}
func TestRecordTaskResult_NilMaps(t *testing.T) {