mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-03 04:38:48 +00:00
Integrate reviewed local-clock quiet-hours repair
Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
commit
452ca8ed10
4 changed files with 318 additions and 77 deletions
|
|
@ -1982,6 +1982,20 @@ keep schedules such as `00:00` to `23:59` active through the full final
|
|||
minute instead of expiring at `23:59:00`. Alert quiet-hours proofs should
|
||||
control time through the alert manager clock hook instead of depending on wall
|
||||
clock execution at whatever second the test runner happens to hit.
|
||||
The schedule compares local civil clock minutes on the selected calendar day,
|
||||
not `time.Date` instants. Both occurrences of a repeated minute have the same
|
||||
quiet-hours policy; nonexistent clock minutes do not shift the configured
|
||||
start or end. Initial queued replay uses the first real minute outside the
|
||||
current non-full-day window, including an unselected day reached at midnight.
|
||||
This does not reinterpret the day toggles as the previous evening's ownership.
|
||||
Full-day windows retain the existing daily replay boundary; this clock repair
|
||||
is not current-policy revalidation of already queued work or external receipt.
|
||||
`TestQuietHoursDaylightSavingClockAndReplay` in
|
||||
`internal/alerts/quiet_hours_test.go` checks London, New York and Lord Howe
|
||||
(one-hour and half-hour changes), inclusive boundaries and exact UTC replay.
|
||||
`TestQuietHoursOvernightSelectedCalendarDays` checks overnight day selection.
|
||||
Quiet-hours location fallback is read-only; configuration updates own the
|
||||
location cache, including when public suppression helpers run concurrently.
|
||||
Quiet-hours suppression also applies to alert delivery lifecycle, not only the
|
||||
initial raised notification. Resolved notifications must not fan out when the
|
||||
alert was never notified or was already acknowledged, and monitoring-driven
|
||||
|
|
|
|||
|
|
@ -2778,6 +2778,7 @@
|
|||
"internal/alerts/incident_synthesis_test.go",
|
||||
"internal/alerts/intent_policy_test.go",
|
||||
"internal/alerts/operational_contract_test.go",
|
||||
"internal/alerts/quiet_hours_test.go",
|
||||
"internal/alerts/reducer/ack_test.go",
|
||||
"internal/alerts/reducer/discrete_test.go",
|
||||
"internal/alerts/reducer/intent_test.go",
|
||||
|
|
|
|||
|
|
@ -385,70 +385,66 @@ func isSupportedInfrastructureSymptom(alert *Alert) bool {
|
|||
strings.TrimSpace(alert.Correlation.PrimaryAlertID) != ""
|
||||
}
|
||||
|
||||
// isInQuietHours checks if the current time is within quiet hours
|
||||
func (m *Manager) isInQuietHours() bool {
|
||||
if !m.config.Schedule.QuietHours.Enabled {
|
||||
return false
|
||||
}
|
||||
// quietHoursClock compares civil clock minutes, not date-constructed instants.
|
||||
// time.Date chooses only one side of a repeated hour and normalizes missing
|
||||
// minutes to a different hour, neither of which is the user's daily schedule.
|
||||
type quietHoursClock struct {
|
||||
location *time.Location
|
||||
start int
|
||||
end int
|
||||
days [7]bool
|
||||
}
|
||||
|
||||
// Use cached location if available
|
||||
func (m *Manager) quietHoursClock() (quietHoursClock, bool) {
|
||||
quiet := m.config.Schedule.QuietHours
|
||||
if !quiet.Enabled {
|
||||
return quietHoursClock{}, false
|
||||
}
|
||||
loc := m.quietHoursLoc
|
||||
if loc == nil {
|
||||
// Fallback to loading if not cached yet (shouldn't happen with UpdateConfig)
|
||||
var err error
|
||||
loc, err = time.LoadLocation(m.config.Schedule.QuietHours.Timezone)
|
||||
loc, err = time.LoadLocation(quiet.Timezone)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("timezone", m.config.Schedule.QuietHours.Timezone).Msg("failed to load timezone, using local time")
|
||||
log.Warn().Err(err).Str("timezone", quiet.Timezone).Msg("failed to load timezone, using local time")
|
||||
loc = time.Local
|
||||
}
|
||||
m.quietHoursLoc = loc
|
||||
// Do not populate the cache here: public suppression helpers can call
|
||||
// this path while holding only a read lock. UpdateConfig owns the cache.
|
||||
}
|
||||
|
||||
nowFn := m.now
|
||||
if nowFn == nil {
|
||||
nowFn = time.Now
|
||||
}
|
||||
now := nowFn().In(loc).Truncate(time.Minute)
|
||||
dayName := strings.ToLower(now.Format("Monday"))
|
||||
|
||||
// Check if today is enabled for quiet hours
|
||||
if enabled, ok := m.config.Schedule.QuietHours.Days[dayName]; !ok || !enabled {
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse start and end times
|
||||
startTime, err := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.Start, loc)
|
||||
start, err := time.Parse("15:04", quiet.Start)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("start", m.config.Schedule.QuietHours.Start).Msg("failed to parse quiet hours start time")
|
||||
return false
|
||||
log.Warn().Err(err).Str("start", quiet.Start).Msg("failed to parse quiet hours start time")
|
||||
return quietHoursClock{}, false
|
||||
}
|
||||
|
||||
endTime, err := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.End, loc)
|
||||
end, err := time.Parse("15:04", quiet.End)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("end", m.config.Schedule.QuietHours.End).Msg("failed to parse quiet hours end time")
|
||||
log.Warn().Err(err).Str("end", quiet.End).Msg("failed to parse quiet hours end time")
|
||||
return quietHoursClock{}, false
|
||||
}
|
||||
clock := quietHoursClock{location: loc, start: start.Hour()*60 + start.Minute(), end: end.Hour()*60 + end.Minute()}
|
||||
for i, day := range [...]string{"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"} {
|
||||
clock.days[i] = quiet.Days[day]
|
||||
}
|
||||
return clock, true
|
||||
}
|
||||
|
||||
func (clock quietHoursClock) contains(at time.Time) bool {
|
||||
local := at.In(clock.location)
|
||||
if !clock.days[local.Weekday()] {
|
||||
return false
|
||||
}
|
||||
|
||||
// Set to today's date
|
||||
startTime = time.Date(now.Year(), now.Month(), now.Day(), startTime.Hour(), startTime.Minute(), 0, 0, loc)
|
||||
endTime = time.Date(now.Year(), now.Month(), now.Day(), endTime.Hour(), endTime.Minute(), 0, 0, loc)
|
||||
|
||||
// Quiet hours are configured with minute precision, so treat the start and
|
||||
// end minute as inclusive for user-facing schedules such as 00:00-23:59.
|
||||
endExclusive := endTime.Add(time.Minute)
|
||||
|
||||
// Handle overnight quiet hours (e.g., 22:00 to 08:00)
|
||||
if endTime.Before(startTime) {
|
||||
if !now.Before(startTime) || now.Before(endExclusive) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
if !now.Before(startTime) && now.Before(endExclusive) {
|
||||
return true
|
||||
}
|
||||
minute := local.Hour()*60 + local.Minute()
|
||||
if clock.start > clock.end {
|
||||
return minute >= clock.start || minute <= clock.end
|
||||
}
|
||||
return minute >= clock.start && minute <= clock.end
|
||||
}
|
||||
|
||||
return false
|
||||
// isInQuietHours checks the selected local calendar day and includes the whole
|
||||
// configured end minute, including both copies of it during a clock rollback.
|
||||
func (m *Manager) isInQuietHours() bool {
|
||||
clock, valid := m.quietHoursClock()
|
||||
return valid && clock.contains(m.policyNow())
|
||||
}
|
||||
|
||||
func quietHoursCategoryForAlert(alert *Alert) string {
|
||||
|
|
@ -518,41 +514,42 @@ func (m *Manager) shouldSuppressNotification(alert *Alert) (bool, string) {
|
|||
}
|
||||
|
||||
func (m *Manager) quietHoursReplayAt() time.Time {
|
||||
nowFn := m.now
|
||||
if nowFn == nil {
|
||||
nowFn = time.Now
|
||||
now := m.policyNow()
|
||||
clock, valid := m.quietHoursClock()
|
||||
if !valid || !clock.contains(now) {
|
||||
return now.Add(time.Minute).UTC()
|
||||
}
|
||||
now := nowFn()
|
||||
|
||||
loc := m.quietHoursLoc
|
||||
if loc == nil {
|
||||
var err error
|
||||
loc, err = time.LoadLocation(m.config.Schedule.QuietHours.Timezone)
|
||||
if err != nil {
|
||||
log.Warn().Err(err).Str("timezone", m.config.Schedule.QuietHours.Timezone).Msg("failed to load timezone for quiet-hours replay, using local time")
|
||||
loc = time.Local
|
||||
// A full-day window has no non-quiet clock minute. Preserve its existing
|
||||
// daily replay boundary; queued-policy revalidation is a separate delivery
|
||||
// concern, not permission to invent an end to a continuous schedule.
|
||||
if (clock.end-clock.start+24*60)%(24*60) == 24*60-1 {
|
||||
local := now.In(clock.location)
|
||||
endExclusive := time.Date(local.Year(), local.Month(), local.Day(), clock.end/60, clock.end%60, 0, 0, clock.location).Add(time.Minute)
|
||||
if clock.start > clock.end && local.Hour()*60+local.Minute() >= clock.start {
|
||||
endExclusive = endExclusive.AddDate(0, 0, 1)
|
||||
}
|
||||
if endExclusive.After(now) {
|
||||
return endExclusive.UTC()
|
||||
}
|
||||
m.quietHoursLoc = loc
|
||||
}
|
||||
|
||||
localNow := now.In(loc).Truncate(time.Minute)
|
||||
startTime, startErr := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.Start, loc)
|
||||
endTime, endErr := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.End, loc)
|
||||
if startErr != nil || endErr != nil {
|
||||
return now.Add(time.Minute).UTC()
|
||||
}
|
||||
|
||||
startTime = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), startTime.Hour(), startTime.Minute(), 0, 0, loc)
|
||||
endTime = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), endTime.Hour(), endTime.Minute(), 0, 0, loc)
|
||||
endExclusive := endTime.Add(time.Minute)
|
||||
if endTime.Before(startTime) && !localNow.Before(startTime) {
|
||||
endExclusive = endExclusive.AddDate(0, 0, 1)
|
||||
// Walk real minutes through the current window, rather than constructing
|
||||
// its end on a local date. This handles repeated ends, skipped ends, half-
|
||||
// hour DST transitions and a midnight leading into an unselected day. A
|
||||
// non-full-day window has an exit within three real days even when a
|
||||
// forward clock change skips its entire gap on the first day.
|
||||
candidate := now.Truncate(time.Minute).Add(time.Minute)
|
||||
limit := candidate.Add(72 * time.Hour)
|
||||
for candidate.Before(limit) {
|
||||
if !clock.contains(candidate) {
|
||||
return candidate.UTC()
|
||||
}
|
||||
candidate = candidate.Add(time.Minute)
|
||||
}
|
||||
|
||||
if !endExclusive.After(localNow) {
|
||||
return now.Add(time.Minute).UTC()
|
||||
}
|
||||
return endExclusive.UTC()
|
||||
log.Warn().Msg("could not find quiet-hours window end within three days")
|
||||
return now.Add(time.Minute).UTC()
|
||||
}
|
||||
|
||||
func markQuietHoursNotificationReplay(alert *Alert, reason string, replayAt time.Time) {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package alerts
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
|
|
@ -391,3 +392,231 @@ func TestIsInQuietHours(t *testing.T) {
|
|||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Local-clock schedules apply to both copies of a repeated minute. Missing
|
||||
// minutes do not move a configured start or end into some other wall-clock hour.
|
||||
func TestQuietHoursDaylightSavingClockAndReplay(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, zone, start, end, now, replay string
|
||||
quiet bool
|
||||
}{
|
||||
{"london-first-repeated-hour", "Europe/London", "01:00", "02:00", "2026-10-25T00:30:00Z", "2026-10-25T02:01:00Z", true},
|
||||
{"london-second-repeated-hour", "Europe/London", "01:00", "02:00", "2026-10-25T01:30:00Z", "2026-10-25T02:01:00Z", true},
|
||||
{"london-first-repeated-end", "Europe/London", "00:00", "01:30", "2026-10-25T00:15:00Z", "2026-10-25T00:31:00Z", true},
|
||||
{"london-second-repeated-end", "Europe/London", "00:00", "01:30", "2026-10-25T01:15:00Z", "2026-10-25T01:31:00Z", true},
|
||||
{"london-after-first-end", "Europe/London", "00:00", "01:30", "2026-10-25T00:45:00Z", "", false},
|
||||
{"london-after-second-end", "Europe/London", "00:00", "01:30", "2026-10-25T01:45:00Z", "", false},
|
||||
{"london-missing-start", "Europe/London", "01:30", "02:30", "2026-03-29T01:15:00Z", "2026-03-29T01:31:00Z", true},
|
||||
{"london-missing-end-replay", "Europe/London", "00:00", "01:30", "2026-03-29T00:55:30Z", "2026-03-29T01:00:00Z", true},
|
||||
{"london-after-missing-end", "Europe/London", "00:00", "01:30", "2026-03-29T01:15:00Z", "", false},
|
||||
{"london-overnight-missing-end", "Europe/London", "22:00", "01:30", "2026-03-28T23:00:00Z", "2026-03-29T01:00:00Z", true},
|
||||
{"london-overnight-repeated-end", "Europe/London", "22:00", "01:30", "2026-10-24T23:15:00Z", "2026-10-25T00:31:00Z", true},
|
||||
{"london-end-minute-before-jump", "Europe/London", "00:00", "00:59", "2026-03-29T00:59:59Z", "2026-03-29T01:00:00Z", true},
|
||||
{"new-york-first-repeated-end", "America/New_York", "00:00", "01:30", "2026-11-01T05:15:00Z", "2026-11-01T05:31:00Z", true},
|
||||
{"new-york-second-repeated-end", "America/New_York", "00:00", "01:30", "2026-11-01T06:15:00Z", "2026-11-01T06:31:00Z", true},
|
||||
{"new-york-after-first-end", "America/New_York", "00:00", "01:30", "2026-11-01T05:45:00Z", "", false},
|
||||
{"new-york-after-second-end", "America/New_York", "00:00", "01:30", "2026-11-01T06:45:00Z", "", false},
|
||||
{"new-york-missing-start", "America/New_York", "02:30", "03:30", "2026-03-08T07:15:00Z", "2026-03-08T07:31:00Z", true},
|
||||
{"new-york-missing-end", "America/New_York", "00:00", "02:30", "2026-03-08T06:55:00Z", "2026-03-08T07:00:00Z", true},
|
||||
{"new-york-after-missing-end", "America/New_York", "00:00", "02:30", "2026-03-08T07:15:00Z", "", false},
|
||||
{"lord-howe-first-repeated-half-hour", "Australia/Lord_Howe", "01:30", "02:00", "2026-04-04T14:40:00Z", "2026-04-04T15:31:00Z", true},
|
||||
{"lord-howe-second-repeated-half-hour", "Australia/Lord_Howe", "01:30", "02:00", "2026-04-04T15:10:00Z", "2026-04-04T15:31:00Z", true},
|
||||
{"lord-howe-missing-start", "Australia/Lord_Howe", "02:00", "02:45", "2026-10-03T15:35:00Z", "2026-10-03T15:46:00Z", true},
|
||||
{"lord-howe-after-missing-end", "Australia/Lord_Howe", "00:00", "02:15", "2026-10-03T15:40:00Z", "", false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
now, err := time.Parse(time.RFC3339, tc.now)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := newTestManager(t)
|
||||
m.now = func() time.Time { return now }
|
||||
cfg := m.GetConfig()
|
||||
cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: tc.zone,
|
||||
Days: map[string]bool{"monday": true, "tuesday": true, "wednesday": true, "thursday": true, "friday": true, "saturday": true, "sunday": true}}
|
||||
m.UpdateConfig(cfg)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if got := m.isInQuietHours(); got != tc.quiet {
|
||||
t.Errorf("quiet at %s for %s %s-%s = %v, want %v", now, tc.zone, tc.start, tc.end, got, tc.quiet)
|
||||
}
|
||||
if tc.replay == "" {
|
||||
return
|
||||
}
|
||||
want, err := time.Parse(time.RFC3339, tc.replay)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := m.quietHoursReplayAt()
|
||||
if !got.Equal(want) {
|
||||
t.Errorf("replay = %s, want first eligible real minute %s", got, want)
|
||||
}
|
||||
if !got.After(now) {
|
||||
t.Error("replay must be in the future")
|
||||
}
|
||||
now = want.Add(-time.Nanosecond)
|
||||
if !m.isInQuietHours() {
|
||||
t.Error("quiet hours ended before the final included minute finished")
|
||||
}
|
||||
now = want
|
||||
if m.isInQuietHours() {
|
||||
t.Error("computed replay minute is still quiet")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietHoursOvernightSelectedCalendarDays(t *testing.T) {
|
||||
m := newTestManager(t)
|
||||
now := time.Date(2026, 10, 2, 23, 30, 0, 0, time.UTC) // Friday
|
||||
m.now = func() time.Time { return now }
|
||||
cfg := m.GetConfig()
|
||||
cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: "22:00", End: "06:00", Timezone: "UTC", Days: map[string]bool{"friday": true}}
|
||||
m.UpdateConfig(cfg)
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if !m.isInQuietHours() {
|
||||
t.Fatal("selected Friday evening must be quiet")
|
||||
}
|
||||
want := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC)
|
||||
if got := m.quietHoursReplayAt(); !got.Equal(want) {
|
||||
t.Fatalf("unselected Saturday must end suppression at midnight: got %s, want %s", got, want)
|
||||
}
|
||||
now = want
|
||||
if m.isInQuietHours() {
|
||||
t.Fatal("overnight settings must not silently enable an unselected day")
|
||||
}
|
||||
now = now.AddDate(0, 0, 6).Add(6*time.Hour + 59*time.Second) // Friday's inclusive 06:00 minute
|
||||
if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(now.Truncate(time.Minute).Add(time.Minute)) {
|
||||
t.Fatal("selected morning must keep the inclusive end minute")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietHoursReplaySkipsNonexistentGap(t *testing.T) {
|
||||
// This overnight window has a one-minute daily gap, 01:29. London's
|
||||
// spring jump skips that entire gap, so the next real exit is Monday.
|
||||
now := time.Date(2026, 3, 28, 23, 0, 0, 0, time.UTC)
|
||||
m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: "01:30", End: "01:28", Timezone: "Europe/London",
|
||||
Days: map[string]bool{"saturday": true, "sunday": true, "monday": true}})
|
||||
want := time.Date(2026, 3, 30, 0, 29, 0, 0, time.UTC)
|
||||
if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(want) {
|
||||
t.Fatalf("skipped gap must not release a notification into quiet hours: replay=%s want=%s", m.quietHoursReplayAt(), want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietHoursFullDayRetainsDailyReplayBoundary(t *testing.T) {
|
||||
for _, tc := range []struct{ start, end, now, replay string }{
|
||||
{"00:00", "23:59", "2026-10-01T12:30:00Z", "2026-10-02T00:00:00Z"},
|
||||
{"22:00", "21:59", "2026-10-01T23:30:00Z", "2026-10-02T22:00:00Z"},
|
||||
{"22:00", "21:59", "2026-10-01T12:30:00Z", "2026-10-01T22:00:00Z"},
|
||||
} {
|
||||
t.Run(tc.start+"/"+tc.now, func(t *testing.T) {
|
||||
now, _ := time.Parse(time.RFC3339, tc.now)
|
||||
want, _ := time.Parse(time.RFC3339, tc.replay)
|
||||
m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: "UTC", Days: map[string]bool{"thursday": true, "friday": true}})
|
||||
if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(want) {
|
||||
t.Fatalf("daily replay changed: %s, want %s", m.quietHoursReplayAt(), want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietHoursClockDispatchAndDiagnosis(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name, now, start, end, replay string
|
||||
quiet bool
|
||||
}{
|
||||
{"repeated-hour", "2026-10-25T00:30:00Z", "01:00", "02:00", "2026-10-25T02:01:00Z", true},
|
||||
{"skipped-end", "2026-03-29T00:55:00Z", "00:00", "01:30", "2026-03-29T01:00:00Z", true},
|
||||
{"after-skipped-end", "2026-03-29T01:15:00Z", "00:00", "01:30", "", false},
|
||||
} {
|
||||
for _, critical := range []bool{false, true} {
|
||||
name := tc.name + "/warning"
|
||||
if critical {
|
||||
name = tc.name + "/unsuppressed-critical"
|
||||
}
|
||||
t.Run(name, func(t *testing.T) {
|
||||
now, _ := time.Parse(time.RFC3339, tc.now)
|
||||
m := newTestManager(t)
|
||||
m.now = func() time.Time { return now }
|
||||
cfg := m.GetConfig()
|
||||
cfg.Enabled = true
|
||||
cfg.ActivationState = ActivationActive
|
||||
cfg.FlappingEnabled = false
|
||||
cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: "Europe/London", Days: map[string]bool{"sunday": true}}
|
||||
m.UpdateConfig(cfg)
|
||||
_, alert := testNewCanonicalAlert("vm-100", "vm-100-cpu", "vm", "cpu")
|
||||
alert.Level = AlertLevelWarning
|
||||
if critical {
|
||||
alert.Level = AlertLevelCritical
|
||||
}
|
||||
alert.StartTime = now
|
||||
alert.LastSeen = now
|
||||
// Old metadata must be cleared when quiet hours no longer apply.
|
||||
alert.Metadata = map[string]interface{}{MetadataQuietHoursSuppressed: true, MetadataQuietHoursReplayAt: "2099-01-01T00:00:00Z"}
|
||||
m.mu.Lock()
|
||||
m.setActiveAlertNoLock(alert.ID, alert)
|
||||
m.mu.Unlock()
|
||||
deferred := tc.quiet && !critical
|
||||
diagnosis, found := m.DiagnoseAlertDelivery(alert.ID)
|
||||
if !found {
|
||||
t.Fatal("active occurrence unavailable")
|
||||
}
|
||||
if deferred {
|
||||
want, _ := time.Parse(time.RFC3339, tc.replay)
|
||||
if diagnosis.Status != AlertDeliveryStatusDeferred || diagnosis.Reason != AlertDeliveryReasonQuietHours+":non-critical" || diagnosis.QuietHoursReplayAt == nil || !diagnosis.QuietHoursReplayAt.Equal(want) {
|
||||
t.Fatalf("diagnosis disagrees with local-clock policy: %+v", diagnosis)
|
||||
}
|
||||
} else if diagnosis.Status != AlertDeliveryStatusWouldSend || diagnosis.Reason != AlertDeliveryReasonReady {
|
||||
t.Fatalf("eligible alert was diagnosed as held: %+v", diagnosis)
|
||||
}
|
||||
var dispatched *Alert
|
||||
m.SetAlertCallback(func(a *Alert) { dispatched = a })
|
||||
m.mu.Lock()
|
||||
admitted := m.dispatchAlert(alert, false)
|
||||
m.mu.Unlock()
|
||||
if !admitted || dispatched == nil {
|
||||
t.Fatal("quiet-hours replay must enter the existing delivery pipeline, not be dropped")
|
||||
}
|
||||
if deferred {
|
||||
if dispatched.Metadata[MetadataQuietHoursSuppressed] != true || dispatched.Metadata[MetadataQuietHoursReplayAt] != tc.replay {
|
||||
t.Fatalf("pipeline replay timestamp = %#v, want %s", dispatched.Metadata, tc.replay)
|
||||
}
|
||||
} else if hasQuietHoursNotificationReplay(dispatched) {
|
||||
t.Fatalf("eligible alert kept stale deferral metadata: %#v", dispatched.Metadata)
|
||||
}
|
||||
// Escalation/bypass sends consult this same public helper.
|
||||
bypass := cloneAlertForOutput(dispatched)
|
||||
if m.ShouldSuppressNotification(bypass) || hasQuietHoursNotificationReplay(bypass) != deferred {
|
||||
t.Fatal("public bypass helper did not apply the same clock policy")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuietHoursConcurrentUncachedReaders(t *testing.T) {
|
||||
now := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC)
|
||||
m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: "01:00", End: "02:00", Timezone: "Europe/London", Days: map[string]bool{"sunday": true}})
|
||||
start := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
for worker := 0; worker < 32; worker++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
<-start
|
||||
for i := 0; i < 25; i++ {
|
||||
alert := &Alert{ID: "uncached-reader", Type: "cpu", Level: AlertLevelWarning}
|
||||
if m.ShouldSuppressNotification(alert) || alert.Metadata[MetadataQuietHoursReplayAt] != "2026-10-25T02:01:00Z" {
|
||||
t.Error("concurrent reader disagrees with quiet-hours replay")
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
close(start)
|
||||
wg.Wait()
|
||||
if m.quietHoursLoc != nil {
|
||||
t.Fatal("read-locked evaluation mutated the configuration-owned location cache")
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue