diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index bc7c4bc4a..eb93915d9 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1982,6 +1982,20 @@ keep schedules such as `00:00` to `23:59` active through the full final minute instead of expiring at `23:59:00`. Alert quiet-hours proofs should control time through the alert manager clock hook instead of depending on wall clock execution at whatever second the test runner happens to hit. +The schedule compares local civil clock minutes on the selected calendar day, +not `time.Date` instants. Both occurrences of a repeated minute have the same +quiet-hours policy; nonexistent clock minutes do not shift the configured +start or end. Initial queued replay uses the first real minute outside the +current non-full-day window, including an unselected day reached at midnight. +This does not reinterpret the day toggles as the previous evening's ownership. +Full-day windows retain the existing daily replay boundary; this clock repair +is not current-policy revalidation of already queued work or external receipt. +`TestQuietHoursDaylightSavingClockAndReplay` in +`internal/alerts/quiet_hours_test.go` checks London, New York and Lord Howe +(one-hour and half-hour changes), inclusive boundaries and exact UTC replay. +`TestQuietHoursOvernightSelectedCalendarDays` checks overnight day selection. +Quiet-hours location fallback is read-only; configuration updates own the +location cache, including when public suppression helpers run concurrently. Quiet-hours suppression also applies to alert delivery lifecycle, not only the initial raised notification. Resolved notifications must not fan out when the alert was never notified or was already acknowledged, and monitoring-driven diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 788a92cd5..15da522e0 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -2756,6 +2756,7 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ + "internal/alerts/quiet_hours_test.go", "internal/alerts/alert_admission_test.go", "internal/alerts/alerts_test.go", "internal/alerts/callback_config_coverage_test.go", diff --git a/internal/alerts/notification_policy.go b/internal/alerts/notification_policy.go index 24a3b4951..cfa5d1950 100644 --- a/internal/alerts/notification_policy.go +++ b/internal/alerts/notification_policy.go @@ -385,70 +385,66 @@ func isSupportedInfrastructureSymptom(alert *Alert) bool { strings.TrimSpace(alert.Correlation.PrimaryAlertID) != "" } -// isInQuietHours checks if the current time is within quiet hours -func (m *Manager) isInQuietHours() bool { - if !m.config.Schedule.QuietHours.Enabled { - return false - } +// quietHoursClock compares civil clock minutes, not date-constructed instants. +// time.Date chooses only one side of a repeated hour and normalizes missing +// minutes to a different hour, neither of which is the user's daily schedule. +type quietHoursClock struct { + location *time.Location + start int + end int + days [7]bool +} - // Use cached location if available +func (m *Manager) quietHoursClock() (quietHoursClock, bool) { + quiet := m.config.Schedule.QuietHours + if !quiet.Enabled { + return quietHoursClock{}, false + } loc := m.quietHoursLoc if loc == nil { - // Fallback to loading if not cached yet (shouldn't happen with UpdateConfig) var err error - loc, err = time.LoadLocation(m.config.Schedule.QuietHours.Timezone) + loc, err = time.LoadLocation(quiet.Timezone) if err != nil { - log.Warn().Err(err).Str("timezone", m.config.Schedule.QuietHours.Timezone).Msg("failed to load timezone, using local time") + log.Warn().Err(err).Str("timezone", quiet.Timezone).Msg("failed to load timezone, using local time") loc = time.Local } - m.quietHoursLoc = loc + // Do not populate the cache here: public suppression helpers can call + // this path while holding only a read lock. UpdateConfig owns the cache. } - - nowFn := m.now - if nowFn == nil { - nowFn = time.Now - } - now := nowFn().In(loc).Truncate(time.Minute) - dayName := strings.ToLower(now.Format("Monday")) - - // Check if today is enabled for quiet hours - if enabled, ok := m.config.Schedule.QuietHours.Days[dayName]; !ok || !enabled { - return false - } - - // Parse start and end times - startTime, err := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.Start, loc) + start, err := time.Parse("15:04", quiet.Start) if err != nil { - log.Warn().Err(err).Str("start", m.config.Schedule.QuietHours.Start).Msg("failed to parse quiet hours start time") - return false + log.Warn().Err(err).Str("start", quiet.Start).Msg("failed to parse quiet hours start time") + return quietHoursClock{}, false } - - endTime, err := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.End, loc) + end, err := time.Parse("15:04", quiet.End) if err != nil { - log.Warn().Err(err).Str("end", m.config.Schedule.QuietHours.End).Msg("failed to parse quiet hours end time") + log.Warn().Err(err).Str("end", quiet.End).Msg("failed to parse quiet hours end time") + return quietHoursClock{}, false + } + clock := quietHoursClock{location: loc, start: start.Hour()*60 + start.Minute(), end: end.Hour()*60 + end.Minute()} + for i, day := range [...]string{"sunday", "monday", "tuesday", "wednesday", "thursday", "friday", "saturday"} { + clock.days[i] = quiet.Days[day] + } + return clock, true +} + +func (clock quietHoursClock) contains(at time.Time) bool { + local := at.In(clock.location) + if !clock.days[local.Weekday()] { return false } - - // Set to today's date - startTime = time.Date(now.Year(), now.Month(), now.Day(), startTime.Hour(), startTime.Minute(), 0, 0, loc) - endTime = time.Date(now.Year(), now.Month(), now.Day(), endTime.Hour(), endTime.Minute(), 0, 0, loc) - - // Quiet hours are configured with minute precision, so treat the start and - // end minute as inclusive for user-facing schedules such as 00:00-23:59. - endExclusive := endTime.Add(time.Minute) - - // Handle overnight quiet hours (e.g., 22:00 to 08:00) - if endTime.Before(startTime) { - if !now.Before(startTime) || now.Before(endExclusive) { - return true - } - } else { - if !now.Before(startTime) && now.Before(endExclusive) { - return true - } + minute := local.Hour()*60 + local.Minute() + if clock.start > clock.end { + return minute >= clock.start || minute <= clock.end } + return minute >= clock.start && minute <= clock.end +} - return false +// isInQuietHours checks the selected local calendar day and includes the whole +// configured end minute, including both copies of it during a clock rollback. +func (m *Manager) isInQuietHours() bool { + clock, valid := m.quietHoursClock() + return valid && clock.contains(m.policyNow()) } func quietHoursCategoryForAlert(alert *Alert) string { @@ -518,41 +514,42 @@ func (m *Manager) shouldSuppressNotification(alert *Alert) (bool, string) { } func (m *Manager) quietHoursReplayAt() time.Time { - nowFn := m.now - if nowFn == nil { - nowFn = time.Now + now := m.policyNow() + clock, valid := m.quietHoursClock() + if !valid || !clock.contains(now) { + return now.Add(time.Minute).UTC() } - now := nowFn() - loc := m.quietHoursLoc - if loc == nil { - var err error - loc, err = time.LoadLocation(m.config.Schedule.QuietHours.Timezone) - if err != nil { - log.Warn().Err(err).Str("timezone", m.config.Schedule.QuietHours.Timezone).Msg("failed to load timezone for quiet-hours replay, using local time") - loc = time.Local + // A full-day window has no non-quiet clock minute. Preserve its existing + // daily replay boundary; queued-policy revalidation is a separate delivery + // concern, not permission to invent an end to a continuous schedule. + if (clock.end-clock.start+24*60)%(24*60) == 24*60-1 { + local := now.In(clock.location) + endExclusive := time.Date(local.Year(), local.Month(), local.Day(), clock.end/60, clock.end%60, 0, 0, clock.location).Add(time.Minute) + if clock.start > clock.end && local.Hour()*60+local.Minute() >= clock.start { + endExclusive = endExclusive.AddDate(0, 0, 1) + } + if endExclusive.After(now) { + return endExclusive.UTC() } - m.quietHoursLoc = loc - } - - localNow := now.In(loc).Truncate(time.Minute) - startTime, startErr := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.Start, loc) - endTime, endErr := time.ParseInLocation("15:04", m.config.Schedule.QuietHours.End, loc) - if startErr != nil || endErr != nil { return now.Add(time.Minute).UTC() } - startTime = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), startTime.Hour(), startTime.Minute(), 0, 0, loc) - endTime = time.Date(localNow.Year(), localNow.Month(), localNow.Day(), endTime.Hour(), endTime.Minute(), 0, 0, loc) - endExclusive := endTime.Add(time.Minute) - if endTime.Before(startTime) && !localNow.Before(startTime) { - endExclusive = endExclusive.AddDate(0, 0, 1) + // Walk real minutes through the current window, rather than constructing + // its end on a local date. This handles repeated ends, skipped ends, half- + // hour DST transitions and a midnight leading into an unselected day. A + // non-full-day window has an exit within three real days even when a + // forward clock change skips its entire gap on the first day. + candidate := now.Truncate(time.Minute).Add(time.Minute) + limit := candidate.Add(72 * time.Hour) + for candidate.Before(limit) { + if !clock.contains(candidate) { + return candidate.UTC() + } + candidate = candidate.Add(time.Minute) } - - if !endExclusive.After(localNow) { - return now.Add(time.Minute).UTC() - } - return endExclusive.UTC() + log.Warn().Msg("could not find quiet-hours window end within three days") + return now.Add(time.Minute).UTC() } func markQuietHoursNotificationReplay(alert *Alert, reason string, replayAt time.Time) { diff --git a/internal/alerts/quiet_hours_test.go b/internal/alerts/quiet_hours_test.go index 0d957d7c6..e8176acc3 100644 --- a/internal/alerts/quiet_hours_test.go +++ b/internal/alerts/quiet_hours_test.go @@ -1,6 +1,7 @@ package alerts import ( + "sync" "testing" "time" @@ -391,3 +392,231 @@ func TestIsInQuietHours(t *testing.T) { } }) } + +// Local-clock schedules apply to both copies of a repeated minute. Missing +// minutes do not move a configured start or end into some other wall-clock hour. +func TestQuietHoursDaylightSavingClockAndReplay(t *testing.T) { + for _, tc := range []struct { + name, zone, start, end, now, replay string + quiet bool + }{ + {"london-first-repeated-hour", "Europe/London", "01:00", "02:00", "2026-10-25T00:30:00Z", "2026-10-25T02:01:00Z", true}, + {"london-second-repeated-hour", "Europe/London", "01:00", "02:00", "2026-10-25T01:30:00Z", "2026-10-25T02:01:00Z", true}, + {"london-first-repeated-end", "Europe/London", "00:00", "01:30", "2026-10-25T00:15:00Z", "2026-10-25T00:31:00Z", true}, + {"london-second-repeated-end", "Europe/London", "00:00", "01:30", "2026-10-25T01:15:00Z", "2026-10-25T01:31:00Z", true}, + {"london-after-first-end", "Europe/London", "00:00", "01:30", "2026-10-25T00:45:00Z", "", false}, + {"london-after-second-end", "Europe/London", "00:00", "01:30", "2026-10-25T01:45:00Z", "", false}, + {"london-missing-start", "Europe/London", "01:30", "02:30", "2026-03-29T01:15:00Z", "2026-03-29T01:31:00Z", true}, + {"london-missing-end-replay", "Europe/London", "00:00", "01:30", "2026-03-29T00:55:30Z", "2026-03-29T01:00:00Z", true}, + {"london-after-missing-end", "Europe/London", "00:00", "01:30", "2026-03-29T01:15:00Z", "", false}, + {"london-overnight-missing-end", "Europe/London", "22:00", "01:30", "2026-03-28T23:00:00Z", "2026-03-29T01:00:00Z", true}, + {"london-overnight-repeated-end", "Europe/London", "22:00", "01:30", "2026-10-24T23:15:00Z", "2026-10-25T00:31:00Z", true}, + {"london-end-minute-before-jump", "Europe/London", "00:00", "00:59", "2026-03-29T00:59:59Z", "2026-03-29T01:00:00Z", true}, + {"new-york-first-repeated-end", "America/New_York", "00:00", "01:30", "2026-11-01T05:15:00Z", "2026-11-01T05:31:00Z", true}, + {"new-york-second-repeated-end", "America/New_York", "00:00", "01:30", "2026-11-01T06:15:00Z", "2026-11-01T06:31:00Z", true}, + {"new-york-after-first-end", "America/New_York", "00:00", "01:30", "2026-11-01T05:45:00Z", "", false}, + {"new-york-after-second-end", "America/New_York", "00:00", "01:30", "2026-11-01T06:45:00Z", "", false}, + {"new-york-missing-start", "America/New_York", "02:30", "03:30", "2026-03-08T07:15:00Z", "2026-03-08T07:31:00Z", true}, + {"new-york-missing-end", "America/New_York", "00:00", "02:30", "2026-03-08T06:55:00Z", "2026-03-08T07:00:00Z", true}, + {"new-york-after-missing-end", "America/New_York", "00:00", "02:30", "2026-03-08T07:15:00Z", "", false}, + {"lord-howe-first-repeated-half-hour", "Australia/Lord_Howe", "01:30", "02:00", "2026-04-04T14:40:00Z", "2026-04-04T15:31:00Z", true}, + {"lord-howe-second-repeated-half-hour", "Australia/Lord_Howe", "01:30", "02:00", "2026-04-04T15:10:00Z", "2026-04-04T15:31:00Z", true}, + {"lord-howe-missing-start", "Australia/Lord_Howe", "02:00", "02:45", "2026-10-03T15:35:00Z", "2026-10-03T15:46:00Z", true}, + {"lord-howe-after-missing-end", "Australia/Lord_Howe", "00:00", "02:15", "2026-10-03T15:40:00Z", "", false}, + } { + t.Run(tc.name, func(t *testing.T) { + now, err := time.Parse(time.RFC3339, tc.now) + if err != nil { + t.Fatal(err) + } + m := newTestManager(t) + m.now = func() time.Time { return now } + cfg := m.GetConfig() + cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: tc.zone, + Days: map[string]bool{"monday": true, "tuesday": true, "wednesday": true, "thursday": true, "friday": true, "saturday": true, "sunday": true}} + m.UpdateConfig(cfg) + m.mu.Lock() + defer m.mu.Unlock() + if got := m.isInQuietHours(); got != tc.quiet { + t.Errorf("quiet at %s for %s %s-%s = %v, want %v", now, tc.zone, tc.start, tc.end, got, tc.quiet) + } + if tc.replay == "" { + return + } + want, err := time.Parse(time.RFC3339, tc.replay) + if err != nil { + t.Fatal(err) + } + got := m.quietHoursReplayAt() + if !got.Equal(want) { + t.Errorf("replay = %s, want first eligible real minute %s", got, want) + } + if !got.After(now) { + t.Error("replay must be in the future") + } + now = want.Add(-time.Nanosecond) + if !m.isInQuietHours() { + t.Error("quiet hours ended before the final included minute finished") + } + now = want + if m.isInQuietHours() { + t.Error("computed replay minute is still quiet") + } + }) + } +} + +func TestQuietHoursOvernightSelectedCalendarDays(t *testing.T) { + m := newTestManager(t) + now := time.Date(2026, 10, 2, 23, 30, 0, 0, time.UTC) // Friday + m.now = func() time.Time { return now } + cfg := m.GetConfig() + cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: "22:00", End: "06:00", Timezone: "UTC", Days: map[string]bool{"friday": true}} + m.UpdateConfig(cfg) + m.mu.Lock() + defer m.mu.Unlock() + if !m.isInQuietHours() { + t.Fatal("selected Friday evening must be quiet") + } + want := time.Date(2026, 10, 3, 0, 0, 0, 0, time.UTC) + if got := m.quietHoursReplayAt(); !got.Equal(want) { + t.Fatalf("unselected Saturday must end suppression at midnight: got %s, want %s", got, want) + } + now = want + if m.isInQuietHours() { + t.Fatal("overnight settings must not silently enable an unselected day") + } + now = now.AddDate(0, 0, 6).Add(6*time.Hour + 59*time.Second) // Friday's inclusive 06:00 minute + if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(now.Truncate(time.Minute).Add(time.Minute)) { + t.Fatal("selected morning must keep the inclusive end minute") + } +} + +func TestQuietHoursReplaySkipsNonexistentGap(t *testing.T) { + // This overnight window has a one-minute daily gap, 01:29. London's + // spring jump skips that entire gap, so the next real exit is Monday. + now := time.Date(2026, 3, 28, 23, 0, 0, 0, time.UTC) + m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: "01:30", End: "01:28", Timezone: "Europe/London", + Days: map[string]bool{"saturday": true, "sunday": true, "monday": true}}) + want := time.Date(2026, 3, 30, 0, 29, 0, 0, time.UTC) + if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(want) { + t.Fatalf("skipped gap must not release a notification into quiet hours: replay=%s want=%s", m.quietHoursReplayAt(), want) + } +} + +func TestQuietHoursFullDayRetainsDailyReplayBoundary(t *testing.T) { + for _, tc := range []struct{ start, end, now, replay string }{ + {"00:00", "23:59", "2026-10-01T12:30:00Z", "2026-10-02T00:00:00Z"}, + {"22:00", "21:59", "2026-10-01T23:30:00Z", "2026-10-02T22:00:00Z"}, + {"22:00", "21:59", "2026-10-01T12:30:00Z", "2026-10-01T22:00:00Z"}, + } { + t.Run(tc.start+"/"+tc.now, func(t *testing.T) { + now, _ := time.Parse(time.RFC3339, tc.now) + want, _ := time.Parse(time.RFC3339, tc.replay) + m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: "UTC", Days: map[string]bool{"thursday": true, "friday": true}}) + if !m.isInQuietHours() || !m.quietHoursReplayAt().Equal(want) { + t.Fatalf("daily replay changed: %s, want %s", m.quietHoursReplayAt(), want) + } + }) + } +} + +func TestQuietHoursClockDispatchAndDiagnosis(t *testing.T) { + for _, tc := range []struct { + name, now, start, end, replay string + quiet bool + }{ + {"repeated-hour", "2026-10-25T00:30:00Z", "01:00", "02:00", "2026-10-25T02:01:00Z", true}, + {"skipped-end", "2026-03-29T00:55:00Z", "00:00", "01:30", "2026-03-29T01:00:00Z", true}, + {"after-skipped-end", "2026-03-29T01:15:00Z", "00:00", "01:30", "", false}, + } { + for _, critical := range []bool{false, true} { + name := tc.name + "/warning" + if critical { + name = tc.name + "/unsuppressed-critical" + } + t.Run(name, func(t *testing.T) { + now, _ := time.Parse(time.RFC3339, tc.now) + m := newTestManager(t) + m.now = func() time.Time { return now } + cfg := m.GetConfig() + cfg.Enabled = true + cfg.ActivationState = ActivationActive + cfg.FlappingEnabled = false + cfg.Schedule.QuietHours = QuietHours{Enabled: true, Start: tc.start, End: tc.end, Timezone: "Europe/London", Days: map[string]bool{"sunday": true}} + m.UpdateConfig(cfg) + _, alert := testNewCanonicalAlert("vm-100", "vm-100-cpu", "vm", "cpu") + alert.Level = AlertLevelWarning + if critical { + alert.Level = AlertLevelCritical + } + alert.StartTime = now + alert.LastSeen = now + // Old metadata must be cleared when quiet hours no longer apply. + alert.Metadata = map[string]interface{}{MetadataQuietHoursSuppressed: true, MetadataQuietHoursReplayAt: "2099-01-01T00:00:00Z"} + m.mu.Lock() + m.setActiveAlertNoLock(alert.ID, alert) + m.mu.Unlock() + deferred := tc.quiet && !critical + diagnosis, found := m.DiagnoseAlertDelivery(alert.ID) + if !found { + t.Fatal("active occurrence unavailable") + } + if deferred { + want, _ := time.Parse(time.RFC3339, tc.replay) + if diagnosis.Status != AlertDeliveryStatusDeferred || diagnosis.Reason != AlertDeliveryReasonQuietHours || diagnosis.QuietHoursReplayAt == nil || !diagnosis.QuietHoursReplayAt.Equal(want) { + t.Fatalf("diagnosis disagrees with local-clock policy: %+v", diagnosis) + } + } else if diagnosis.Status != AlertDeliveryStatusWouldSend || diagnosis.Reason != AlertDeliveryReasonReady { + t.Fatalf("eligible alert was diagnosed as held: %+v", diagnosis) + } + var dispatched *Alert + m.SetAlertCallback(func(a *Alert) { dispatched = a }) + m.mu.Lock() + admitted := m.dispatchAlert(alert, false) + m.mu.Unlock() + if !admitted || dispatched == nil { + t.Fatal("quiet-hours replay must enter the existing delivery pipeline, not be dropped") + } + if deferred { + if dispatched.Metadata[MetadataQuietHoursSuppressed] != true || dispatched.Metadata[MetadataQuietHoursReplayAt] != tc.replay { + t.Fatalf("pipeline replay timestamp = %#v, want %s", dispatched.Metadata, tc.replay) + } + } else if hasQuietHoursNotificationReplay(dispatched) { + t.Fatalf("eligible alert kept stale deferral metadata: %#v", dispatched.Metadata) + } + // Escalation/bypass sends consult this same public helper. + bypass := cloneAlertForOutput(dispatched) + if m.ShouldSuppressNotification(bypass) || hasQuietHoursNotificationReplay(bypass) != deferred { + t.Fatal("public bypass helper did not apply the same clock policy") + } + }) + } + } +} + +func TestQuietHoursConcurrentUncachedReaders(t *testing.T) { + now := time.Date(2026, 10, 25, 0, 30, 0, 0, time.UTC) + m := fixedQuietHoursTestManager(now, QuietHours{Enabled: true, Start: "01:00", End: "02:00", Timezone: "Europe/London", Days: map[string]bool{"sunday": true}}) + start := make(chan struct{}) + var wg sync.WaitGroup + for worker := 0; worker < 32; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for i := 0; i < 25; i++ { + alert := &Alert{ID: "uncached-reader", Type: "cpu", Level: AlertLevelWarning} + if m.ShouldSuppressNotification(alert) || alert.Metadata[MetadataQuietHoursReplayAt] != "2026-10-25T02:01:00Z" { + t.Error("concurrent reader disagrees with quiet-hours replay") + return + } + } + }() + } + close(start) + wg.Wait() + if m.quietHoursLoc != nil { + t.Fatal("read-locked evaluation mutated the configuration-owned location cache") + } +}