From 9da4d4e9666dbc7e387c868ae1239951d2cbd137 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Fri, 24 Jul 2026 12:18:26 +0100 Subject: [PATCH] Add powered-off alert tolerance --- docs/CONFIGURATION.md | 20 ++ .../v6/internal/subsystems/agent-lifecycle.md | 5 + .../v6/internal/subsystems/alerts.md | 30 ++- .../v6/internal/subsystems/api-contracts.md | 11 +- .../subsystems/frontend-primitives.md | 11 + .../internal/subsystems/storage-recovery.md | 5 + frontend-modern/public/docs/CONFIGURATION.md | 20 ++ .../alerts/AlertIntentPolicyPanel.tsx | 166 ++++++++++-- .../__tests__/AlertIntentPolicyPanel.test.tsx | 50 +++- internal/alerts/active_cleanup.go | 4 +- internal/alerts/canonical_lifecycle.go | 2 +- internal/alerts/canonical_metric.go | 2 +- internal/alerts/config/intent.go | 6 + internal/alerts/config_facade.go | 4 + internal/alerts/guest.go | 38 ++- internal/alerts/intent_persistence.go | 44 +++- internal/alerts/intent_policy.go | 145 ++++++++--- internal/alerts/intent_policy_test.go | 97 ++++++- internal/alerts/manager.go | 14 +- internal/alerts/metric_runtime.go | 2 +- internal/alerts/powered_off_tolerance_test.go | 240 ++++++++++++++++++ internal/alerts/tracking_cleanup.go | 10 + internal/alerts/unified_incidents.go | 6 +- internal/api/alerts.go | 17 +- internal/api/alerts_endpoints_test.go | 36 +++ .../tests/85-powered-off-tolerance.spec.ts | 165 ++++++++++++ 26 files changed, 1065 insertions(+), 85 deletions(-) create mode 100644 internal/alerts/powered_off_tolerance_test.go create mode 100644 tests/integration/tests/85-powered-off-tolerance.spec.ts diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 8306b895e..9341f4677 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -396,6 +396,26 @@ Pulse uses a powerful alerting engine with hysteresis (separate trigger/clear th **Managed via UI**: Alerts → Thresholds +### VM and container powered-off tolerance + +Open **Alerts → Thresholds → Alert intent & grace** to configure how long a +Proxmox VM or LXC container may remain stopped before Pulse raises its +powered-off alert. + +- Leave the VM/container default blank to inherit the existing policy. An + installation with no applicable policy keeps the legacy two-poll behavior. +- Set it to `0` to alert on the first authoritative stopped observation. +- Set a positive number of seconds (for example, `300`) to tolerate a short + stop regardless of polling cadence. Per-resource values override the + VM/container default; a blank per-resource value inherits it. +- **Extend offline grace during Proxmox backups** applies only while Pulse has + fresh matching evidence of an active backup. The maximum deferral is a hard + cap, so a stale or stuck backup cannot hide a sustained outage. + +This tolerance delays activation; it does not disable powered-off monitoring. +Use the existing guest offline-alert toggle when a guest should never produce +powered-off alerts. +
Manual Configuration (JSON) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 1e65da995..2124061b8 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -5066,6 +5066,11 @@ Neither an expected-transient decision nor an indeterminate UDP outcome may be translated into an agent action. Any later customer-infrastructure mutation still requires the canonical Actions planner, approval, executor, receipt, audit, and verification boundaries. +Powered-off tolerance is evaluated from the Pulse server's monotonic elapsed +time and the existing VM/LXC observation stream. It does not accept agent time, +poll count, a new report field, or remote configuration authority, and a guest +resource-type default cannot change node or host-agent connectivity +confirmations. ### Server-side host deletion and re-enrollment authority diff --git a/docs/release-control/v6/internal/subsystems/alerts.md b/docs/release-control/v6/internal/subsystems/alerts.md index dd241253a..9301177a1 100644 --- a/docs/release-control/v6/internal/subsystems/alerts.md +++ b/docs/release-control/v6/internal/subsystems/alerts.md @@ -1283,6 +1283,27 @@ same resolver but restores pending state before returning, so it is read-only. Invalid documents, persistence failures, or revision conflicts must leave the prior in-memory and durable policy active. +Resource-type inheritance is field-wise from general to specific before the +canonical-resource rule: for example `guest` supplies a VM/LXC default and +`vm` or `system-container` may replace individual fields. A guest policy must +not leak into node or agent connectivity. Omitted grace preserves inherited or +factory behavior, explicit zero is a deliberate no-wait policy, and a positive +powered-off grace is duration authority rather than a poll-count threshold. +The no-policy powered-state path retains the legacy two-confirmation contract +for migration compatibility; an applicable powered-state duration activates +on the first stopped observation at or after elapsed eligibility. + +Runtime eligibility uses server receipt time plus process-monotonic elapsed +progress, never client/provider timestamps or poll counts. Duplicate reports +advance no time, delayed polls count actual elapsed time, and wall-clock +forward/backward changes do not change the decision. Pending state persists +accumulated elapsed progress. Restart re-baselines the monotonic clock without +counting unobserved process downtime, while legacy timestamp-only pending files +are imported once into the elapsed representation. Recovery, explicit guest +suppression, disabling offline alerts, and tracking cleanup remove both +durable pending state and its transient monotonic baseline before a later +outage can start. + Operator maintenance and intentionally-offline state are read only through the canonical unified-resource identity. Backup-aware offline deferral consumes fresh, matching, active task evidence, applies the configured post-backup grace, @@ -1291,6 +1312,9 @@ finished, or mismatched backup evidence cannot suppress an outage. This policy changes alert activation only: notification delivery, recovery assurance, and customer-infrastructure mutation retain their existing owners. -`internal/alerts/intent_policy_test.go` proves precedence, normalization, -operator and backup contexts, preview immutability, restart continuity, and -first-match lifecycle identity. +`internal/alerts/intent_policy_test.go` and +`internal/alerts/powered_off_tolerance_test.go` prove field-wise type +precedence, normalization, exact zero/positive duration boundaries, VM/LXC +coverage, duplicate and delayed reports, wall-clock changes, suppression +reset, migration/restart continuity, backup hard-cap behavior, preview +immutability, and first-match lifecycle identity. diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 8b2dcfbf9..359173864 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -8247,7 +8247,14 @@ default, resource-type, and canonical-resource rules. A stale revision returns `409 Conflict`; invalid policy or preview input returns `400`; unavailable runtime or persistence ownership returns `503`. A save failure restores the previous in-memory document and returns `500`. Preview is bounded and read-only -and cannot advance pending grace state. +and cannot advance pending grace state. Both write and preview transports +accept exactly one JSON object, reject unknown fields, and validate the signal +against the same stable policy vocabulary as persistence. JSON coercion, +trailing objects, unsupported signals, fractional durations, negative +durations, and values above the 30-day bound fail with `400` rather than being +silently rounded or ignored. Tenant selection remains request-context owned; +policy reads, preview, and writes never fall back to another organization's +manager once an organization context is present. Availability targets add `udp` plus `response_required` and `open_or_filtered` modes. Request and expected-response payloads remain @@ -8257,7 +8264,7 @@ clients. A silent open-or-filtered result is `indeterminate`, not successful reachability and not a transport failure that may be promoted to an outage. `internal/api/alerts_endpoints_test.go` proves scopes, revision conflicts, -rollback, and preview behavior. +rollback, exact decoding, invalid-signal rejection, and preview behavior. `frontend-modern/src/api/__tests__/alertIntentPolicies.test.ts` and the availability target API and settings tests prove the canonical browser routes and additive UDP wire fields. diff --git a/docs/release-control/v6/internal/subsystems/frontend-primitives.md b/docs/release-control/v6/internal/subsystems/frontend-primitives.md index 28919e7ae..e193c070b 100644 --- a/docs/release-control/v6/internal/subsystems/frontend-primitives.md +++ b/docs/release-control/v6/internal/subsystems/frontend-primitives.md @@ -5263,6 +5263,16 @@ server-owned document after success. Preview renders clear, expected-transient, pending-grace, and would-activate as distinct states and never presents preview as a write. +The powered-off default is presented and persisted as the `guest` resource-type +rule so VM and LXC resources inherit it without changing node or agent +connectivity. Blank means inherit, `0` explicitly means no wait, and the UI +states both meanings next to the control. Duration fields accept only base-10 +whole seconds from zero through 30 days; an enabled backup hard cap must be +positive. Invalid, fractional, negative, or oversized values remain local, +show an actionable error, and issue no API write. Disabling backup extension is +an explicit `enabled: false` rule and remains separate from disabling a guest's +powered-off alerts. + Availability controls expose UDP mode, request payload, and optional expected response only where valid for the selected protocol. Unified-resource presentation keeps `indeterminate` visibly distinct from reachable and @@ -5273,6 +5283,7 @@ does not infer detector, operator-intent, or recovery truth. The focused proofs are `frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx`, `frontend-modern/src/features/alerts/__tests__/ThresholdsTab.test.tsx`, +`tests/integration/tests/85-powered-off-tolerance.spec.ts`, `frontend-modern/src/components/Settings/ConnectionEditor/__tests__/AvailabilityTargetSlot.test.tsx`, and `frontend-modern/src/utils/__tests__/availabilityProbePresentation.test.ts`. diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index a504e8321..b41480b8c 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -2402,6 +2402,11 @@ evidence fails closed for alert deferral, while PBS and provider recovery evidence continues through its independent mapper and posture rules. An alert grace period or hard-cap expiry cannot be interpreted as backup success or failure and grants no restore or infrastructure mutation authority. +The alert runtime measures its base powered-off tolerance and backup hard cap +from server-owned monotonic elapsed time. Backup task timestamps may qualify +fresh matching context but cannot advance, reset, or extend that clock beyond +the cap. A restart conservatively resumes persisted alert progress without +claiming process downtime as backup coverage or recovery evidence. ### Physical-disk collection truth diff --git a/frontend-modern/public/docs/CONFIGURATION.md b/frontend-modern/public/docs/CONFIGURATION.md index a7b4d822c..15b61bc33 100644 --- a/frontend-modern/public/docs/CONFIGURATION.md +++ b/frontend-modern/public/docs/CONFIGURATION.md @@ -394,6 +394,26 @@ Pulse uses a powerful alerting engine with hysteresis (separate trigger/clear th **Managed via UI**: Alerts → Thresholds +### VM and container powered-off tolerance + +Open **Alerts → Thresholds → Alert intent & grace** to configure how long a +Proxmox VM or LXC container may remain stopped before Pulse raises its +powered-off alert. + +- Leave the VM/container default blank to inherit the existing policy. An + installation with no applicable policy keeps the legacy two-poll behavior. +- Set it to `0` to alert on the first authoritative stopped observation. +- Set a positive number of seconds (for example, `300`) to tolerate a short + stop regardless of polling cadence. Per-resource values override the + VM/container default; a blank per-resource value inherits it. +- **Extend offline grace during Proxmox backups** applies only while Pulse has + fresh matching evidence of an active backup. The maximum deferral is a hard + cap, so a stale or stuck backup cannot hide a sustained outage. + +This tolerance delays activation; it does not disable powered-off monitoring. +Use the existing guest offline-alert toggle when a guest should never produce +powered-off alerts. +
Manual Configuration (JSON) diff --git a/frontend-modern/src/features/alerts/AlertIntentPolicyPanel.tsx b/frontend-modern/src/features/alerts/AlertIntentPolicyPanel.tsx index 930d9f39d..372541cc2 100644 --- a/frontend-modern/src/features/alerts/AlertIntentPolicyPanel.tsx +++ b/frontend-modern/src/features/alerts/AlertIntentPolicyPanel.tsx @@ -29,9 +29,26 @@ const SIGNALS: ReadonlyArray<{ value: AlertIntentSignal; label: string }> = [ { value: 'metric.disk', label: 'Disk threshold' }, ]; -const nonNegativeInt = (value: string, fallback = 0): number => { - const parsed = Number.parseInt(value.trim(), 10); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +const MAX_GRACE_SECONDS = 30 * 24 * 60 * 60; + +const durationSeconds = ( + value: string, + label: string, + options: { allowBlank?: boolean; positive?: boolean } = {}, +): number | undefined => { + const normalized = value.trim(); + if (normalized === '' && options.allowBlank) return undefined; + if (!/^\d+$/.test(normalized)) { + throw new Error(`${label} must be a whole number of seconds.`); + } + const parsed = Number(normalized); + const minimum = options.positive ? 1 : 0; + if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > MAX_GRACE_SECONDS) { + throw new Error( + `${label} must be between ${minimum} and ${MAX_GRACE_SECONDS.toLocaleString()} seconds.`, + ); + } + return parsed; }; const detachedDocument = (document: AlertIntentPolicyDocument): AlertIntentPolicyDocument => @@ -51,8 +68,8 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } const [error, setError] = createSignal(null); const [message, setMessage] = createSignal(null); - const [offlineGrace, setOfflineGrace] = createSignal('0'); - const [availabilityGrace, setAvailabilityGrace] = createSignal('0'); + const [offlineGrace, setOfflineGrace] = createSignal(''); + const [availabilityGrace, setAvailabilityGrace] = createSignal(''); const [honorOperatorState, setHonorOperatorState] = createSignal(false); const [backupAware, setBackupAware] = createSignal(false); const [backupPostGrace, setBackupPostGrace] = createSignal('60'); @@ -92,10 +109,13 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } try { const loaded = await AlertIntentPoliciesAPI.get(); setDocument(loaded); - const offline = loaded.defaults?.['state.offline'] ?? {}; + const offline = + loaded.resourceTypes?.guest?.['state.offline'] ?? loaded.defaults?.['state.offline'] ?? {}; const availability = loaded.defaults?.['incident.availability'] ?? {}; - setOfflineGrace(String(offline.graceSeconds ?? 0)); - setAvailabilityGrace(String(availability.graceSeconds ?? 0)); + setOfflineGrace(offline.graceSeconds === undefined ? '' : String(offline.graceSeconds)); + setAvailabilityGrace( + availability.graceSeconds === undefined ? '' : String(availability.graceSeconds), + ); setHonorOperatorState(offline.honorOperatorState ?? false); setBackupAware(offline.backupOffline?.enabled ?? false); setBackupPostGrace(String(offline.backupOffline?.postGraceSeconds ?? 60)); @@ -164,23 +184,64 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } const saveDefaults = async () => { const current = document(); if (!current) return; + let guestOfflineGrace: number | undefined; + let defaultAvailabilityGrace: number | undefined; + let postGrace = 0; + let maxDeferral = 0; + try { + guestOfflineGrace = durationSeconds(offlineGrace(), 'VM / container powered-off tolerance', { + allowBlank: true, + }); + defaultAvailabilityGrace = durationSeconds( + availabilityGrace(), + 'Default availability grace', + { allowBlank: true }, + ); + if (backupAware()) { + postGrace = + durationSeconds(backupPostGrace(), 'Post-backup grace') ?? + /* istanbul ignore next -- nonblank input is required above */ 0; + maxDeferral = + durationSeconds(backupMaxDeferral(), 'Maximum backup deferral', { + positive: true, + }) ?? /* istanbul ignore next -- nonblank input is required above */ 0; + } + } catch (cause) { + setMessage(null); + setError(cause instanceof Error ? cause.message : 'Invalid alert tolerance.'); + return; + } const next = detachedDocument(current); + next.resourceTypes ??= {}; + next.resourceTypes.guest ??= {}; + const guestOffline: AlertIntentRule = { + ...(next.resourceTypes.guest['state.offline'] ?? {}), + honorOperatorState: honorOperatorState(), + backupOffline: backupAware() + ? { + enabled: true, + postGraceSeconds: postGrace, + maxDeferralSeconds: maxDeferral, + } + : { enabled: false }, + }; + if (guestOfflineGrace === undefined) { + delete guestOffline.graceSeconds; + } else { + guestOffline.graceSeconds = guestOfflineGrace; + } + next.resourceTypes.guest['state.offline'] = guestOffline; next.defaults ??= {}; - next.defaults['state.offline'] = { - ...(next.defaults['state.offline'] ?? {}), - graceSeconds: nonNegativeInt(offlineGrace()), - honorOperatorState: honorOperatorState(), - backupOffline: { - enabled: backupAware(), - postGraceSeconds: nonNegativeInt(backupPostGrace(), 60), - maxDeferralSeconds: nonNegativeInt(backupMaxDeferral(), 3600), - }, - }; - next.defaults['incident.availability'] = { + const availability: AlertIntentRule = { ...(next.defaults['incident.availability'] ?? {}), - graceSeconds: nonNegativeInt(availabilityGrace()), honorOperatorState: honorOperatorState(), }; + if (defaultAvailabilityGrace === undefined) { + delete availability.graceSeconds; + } else { + availability.graceSeconds = defaultAvailabilityGrace; + } + next.defaults['incident.availability'] = availability; await persist(next, 'Default intent policy saved.'); }; @@ -188,12 +249,31 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } const current = document(); const id = resourceId(); if (!current || !id) return; + let grace: number | undefined; + let postGrace = 0; + let maxDeferral = 0; + try { + grace = durationSeconds(overrideGrace(), 'Grace override', { allowBlank: true }); + if (signal() === 'state.offline' && overrideBackupMode() === 'enabled') { + postGrace = + durationSeconds(overrideBackupPostGrace(), 'Post-backup grace') ?? + /* istanbul ignore next -- nonblank input is required above */ 0; + maxDeferral = + durationSeconds(overrideBackupMaxDeferral(), 'Maximum deferral', { + positive: true, + }) ?? /* istanbul ignore next -- nonblank input is required above */ 0; + } + } catch (cause) { + setMessage(null); + setError(cause instanceof Error ? cause.message : 'Invalid alert tolerance.'); + return; + } const next = detachedDocument(current); next.resources ??= {}; next.resources[id] ??= {}; const rule: AlertIntentRule = {}; - if (overrideGrace().trim() !== '') { - rule.graceSeconds = nonNegativeInt(overrideGrace()); + if (grace !== undefined) { + rule.graceSeconds = grace; } if (overrideOperatorMode() !== 'inherit') { rule.honorOperatorState = overrideOperatorMode() === 'honor'; @@ -201,8 +281,9 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } if (signal() === 'state.offline' && overrideBackupMode() !== 'inherit') { rule.backupOffline = { enabled: overrideBackupMode() === 'enabled', - postGraceSeconds: nonNegativeInt(overrideBackupPostGrace(), 60), - maxDeferralSeconds: nonNegativeInt(overrideBackupMaxDeferral(), 3600), + ...(overrideBackupMode() === 'enabled' + ? { postGraceSeconds: postGrace, maxDeferralSeconds: maxDeferral } + : {}), }; } if (Object.keys(rule).length === 0) { @@ -289,21 +370,34 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } >
Post-backup grace setOverrideBackupPostGrace(event.currentTarget.value)} @@ -448,7 +558,11 @@ export function AlertIntentPolicyPanel(props: { resources: readonly Resource[] } Maximum deferral setOverrideBackupMaxDeferral(event.currentTarget.value)} diff --git a/frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx b/frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx index bde69b64b..65a1aa287 100644 --- a/frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx +++ b/frontend-modern/src/features/alerts/__tests__/AlertIntentPolicyPanel.test.tsx @@ -85,7 +85,7 @@ describe('AlertIntentPolicyPanel', () => { await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1)); fireEvent.click(screen.getByRole('button', { name: 'Configure policies' })); const grace = await screen.findByLabelText(/^Grace override \(seconds\)/); - expect(grace).toHaveValue(''); + expect(grace).toHaveValue(null); expect(screen.getByLabelText('Operator state override')).toHaveValue('inherit'); expect(screen.getByLabelText('Backup handling override')).toHaveValue('inherit'); @@ -97,6 +97,54 @@ describe('AlertIntentPolicyPanel', () => { expect(saved.resources?.[resource.id]?.['state.offline']).toEqual({ graceSeconds: 75 }); }); + it('stores the powered-off default on the guest resource type', async () => { + render(() => ); + + await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1)); + fireEvent.click(screen.getByRole('button', { name: 'Configure policies' })); + const tolerance = await screen.findByLabelText( + /^VM \/ container powered-off tolerance \(seconds\)/, + ); + expect(tolerance).toHaveValue(30); + fireEvent.input(tolerance, { target: { value: '300' } }); + fireEvent.click(screen.getByRole('button', { name: 'Save defaults' })); + + await waitFor(() => expect(AlertIntentPoliciesAPI.update).toHaveBeenCalledTimes(1)); + const saved = vi.mocked(AlertIntentPoliciesAPI.update).mock.calls[0][0]; + expect(saved.resourceTypes?.guest?.['state.offline']).toEqual( + expect.objectContaining({ + graceSeconds: 300, + honorOperatorState: true, + backupOffline: { enabled: true, postGraceSeconds: 60, maxDeferralSeconds: 3600 }, + }), + ); + }); + + it('preserves explicit zero and rejects non-integer overrides locally', async () => { + render(() => ); + + await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1)); + fireEvent.click(screen.getByRole('button', { name: 'Configure policies' })); + const grace = await screen.findByLabelText(/^Grace override \(seconds\)/); + + fireEvent.input(grace, { target: { value: '0' } }); + fireEvent.click(screen.getByRole('button', { name: 'Save override' })); + await waitFor(() => expect(AlertIntentPoliciesAPI.update).toHaveBeenCalledTimes(1)); + expect( + vi.mocked(AlertIntentPoliciesAPI.update).mock.calls[0][0].resources?.[resource.id]?.[ + 'state.offline' + ], + ).toEqual({ graceSeconds: 0 }); + + vi.mocked(AlertIntentPoliciesAPI.update).mockClear(); + fireEvent.input(grace, { target: { value: '1.5' } }); + fireEvent.click(screen.getByRole('button', { name: 'Save override' })); + expect(await screen.findByRole('alert')).toHaveTextContent( + 'Grace override must be a whole number of seconds.', + ); + expect(AlertIntentPoliciesAPI.update).not.toHaveBeenCalled(); + }); + it('previews the selected canonical resource', async () => { render(() => ); await waitFor(() => expect(AlertIntentPoliciesAPI.get).toHaveBeenCalledTimes(1)); diff --git a/internal/alerts/active_cleanup.go b/internal/alerts/active_cleanup.go index d5ba6d01c..c153fff87 100644 --- a/internal/alerts/active_cleanup.go +++ b/internal/alerts/active_cleanup.go @@ -314,13 +314,15 @@ func (m *Manager) CleanupAlertsForNodes(existingNodes map[string]bool) { // ClearActiveAlerts removes all active and pending alerts, resetting the manager state. func (m *Manager) ClearActiveAlerts() { m.mu.Lock() - if len(m.activeAlerts) == 0 && len(m.pendingAlerts) == 0 { + if len(m.activeAlerts) == 0 && len(m.pendingAlerts) == 0 && len(m.intentPending) == 0 { m.mu.Unlock() return } m.activeAlerts = make(map[string]*Alert) m.activeAlertAlias = make(map[string]string) m.pendingAlerts = make(map[string]time.Time) + m.intentPending = make(map[string]IntentPendingState) + m.intentRuntimeTicks = make(map[string]time.Duration) m.recentAlerts = make(map[string]*Alert) m.suppressedUntil = make(map[string]time.Time) m.alertRateLimit = make(map[string][]time.Time) diff --git a/internal/alerts/canonical_lifecycle.go b/internal/alerts/canonical_lifecycle.go index c97d829ac..4f2eb2c9f 100644 --- a/internal/alerts/canonical_lifecycle.go +++ b/internal/alerts/canonical_lifecycle.go @@ -396,7 +396,7 @@ func (m *Manager) evaluateCanonicalLifecycleAlert(params canonicalLifecycleAlert if pending, ok := m.intentPending[storageKey]; ok && !pending.FirstMatchedAt.IsZero() { result.State.FirstMatchedAt = pending.FirstMatchedAt } - delete(m.intentPending, storageKey) + m.clearIntentPendingNoLock(storageKey) m.saveActiveAlertsAsync("lifecycle intent activated") } } diff --git a/internal/alerts/canonical_metric.go b/internal/alerts/canonical_metric.go index 8b62f0169..b19ea3349 100644 --- a/internal/alerts/canonical_metric.go +++ b/internal/alerts/canonical_metric.go @@ -156,7 +156,7 @@ func (m *Manager) evaluateCanonicalMetricAlert(spec alertspecs.ResourceAlertSpec if !decision.ShouldActivate { return } - delete(m.intentPending, trackingKey) + m.clearIntentPendingNoLock(trackingKey) m.saveActiveAlertsAsync("canonical metric intent activated") } else if timeThreshold := m.getTimeThreshold(spec.ResourceID, resourceType, metricType); timeThreshold > 0 { if pendingTime, isPending := m.pendingAlerts[trackingKey]; isPending { diff --git a/internal/alerts/config/intent.go b/internal/alerts/config/intent.go index d9e2674c0..84c31035c 100644 --- a/internal/alerts/config/intent.go +++ b/internal/alerts/config/intent.go @@ -219,3 +219,9 @@ func validAlertIntentSignal(signal string) bool { } return strings.HasPrefix(signal, "metric.") && strings.TrimPrefix(signal, "metric.") != "" } + +// ValidAlertIntentSignal reports whether signal is accepted by both persisted +// policy rules and preview requests. +func ValidAlertIntentSignal(signal string) bool { + return validAlertIntentSignal(strings.ToLower(strings.TrimSpace(signal))) +} diff --git a/internal/alerts/config_facade.go b/internal/alerts/config_facade.go index a9844ebe5..0fb0c0692 100644 --- a/internal/alerts/config_facade.go +++ b/internal/alerts/config_facade.go @@ -60,6 +60,10 @@ func ValidateAlertIntentPolicyDocument(document AlertIntentPolicyDocument) error return alertconfig.ValidateAlertIntentPolicyDocument(document) } +func ValidAlertIntentSignal(signal string) bool { + return alertconfig.ValidAlertIntentSignal(signal) +} + var ErrAlertNotFound = errors.New("alert not found") func NormalizeAlertConfigAliases(config *AlertConfig) { diff --git a/internal/alerts/guest.go b/internal/alerts/guest.go index c0c62b3a1..0d824f29b 100644 --- a/internal/alerts/guest.go +++ b/internal/alerts/guest.go @@ -165,7 +165,11 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { // Clear any pending powered-off tracking and alerts when globally disabled m.mu.Lock() delete(m.offlineConfirmations, guestID) + intentCleared := m.clearIntentPendingNoLock(canonicalPoweredStateStateID(guestID)) m.mu.Unlock() + if intentCleared { + m.saveActiveAlertsAsync("guest offline intent disabled") + } m.rehomeStrandedGuestAlert(canonicalPoweredStateStateID(guestID), canonicalPoweredStateSpecID(guestID), string(alertspecs.AlertSpecKindPoweredState), guestID, name, node, instanceName, "guest") m.clearAlert(canonicalPoweredStateStateID(guestID)) } else if snapshot.OnBoot != nil && !*snapshot.OnBoot { @@ -180,9 +184,10 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { m.mu.RLock() thresholds := m.getGuestThresholds(guest, guestID) m.mu.RUnlock() + observedAt := m.policyNow().UTC() backupContext := BackupIntentContext{ Active: strings.EqualFold(strings.TrimSpace(snapshot.Lock), "backup"), - ObservedAt: time.Now(), + ObservedAt: observedAt, Evidence: "guest_lock", } if !backupContext.Active { @@ -190,7 +195,7 @@ func (m *Manager) CheckGuest(guest any, instanceName string) { resolver := m.backupIntentResolver m.mu.RUnlock() if resolver != nil { - if resolved, ok := resolver(guestID, instanceName, node, snapshot.VMID, backupContext.ObservedAt); ok { + if resolved, ok := resolver(guestID, instanceName, node, snapshot.VMID, observedAt); ok { backupContext = resolved } } @@ -407,7 +412,21 @@ func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, nod if strings.EqualFold(guestType, "container") { resourceType = unifiedresources.ResourceTypeSystemContainer } - spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, 2, thresholds.Disabled || thresholds.DisableConnectivity) + m.mu.RLock() + effectiveIntent := m.resolveEffectiveIntentPolicyNoLock(guestID, string(resourceType), string(AlertIntentSignalOffline)) + m.mu.RUnlock() + confirmations := 2 + tracking := m.offlineConfirmations + durationAuthoritative := effectiveIntent.Sources["graceSeconds"] != "factory" || + (effectiveIntent.BackupOffline != nil && effectiveIntent.BackupOffline.Enabled) + if durationAuthoritative { + // Explicit powered-state policies are duration authoritative. A zero + // grace therefore fires on the first stopped observation, while a + // positive grace is independent of polling cadence. + confirmations = 1 + tracking = nil + } + spec, err := buildCanonicalPoweredStateSpec(guestID, name, resourceType, severity, confirmations, thresholds.Disabled || thresholds.DisableConnectivity) if err != nil { log.Warn(). Err(err). @@ -420,13 +439,13 @@ func (m *Manager) checkGuestPoweredOffWithThresholdsAndIntent(guestID, name, nod m.evaluateCanonicalLifecycleAlert(canonicalLifecycleAlertParams{ Spec: spec, Evidence: alertspecs.AlertEvidence{ - ObservedAt: time.Now(), + ObservedAt: m.policyNow().UTC(), PoweredState: &alertspecs.PoweredStateEvidence{ Expected: alertspecs.PowerStateOn, Observed: alertspecs.PowerStateOff, }, }, - Tracking: m.offlineConfirmations, + Tracking: tracking, TrackingKey: guestID, AlertID: alertID, AlertType: "powered-off", @@ -467,7 +486,7 @@ func (m *Manager) clearGuestPoweredOffAlert(guestID, name string) { Msg("Guest is running, resetting powered-off confirmation count") delete(m.offlineConfirmations, guestID) } - if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, time.Now(), false, BackupIntentContext{}); decision.StateChanged { + if decision := m.evaluateIntentNoLock(guestID, "", string(AlertIntentSignalOffline), alertID, m.policyNow().UTC(), false, BackupIntentContext{}); decision.StateChanged { m.saveActiveAlertsAsync("guest offline intent cleared") } @@ -649,6 +668,13 @@ func (m *Manager) suppressGuestAlerts(guestID string) bool { } delete(m.offlineConfirmations, guestID) + for key, pending := range m.intentPending { + if pending.ResourceID == guestID || strings.HasPrefix(pending.ResourceID, guestID+"/") { + if m.clearIntentPendingNoLock(key) { + cleared = true + } + } + } return cleared } diff --git a/internal/alerts/intent_persistence.go b/internal/alerts/intent_persistence.go index 2ce049da4..8ba0cd345 100644 --- a/internal/alerts/intent_persistence.go +++ b/internal/alerts/intent_persistence.go @@ -75,17 +75,57 @@ func (m *Manager) loadIntentPendingNoLock() error { if err := json.Unmarshal(data, &states); err != nil { return fmt.Errorf("decode alert intent pending state: %w", err) } - now := time.Now() + now := m.policyNow().UTC() if m.intentPending == nil { m.intentPending = make(map[string]IntentPendingState) } + if m.intentRuntimeTicks == nil { + m.intentRuntimeTicks = make(map[string]time.Duration) + } for _, state := range states { if state.TrackingKey == "" || state.ResourceID == "" || state.Signal == "" || state.FirstMatchedAt.IsZero() { continue } - if now.Sub(state.LastObservedAt) > 24*time.Hour || state.LastObservedAt.After(now.Add(time.Minute)) || state.FirstMatchedAt.After(now.Add(time.Minute)) { + if state.LastObservedAt.IsZero() || (!state.LastObservedAt.After(now) && now.Sub(state.LastObservedAt) > 24*time.Hour) { continue } + if state.ElapsedNanos < 0 { + state.ElapsedNanos = 0 + } + // Schema-v1 files written before elapsed tracking used wall-clock + // timestamps only. Import their observed progress once, then use the + // process monotonic clock for all future decisions. + if state.ElapsedNanos == 0 && state.LastObservedAt.After(state.FirstMatchedAt) { + elapsed := state.LastObservedAt.Sub(state.FirstMatchedAt) + if elapsed > 24*time.Hour { + elapsed = 24 * time.Hour + } + state.ElapsedNanos = int64(elapsed) + } + elapsed := time.Duration(state.ElapsedNanos) + if state.LastObservedAt.After(now.Add(time.Minute)) || state.FirstMatchedAt.After(now.Add(time.Minute)) { + state.LastObservedAt = now + state.FirstMatchedAt = now.Add(-elapsed) + } + if state.BackupEndedElapsedNanos == nil && state.BackupEndedAt != nil && state.BackupEndedAt.After(state.FirstMatchedAt) { + endedElapsed := state.BackupEndedAt.Sub(state.FirstMatchedAt) + if endedElapsed > time.Duration(state.ElapsedNanos) { + endedElapsed = time.Duration(state.ElapsedNanos) + } + value := int64(endedElapsed) + state.BackupEndedElapsedNanos = &value + } + if state.BackupEndedElapsedNanos != nil { + if *state.BackupEndedElapsedNanos < 0 { + value := int64(0) + state.BackupEndedElapsedNanos = &value + } else if *state.BackupEndedElapsedNanos > state.ElapsedNanos { + value := state.ElapsedNanos + state.BackupEndedElapsedNanos = &value + } + backupEndedElapsed := time.Duration(*state.BackupEndedElapsedNanos) + state.BackupEndedAt = intentTimePointer(now.Add(-(elapsed - backupEndedElapsed))) + } m.intentPending[state.TrackingKey] = state } return nil diff --git a/internal/alerts/intent_policy.go b/internal/alerts/intent_policy.go index 11fc56535..6264313d2 100644 --- a/internal/alerts/intent_policy.go +++ b/internal/alerts/intent_policy.go @@ -32,15 +32,17 @@ type BackupIntentContextResolver func(resourceID, instance, node string, vmid in type ResourceIntentIdentityResolver func(resourceID string) (canonicalID string, found bool) type IntentPendingState struct { - TrackingKey string `json:"trackingKey"` - ResourceID string `json:"resourceId"` - ResourceType string `json:"resourceType"` - Signal string `json:"signal"` - FirstMatchedAt time.Time `json:"firstMatchedAt"` - LastObservedAt time.Time `json:"lastObservedAt"` - BackupActive bool `json:"backupActive,omitempty"` - BackupEndedAt *time.Time `json:"backupEndedAt,omitempty"` - BackupEvidence string `json:"backupEvidence,omitempty"` + TrackingKey string `json:"trackingKey"` + ResourceID string `json:"resourceId"` + ResourceType string `json:"resourceType"` + Signal string `json:"signal"` + FirstMatchedAt time.Time `json:"firstMatchedAt"` + LastObservedAt time.Time `json:"lastObservedAt"` + ElapsedNanos int64 `json:"elapsedNanos,omitempty"` + BackupActive bool `json:"backupActive,omitempty"` + BackupEndedAt *time.Time `json:"backupEndedAt,omitempty"` + BackupEndedElapsedNanos *int64 `json:"backupEndedElapsedNanos,omitempty"` + BackupEvidence string `json:"backupEvidence,omitempty"` } type EffectiveAlertIntentPolicy struct { @@ -210,10 +212,11 @@ func (m *Manager) resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, s } applyRules(m.intentPolicies.Defaults, "defaults") - for _, typeKey := range CanonicalResourceTypeKeys(resourceType) { + typeKeys := CanonicalResourceTypeKeys(resourceType) + for i := len(typeKeys) - 1; i >= 0; i-- { + typeKey := typeKeys[i] if rules, ok := m.intentPolicies.ResourceTypes[typeKey]; ok { applyRules(rules, "resourceTypes."+typeKey) - break } } resourceIDs := make([]string, 0, 2) @@ -250,37 +253,97 @@ func (m *Manager) ResolveEffectiveIntentPolicy(resourceID, resourceType, signal return effective } -func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackingKey string, observedAt time.Time, conditionActive bool, backup BackupIntentContext) intentDecision { +func (m *Manager) intentTickNoLock() time.Duration { + if m.intentClock == nil { + return 0 + } + return m.intentClock() +} + +func (m *Manager) clearIntentPendingNoLock(trackingKey string) bool { + _, pending := m.intentPending[trackingKey] + _, ticking := m.intentRuntimeTicks[trackingKey] + delete(m.intentPending, trackingKey) + delete(m.intentRuntimeTicks, trackingKey) + return pending || ticking +} + +func (m *Manager) advanceIntentElapsedNoLock(trackingKey string, state *IntentPendingState, tick time.Duration) { + if state == nil { + return + } + if previous, ok := m.intentRuntimeTicks[trackingKey]; ok && tick >= previous { + delta := tick - previous + if delta > 0 && state.ElapsedNanos <= int64(^uint64(0)>>1)-int64(delta) { + state.ElapsedNanos += int64(delta) + } + } + m.intentRuntimeTicks[trackingKey] = tick +} + +func durationUntil(total, elapsed time.Duration) time.Duration { + if total <= elapsed { + return 0 + } + return total - elapsed +} + +func absoluteDuration(value time.Duration) time.Duration { + if value < 0 { + return -value + } + return value +} + +func intentTimePointer(value time.Time) *time.Time { + return &value +} + +func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackingKey string, _ time.Time, conditionActive bool, backup BackupIntentContext) intentDecision { effective := m.resolveEffectiveIntentPolicyNoLock(resourceID, resourceType, signal) decision := intentDecision{Effective: effective} if !conditionActive { - if _, exists := m.intentPending[trackingKey]; exists { - delete(m.intentPending, trackingKey) + if m.clearIntentPendingNoLock(trackingKey) { decision.StateChanged = true } decision.Reason = "condition_clear" return decision } if !effective.Explicit { - if _, exists := m.intentPending[trackingKey]; exists { - delete(m.intentPending, trackingKey) + if m.clearIntentPendingNoLock(trackingKey) { decision.StateChanged = true } decision.ShouldActivate = true return decision } - if observedAt.IsZero() { - observedAt = m.policyNow() - } + observedAt := m.policyNow().UTC() + tick := m.intentTickNoLock() state, exists := m.intentPending[trackingKey] if !exists || state.FirstMatchedAt.IsZero() { state = IntentPendingState{ TrackingKey: trackingKey, ResourceID: resourceID, ResourceType: resourceType, Signal: signal, FirstMatchedAt: observedAt, } + m.intentRuntimeTicks[trackingKey] = tick decision.StateChanged = true + } else { + previousElapsed := state.ElapsedNanos + m.advanceIntentElapsedNoLock(trackingKey, &state, tick) + if state.ElapsedNanos != previousElapsed { + decision.StateChanged = true + } } state.LastObservedAt = observedAt + elapsed := time.Duration(state.ElapsedNanos) + projectedFirstMatchedAt := observedAt.Add(-elapsed) + if absoluteDuration(state.FirstMatchedAt.Add(elapsed).Sub(observedAt)) > time.Minute { + state.FirstMatchedAt = projectedFirstMatchedAt + if state.BackupEndedElapsedNanos != nil { + backupEndedElapsed := time.Duration(*state.BackupEndedElapsedNanos) + state.BackupEndedAt = intentTimePointer(observedAt.Add(-(elapsed - backupEndedElapsed))) + } + decision.StateChanged = true + } if effective.HonorOperatorState && m.operatorIntentResolver != nil { if operator, ok := m.operatorIntentResolver(resourceID, observedAt); ok { @@ -304,7 +367,7 @@ func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackin } } - eligibleAt := state.FirstMatchedAt.Add(time.Duration(effective.GraceSeconds) * time.Second) + eligibleElapsed := time.Duration(effective.GraceSeconds) * time.Second if effective.BackupOffline != nil && effective.BackupOffline.Enabled && signal == string(AlertIntentSignalOffline) { if backup.Active { if !state.BackupActive || state.BackupEvidence != backup.Evidence { @@ -312,17 +375,21 @@ func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackin } state.BackupActive = true state.BackupEndedAt = nil + state.BackupEndedElapsedNanos = nil state.BackupEvidence = backup.Evidence } else if state.BackupActive { endedAt := observedAt + endedElapsed := state.ElapsedNanos state.BackupActive = false state.BackupEndedAt = &endedAt + state.BackupEndedElapsedNanos = &endedElapsed decision.StateChanged = true } - capAt := state.FirstMatchedAt.Add(time.Duration(effective.BackupOffline.MaxDeferralSeconds) * time.Second) + capElapsed := time.Duration(effective.BackupOffline.MaxDeferralSeconds) * time.Second + capAt := observedAt.Add(durationUntil(capElapsed, elapsed)) decision.HardCapAt = capAt - if state.BackupActive && observedAt.Before(capAt) { + if state.BackupActive && elapsed < capElapsed { m.intentPending[trackingKey] = state decision.Pending = true decision.Suppressed = true @@ -330,23 +397,24 @@ func (m *Manager) evaluateIntentNoLock(resourceID, resourceType, signal, trackin decision.EligibleAt = capAt return decision } - if state.BackupEndedAt != nil { - postEligible := state.BackupEndedAt.Add(time.Duration(effective.BackupOffline.PostGraceSeconds) * time.Second) - if postEligible.After(eligibleAt) { - eligibleAt = postEligible + if state.BackupEndedElapsedNanos != nil { + postEligibleElapsed := time.Duration(*state.BackupEndedElapsedNanos) + time.Duration(effective.BackupOffline.PostGraceSeconds)*time.Second + if postEligibleElapsed > eligibleElapsed { + eligibleElapsed = postEligibleElapsed } } - if eligibleAt.After(capAt) { - eligibleAt = capAt + if eligibleElapsed > capElapsed { + eligibleElapsed = capElapsed } - if !observedAt.Before(capAt) { + if elapsed >= capElapsed { decision.Reason = "backup_grace_cap_exceeded" } } m.intentPending[trackingKey] = state + eligibleAt := observedAt.Add(durationUntil(eligibleElapsed, elapsed)) decision.EligibleAt = eligibleAt - if observedAt.Before(eligibleAt) { + if elapsed < eligibleElapsed { decision.Pending = true if decision.Reason == "" { decision.Reason = "grace_period" @@ -367,6 +435,9 @@ func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) ( if request.ResourceID == "" || request.ResourceType == "" || request.Signal == "" { return AlertIntentPolicyPreview{}, errors.New("resourceId, resourceType, and signal are required") } + if !ValidAlertIntentSignal(request.Signal) { + return AlertIntentPolicyPreview{}, fmt.Errorf("unsupported alert intent signal %q", request.Signal) + } now := m.policyNow().UTC() trackingKey := "preview:" + request.ResourceID + ":" + request.Signal backup := BackupIntentContext{} @@ -380,11 +451,18 @@ func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) ( m.mu.Lock() previous, hadPrevious := m.intentPending[trackingKey] + previousTick, hadPreviousTick := m.intentRuntimeTicks[trackingKey] if request.FirstMatchedAt != nil { + firstMatchedAt := request.FirstMatchedAt.UTC() + elapsed := now.Sub(firstMatchedAt) + if elapsed < 0 { + elapsed = 0 + } m.intentPending[trackingKey] = IntentPendingState{ TrackingKey: trackingKey, ResourceID: request.ResourceID, ResourceType: request.ResourceType, - Signal: request.Signal, FirstMatchedAt: request.FirstMatchedAt.UTC(), + Signal: request.Signal, FirstMatchedAt: firstMatchedAt, LastObservedAt: now, ElapsedNanos: int64(elapsed), } + m.intentRuntimeTicks[trackingKey] = m.intentTickNoLock() } decision := m.evaluateIntentNoLock(request.ResourceID, request.ResourceType, request.Signal, trackingKey, now, request.ConditionActive, backup) state, stateExists := m.intentPending[trackingKey] @@ -398,6 +476,11 @@ func (m *Manager) PreviewIntentPolicy(request AlertIntentPolicyPreviewRequest) ( } else { delete(m.intentPending, trackingKey) } + if hadPreviousTick { + m.intentRuntimeTicks[trackingKey] = previousTick + } else { + delete(m.intentRuntimeTicks, trackingKey) + } m.mu.Unlock() preview := AlertIntentPolicyPreview{ diff --git a/internal/alerts/intent_policy_test.go b/internal/alerts/intent_policy_test.go index 19216392b..2a97a32fc 100644 --- a/internal/alerts/intent_policy_test.go +++ b/internal/alerts/intent_policy_test.go @@ -44,6 +44,38 @@ func TestAlertIntentPolicyResolutionPrecedenceIsFieldByField(t *testing.T) { } } +func TestAlertIntentPolicyResourceTypeInheritanceIsGeneralThenSpecific(t *testing.T) { + m := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(m.Stop) + document := NewAlertIntentPolicyDocument() + document.ResourceTypes["guest"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): { + GraceSeconds: intPointer(300), + HonorOperatorState: boolPointer(true), + }, + } + document.ResourceTypes["vm"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): {HonorOperatorState: boolPointer(false)}, + } + if err := m.LoadIntentPolicies(document); err != nil { + t.Fatal(err) + } + + vm := m.ResolveEffectiveIntentPolicy("vm:101", "vm", string(AlertIntentSignalOffline)) + if vm.GraceSeconds != 300 || vm.HonorOperatorState { + t.Fatalf("vm effective policy = %+v", vm) + } + if vm.Sources["graceSeconds"] != "resourceTypes.guest.state.offline" || + vm.Sources["honorOperatorState"] != "resourceTypes.vm.state.offline" { + t.Fatalf("vm policy sources = %+v", vm.Sources) + } + + node := m.ResolveEffectiveIntentPolicy("node:a", "node", string(AlertIntentSignalOffline)) + if node.Explicit || node.Sources["graceSeconds"] != "factory" { + t.Fatalf("guest powered-off default leaked into node connectivity policy: %+v", node) + } +} + func TestAlertIntentPolicyResolvesCanonicalResourceFromSourceID(t *testing.T) { m := NewManagerWithDataDir(t.TempDir()) t.Cleanup(m.Stop) @@ -86,6 +118,11 @@ func TestAlertIntentPolicyValidationRejectsAmbiguousNormalizedKeys(t *testing.T) func TestAlertIntentBackupDeferralEndsWithPostGraceAndHardCap(t *testing.T) { m := NewManagerWithDataDir(t.TempDir()) t.Cleanup(m.Stop) + start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC) + now := start + var tick time.Duration + m.now = func() time.Time { return now } + m.intentClock = func() time.Duration { return tick } document := NewAlertIntentPolicyDocument() document.Resources["vm:101"] = map[string]AlertIntentRule{ string(AlertIntentSignalOffline): { @@ -99,11 +136,13 @@ func TestAlertIntentBackupDeferralEndsWithPostGraceAndHardCap(t *testing.T) { t.Fatal(err) } - start := time.Date(2026, 7, 13, 9, 0, 0, 0, time.UTC) m.mu.Lock() active := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start, true, BackupIntentContext{Active: true, Evidence: "guest_lock"}) + now, tick = start.Add(100*time.Second), 100*time.Second ended := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(100*time.Second), true, BackupIntentContext{}) + now, tick = start.Add(159*time.Second), 159*time.Second pending := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(159*time.Second), true, BackupIntentContext{}) + now, tick = start.Add(160*time.Second), 160*time.Second eligible := m.evaluateIntentNoLock("vm:101", "vm", string(AlertIntentSignalOffline), "offline:vm:101", start.Add(160*time.Second), true, BackupIntentContext{}) m.mu.Unlock() @@ -158,6 +197,11 @@ func TestAlertIntentPreviewHonorsOperatorStateWithoutMutatingRuntime(t *testing. func TestLifecycleAlertStartsAtFirstIntentMatch(t *testing.T) { m := NewManagerWithDataDir(t.TempDir()) t.Cleanup(m.Stop) + start := time.Date(2026, 7, 13, 11, 0, 0, 0, time.UTC) + now := start + var tick time.Duration + m.now = func() time.Time { return now } + m.intentClock = func() time.Duration { return tick } document := NewAlertIntentPolicyDocument() document.Resources["vm:101"] = map[string]AlertIntentRule{ string(AlertIntentSignalOffline): {GraceSeconds: intPointer(60)}, @@ -170,7 +214,6 @@ func TestLifecycleAlertStartsAtFirstIntentMatch(t *testing.T) { t.Fatal(err) } tracking := make(map[string]int) - start := time.Date(2026, 7, 13, 11, 0, 0, 0, time.UTC) params := canonicalLifecycleAlertParams{ Spec: spec, Evidence: alertspecs.AlertEvidence{ @@ -186,6 +229,7 @@ func TestLifecycleAlertStartsAtFirstIntentMatch(t *testing.T) { if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStatePending { t.Fatalf("initial state = %s, want pending", result.State.State) } + now, tick = start.Add(60*time.Second), 60*time.Second params.Evidence.ObservedAt = start.Add(60 * time.Second) if result, _ := m.evaluateCanonicalLifecycleAlert(params); result.State.State != alertspecs.AlertStateFiring { t.Fatalf("eligible state = %s, want firing", result.State.State) @@ -228,4 +272,53 @@ func TestIntentPendingStatePersistsAcrossRestart(t *testing.T) { if !ok || !restored.FirstMatchedAt.Equal(now.Add(-time.Minute)) { t.Fatalf("restored intent state = %+v, found %v", restored, ok) } + if restored.ElapsedNanos != int64(time.Minute) { + t.Fatalf("restored elapsed = %s, want 1m", time.Duration(restored.ElapsedNanos)) + } +} + +func TestIntentPendingElapsedProgressSurvivesRestartConservatively(t *testing.T) { + dataDir := t.TempDir() + now := time.Now().UTC().Truncate(time.Second) + first := NewManagerWithDataDir(dataDir) + first.mu.Lock() + first.intentPending["state:vm:restart"] = IntentPendingState{ + TrackingKey: "state:vm:restart", ResourceID: "vm:restart", ResourceType: "vm", + Signal: string(AlertIntentSignalOffline), FirstMatchedAt: now.Add(-2 * time.Minute), + LastObservedAt: now, ElapsedNanos: int64(120 * time.Second), + } + first.mu.Unlock() + if err := first.SaveActiveAlerts(); err != nil { + first.Stop() + t.Fatalf("SaveActiveAlerts: %v", err) + } + first.Stop() + + second := NewManagerWithDataDir(dataDir) + t.Cleanup(second.Stop) + document := NewAlertIntentPolicyDocument() + document.Resources["vm:restart"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): {GraceSeconds: intPointer(300)}, + } + if err := second.LoadIntentPolicies(document); err != nil { + t.Fatal(err) + } + wall := now.Add(time.Hour) + var tick time.Duration + second.now = func() time.Time { return wall } + second.intentClock = func() time.Duration { return tick } + + second.mu.Lock() + afterRestart := second.evaluateIntentNoLock("vm:restart", "vm", string(AlertIntentSignalOffline), "state:vm:restart", wall, true, BackupIntentContext{}) + tick = 180 * time.Second + wall = wall.Add(180 * time.Second) + eligible := second.evaluateIntentNoLock("vm:restart", "vm", string(AlertIntentSignalOffline), "state:vm:restart", wall, true, BackupIntentContext{}) + second.mu.Unlock() + + if !afterRestart.Pending || afterRestart.ShouldActivate { + t.Fatalf("restart counted unobserved process downtime: %+v", afterRestart) + } + if !eligible.ShouldActivate { + t.Fatalf("persisted plus post-restart elapsed time did not reach tolerance: %+v", eligible) + } } diff --git a/internal/alerts/manager.go b/internal/alerts/manager.go index b8fd58908..1d1733e97 100644 --- a/internal/alerts/manager.go +++ b/internal/alerts/manager.go @@ -48,10 +48,13 @@ type Manager struct { resolvedMutex sync.RWMutex // Secondary lock - see Lock Ordering Documentation above // Time threshold tracking pendingAlerts map[string]time.Time // Track when thresholds were first exceeded - // Intent-policy pending state retains wall-clock and transient-context - // evidence for policy-enabled candidates. It is keyed by canonical alert - // tracking key and persisted with the alert manager's transition state. + // Intent-policy pending state retains server timestamps, accumulated + // monotonic elapsed time, and transient-context evidence for policy-enabled + // candidates. It is keyed by canonical alert tracking key and persisted + // with the alert manager's transition state. intentPending map[string]IntentPendingState + intentRuntimeTicks map[string]time.Duration + intentClock func() time.Duration intentPolicies AlertIntentPolicyDocument operatorIntentResolver OperatorIntentContextResolver backupIntentResolver BackupIntentContextResolver @@ -137,6 +140,7 @@ func NewManagerWithDataDir(dataDir string) *Manager { resolvedAlias: make(map[string]string), pendingAlerts: make(map[string]time.Time), intentPending: make(map[string]IntentPendingState), + intentRuntimeTicks: make(map[string]time.Duration), intentPolicies: NewAlertIntentPolicyDocument(), nodeOfflineCount: make(map[string]int), connectionDegradedCount: make(map[string]int), @@ -160,6 +164,10 @@ func NewManagerWithDataDir(dataDir string) *Manager { now: time.Now, config: defaultAlertConfig(), } + intentClockEpoch := time.Now() + m.intentClock = func() time.Duration { + return time.Since(intentClockEpoch) + } // Load saved active alerts if err := m.LoadActiveAlerts(); err != nil { diff --git a/internal/alerts/metric_runtime.go b/internal/alerts/metric_runtime.go index 49b8ecd61..1b9c5d172 100644 --- a/internal/alerts/metric_runtime.go +++ b/internal/alerts/metric_runtime.go @@ -237,7 +237,7 @@ func (m *Manager) checkMetric(resourceID, resourceName, node, instance, resource if !decision.ShouldActivate { return } - delete(m.intentPending, trackingKey) + m.clearIntentPendingNoLock(trackingKey) m.saveActiveAlertsAsync("metric intent activated") } else if timeThreshold := m.getTimeThreshold(resourceID, resourceType, metricType); timeThreshold > 0 { // Check if this threshold was already pending diff --git a/internal/alerts/powered_off_tolerance_test.go b/internal/alerts/powered_off_tolerance_test.go new file mode 100644 index 000000000..676c5d77f --- /dev/null +++ b/internal/alerts/powered_off_tolerance_test.go @@ -0,0 +1,240 @@ +package alerts + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +type poweredOffTestClock struct { + wall time.Time + tick time.Duration +} + +func newPoweredOffTestManager(t *testing.T, start time.Time) (*Manager, *poweredOffTestClock) { + t.Helper() + manager := NewManagerWithDataDir(t.TempDir()) + t.Cleanup(manager.Stop) + clock := &poweredOffTestClock{wall: start} + manager.now = func() time.Time { return clock.wall } + manager.intentClock = func() time.Duration { return clock.tick } + return manager, clock +} + +func installPoweredOffPolicy(t *testing.T, manager *Manager, graceSeconds int, backup *BackupOfflineIntentPolicy) { + t.Helper() + document := NewAlertIntentPolicyDocument() + document.ResourceTypes["guest"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): { + GraceSeconds: intPointer(graceSeconds), + BackupOffline: backup, + }, + } + if err := manager.LoadIntentPolicies(document); err != nil { + t.Fatalf("LoadIntentPolicies() error = %v", err) + } +} + +func hasPoweredOffAlert(manager *Manager, resourceID string) bool { + manager.mu.RLock() + defer manager.mu.RUnlock() + _, exists := manager.getActiveAlertNoLock(canonicalPoweredStateStateID(resourceID)) + return exists +} + +func TestPoweredOffToleranceUsesElapsedTimeForVMAndLXC(t *testing.T) { + for _, test := range []struct { + name string + resourceID string + resourceType string + }{ + {name: "vm", resourceID: "vm:101", resourceType: "VM"}, + {name: "lxc", resourceID: "lxc:202", resourceType: "Container"}, + } { + t.Run(test.name, func(t *testing.T) { + start := time.Date(2026, 7, 24, 8, 0, 0, 0, time.UTC) + manager, clock := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 300, nil) + + for range 20 { + manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false) + } + if hasPoweredOffAlert(manager, test.resourceID) { + t.Fatal("duplicate reports activated the alert before the duration elapsed") + } + manager.mu.RLock() + _, counted := manager.offlineConfirmations[test.resourceID] + manager.mu.RUnlock() + if counted { + t.Fatal("duration-based powered-off policy must not retain poll confirmations") + } + + clock.wall, clock.tick = start.Add(299*time.Second), 299*time.Second + manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false) + if hasPoweredOffAlert(manager, test.resourceID) { + t.Fatal("alert activated before the configured duration") + } + + clock.wall, clock.tick = start.Add(300*time.Second), 300*time.Second + manager.checkGuestPoweredOff(test.resourceID, test.name, "node-a", "pve-a", test.resourceType, false) + if !hasPoweredOffAlert(manager, test.resourceID) { + t.Fatal("alert did not activate at the configured duration") + } + }) + } +} + +func TestPoweredOffToleranceZeroAndInheritance(t *testing.T) { + start := time.Date(2026, 7, 24, 9, 0, 0, 0, time.UTC) + manager, _ := newPoweredOffTestManager(t, start) + document := NewAlertIntentPolicyDocument() + document.ResourceTypes["guest"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): {GraceSeconds: intPointer(300)}, + } + document.Resources["vm:zero"] = map[string]AlertIntentRule{ + string(AlertIntentSignalOffline): {GraceSeconds: intPointer(0)}, + } + if err := manager.LoadIntentPolicies(document); err != nil { + t.Fatal(err) + } + + manager.checkGuestPoweredOff("vm:zero", "zero", "node-a", "pve-a", "VM", false) + if !hasPoweredOffAlert(manager, "vm:zero") { + t.Fatal("explicit zero did not activate on the first stopped observation") + } + manager.checkGuestPoweredOff("vm:inherit", "inherit", "node-a", "pve-a", "VM", false) + if hasPoweredOffAlert(manager, "vm:inherit") { + t.Fatal("inherited guest tolerance was not applied") + } +} + +func TestPoweredOffToleranceIgnoresWallClockChanges(t *testing.T) { + start := time.Date(2026, 7, 24, 10, 0, 0, 0, time.UTC) + manager, clock := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 300, nil) + + manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false) + clock.wall = start.Add(24 * time.Hour) + manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false) + if hasPoweredOffAlert(manager, "vm:clock") { + t.Fatal("forward wall-clock jump activated the alert") + } + + clock.wall = start.Add(-24 * time.Hour) + clock.tick = 300 * time.Second + manager.checkGuestPoweredOff("vm:clock", "clock", "node-a", "pve-a", "VM", false) + if !hasPoweredOffAlert(manager, "vm:clock") { + t.Fatal("monotonic elapsed time did not activate after a backward wall-clock jump") + } +} + +func TestPoweredOffToleranceClearsOnRecoveryAndSuppression(t *testing.T) { + start := time.Date(2026, 7, 24, 11, 0, 0, 0, time.UTC) + manager, clock := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 300, nil) + + manager.checkGuestPoweredOff("vm:flap", "flap", "node-a", "pve-a", "VM", false) + clock.wall, clock.tick = start.Add(200*time.Second), 200*time.Second + manager.clearGuestPoweredOffAlert("vm:flap", "flap") + clock.wall, clock.tick = start.Add(300*time.Second), 300*time.Second + manager.checkGuestPoweredOff("vm:flap", "flap", "node-a", "pve-a", "VM", false) + if hasPoweredOffAlert(manager, "vm:flap") { + t.Fatal("recovery did not reset the powered-off duration") + } + + if !manager.suppressGuestAlerts("vm:flap") { + t.Fatal("suppression did not report clearing pending intent state") + } + manager.mu.RLock() + pending := len(manager.intentPending) + ticks := len(manager.intentRuntimeTicks) + manager.mu.RUnlock() + if pending != 0 || ticks != 0 { + t.Fatalf("suppression left pending state: pending=%d ticks=%d", pending, ticks) + } +} + +func TestPoweredOffToleranceRecoveryNotifiesAfterActivation(t *testing.T) { + start := time.Date(2026, 7, 24, 11, 30, 0, 0, time.UTC) + manager, clock := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 60, nil) + resolved := make(chan string, 1) + manager.SetResolvedCallback(func(alertID string) { + resolved <- alertID + }) + + manager.checkGuestPoweredOff("vm:recover", "recover", "node-a", "pve-a", "VM", false) + clock.wall, clock.tick = start.Add(60*time.Second), 60*time.Second + manager.checkGuestPoweredOff("vm:recover", "recover", "node-a", "pve-a", "VM", false) + if !hasPoweredOffAlert(manager, "vm:recover") { + t.Fatal("powered-off alert did not activate before recovery") + } + + manager.clearGuestPoweredOffAlert("vm:recover", "recover") + select { + case alertID := <-resolved: + if alertID == "" { + t.Fatal("recovery callback omitted the alert identity") + } + case <-time.After(time.Second): + t.Fatal("activated powered-off alert did not emit a recovery callback") + } + if hasPoweredOffAlert(manager, "vm:recover") { + t.Fatal("powered-off alert remained active after recovery") + } +} + +func TestPoweredOffBackupDeferralHasHardCap(t *testing.T) { + start := time.Date(2026, 7, 24, 12, 0, 0, 0, time.UTC) + manager, clock := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 300, &BackupOfflineIntentPolicy{ + Enabled: true, PostGraceSeconds: 60, MaxDeferralSeconds: 600, + }) + backup := BackupIntentContext{Active: true, Evidence: "fresh_task"} + + manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup) + clock.wall, clock.tick = start.Add(599*time.Second), 599*time.Second + manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup) + if hasPoweredOffAlert(manager, "vm:backup") { + t.Fatal("backup deferral activated before its hard cap") + } + + clock.wall, clock.tick = start.Add(600*time.Second), 600*time.Second + manager.checkGuestPoweredOffWithThresholdsAndIntent("vm:backup", "backup", "node-a", "pve-a", "VM", manager.config.GuestDefaults, false, backup) + if !hasPoweredOffAlert(manager, "vm:backup") { + t.Fatal("backup evidence hid an outage beyond the hard cap") + } +} + +func TestPoweredOffToleranceConcurrentReportsActivateOnce(t *testing.T) { + start := time.Date(2026, 7, 24, 13, 0, 0, 0, time.UTC) + manager, _ := newPoweredOffTestManager(t, start) + installPoweredOffPolicy(t, manager, 0, nil) + manager.config.ActivationState = ActivationActive + var notifications atomic.Int32 + manager.SetAlertCallback(func(*Alert) { + notifications.Add(1) + }) + + var workers sync.WaitGroup + for range 32 { + workers.Add(1) + go func() { + defer workers.Done() + manager.checkGuestPoweredOff("vm:race", "race", "node-a", "pve-a", "VM", false) + }() + } + workers.Wait() + + if !hasPoweredOffAlert(manager, "vm:race") { + t.Fatal("concurrent reports did not activate the alert") + } + if got := notifications.Load(); got != 1 { + t.Fatalf("notification count = %d, want 1", got) + } + history := manager.GetAlertHistory(10) + if len(history) != 1 { + t.Fatalf("history entries = %d, want 1", len(history)) + } +} diff --git a/internal/alerts/tracking_cleanup.go b/internal/alerts/tracking_cleanup.go index 88295116b..ec77ca3bc 100644 --- a/internal/alerts/tracking_cleanup.go +++ b/internal/alerts/tracking_cleanup.go @@ -60,6 +60,16 @@ func (m *Manager) cleanupStaleMaps() { } } + for trackingKey, pending := range m.intentPending { + if m.hasActiveAlertNoLock(trackingKey) { + continue + } + if pending.LastObservedAt.IsZero() || now.Sub(pending.LastObservedAt) > staleThreshold { + m.clearIntentPendingNoLock(trackingKey) + cleaned++ + } + } + for resourceID := range m.offlineConfirmations { hasRelatedAlert := false for storageKey, alert := range m.activeAlerts { diff --git a/internal/alerts/unified_incidents.go b/internal/alerts/unified_incidents.go index 1643dc80e..e0fce75da 100644 --- a/internal/alerts/unified_incidents.go +++ b/internal/alerts/unified_incidents.go @@ -126,7 +126,7 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso observedAvailability[storageKey] = struct{}{} if existing, exists := m.getActiveAlertNoLock(storageKey); exists && existing != nil { if _, pending := m.intentPending[storageKey]; pending { - delete(m.intentPending, storageKey) + m.clearIntentPendingNoLock(storageKey) intentStateChanged = true } continue @@ -142,7 +142,7 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso alert.StartTime = pending.FirstMatchedAt } if _, ok := m.intentPending[storageKey]; ok { - delete(m.intentPending, storageKey) + m.clearIntentPendingNoLock(storageKey) intentStateChanged = true } } @@ -153,7 +153,7 @@ func (m *Manager) SyncUnifiedResourceIncidents(resources []unifiedresources.Reso if _, observed := observedAvailability[storageKey]; observed { continue } - delete(m.intentPending, storageKey) + m.clearIntentPendingNoLock(storageKey) intentStateChanged = true } if intentStateChanged { diff --git a/internal/api/alerts.go b/internal/api/alerts.go index 0ef441da2..94b345075 100644 --- a/internal/api/alerts.go +++ b/internal/api/alerts.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "strconv" "strings" @@ -326,10 +327,16 @@ func (h *AlertHandlers) GetAlertIntentPolicies(w http.ResponseWriter, r *http.Re func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, 256*1024) var document alerts.AlertIntentPolicyDocument - if err := json.NewDecoder(r.Body).Decode(&document); err != nil { + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&document); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + http.Error(w, "request body must contain one JSON object", http.StatusBadRequest) + return + } // Treat revision check, in-memory install, and durable save as one API // transaction. This also prevents a failed earlier writer from rolling // back a later successful update. @@ -371,10 +378,16 @@ func (h *AlertHandlers) UpdateAlertIntentPolicies(w http.ResponseWriter, r *http func (h *AlertHandlers) PreviewAlertIntentPolicy(w http.ResponseWriter, r *http.Request) { r.Body = http.MaxBytesReader(w, r.Body, 64*1024) var request alerts.AlertIntentPolicyPreviewRequest - if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + decoder := json.NewDecoder(r.Body) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&request); err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return } + if err := decoder.Decode(&struct{}{}); err != io.EOF { + http.Error(w, "request body must contain one JSON object", http.StatusBadRequest) + return + } h.intentPolicyMu.Lock() defer h.intentPolicyMu.Unlock() manager, ok := h.getMonitor(r.Context()).GetAlertManager().(alertIntentPolicyManager) diff --git a/internal/api/alerts_endpoints_test.go b/internal/api/alerts_endpoints_test.go index 72688697b..c3596517f 100644 --- a/internal/api/alerts_endpoints_test.go +++ b/internal/api/alerts_endpoints_test.go @@ -146,6 +146,42 @@ func TestAlertsEndpoints(t *testing.T) { if preview.Status != "pending_grace" || preview.Effective.GraceSeconds != grace { t.Fatalf("preview = %+v", preview) } + + for name, payload := range map[string]string{ + "unsupported signal": `{"resourceId":"vm:101","resourceType":"vm","signal":"state.unknown","conditionActive":true}`, + "unknown field": `{"resourceId":"vm:101","resourceType":"vm","signal":"state.offline","conditionActive":true,"pollCount":2}`, + "trailing document": `{"resourceId":"vm:101","resourceType":"vm","signal":"state.offline","conditionActive":true}{}`, + } { + t.Run(name, func(t *testing.T) { + response, err := http.Post( + srv.server.URL+"/api/alerts/intent-policies/preview", + "application/json", + bytes.NewBufferString(payload), + ) + if err != nil { + t.Fatalf("invalid preview request: %v", err) + } + defer response.Body.Close() + if response.StatusCode != http.StatusBadRequest { + t.Fatalf("invalid preview status = %d, want %d", response.StatusCode, http.StatusBadRequest) + } + }) + } + + unknownPolicy := []byte(`{"schemaVersion":1,"revision":1,"defaults":{},"resourceTypes":{},"resources":{},"pollTolerance":2}`) + unknownRequest, err := http.NewRequest(http.MethodPut, srv.server.URL+"/api/alerts/intent-policies", bytes.NewReader(unknownPolicy)) + if err != nil { + t.Fatalf("create unknown-field policy update: %v", err) + } + unknownRequest.Header.Set("Content-Type", "application/json") + unknownResponse, err := http.DefaultClient.Do(unknownRequest) + if err != nil { + t.Fatalf("unknown-field policy update: %v", err) + } + defer unknownResponse.Body.Close() + if unknownResponse.StatusCode != http.StatusBadRequest { + t.Fatalf("unknown-field policy status = %d, want %d", unknownResponse.StatusCode, http.StatusBadRequest) + } }) // 3. Activate alerts diff --git a/tests/integration/tests/85-powered-off-tolerance.spec.ts b/tests/integration/tests/85-powered-off-tolerance.spec.ts new file mode 100644 index 000000000..37abe97d1 --- /dev/null +++ b/tests/integration/tests/85-powered-off-tolerance.spec.ts @@ -0,0 +1,165 @@ +import fs from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { expect, test as base, type Route } from "@playwright/test"; +import { createAuthenticatedStorageState } from "./helpers"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +type WorkerFixtures = { authStorageStatePath: string }; + +const test = base.extend<{}, WorkerFixtures>({ + storageState: async ({ authStorageStatePath }, use) => + use(authStorageStatePath), + authStorageStatePath: [ + async ({ browser }, use, workerInfo) => { + const storageStatePath = path.resolve( + __dirname, + "..", + "..", + "tmp", + "playwright-auth", + `powered-off-tolerance-${workerInfo.project.name}.json`, + ); + fs.mkdirSync(path.dirname(storageStatePath), { recursive: true }); + await createAuthenticatedStorageState(browser, storageStatePath); + try { + await use(storageStatePath); + } finally { + fs.rmSync(storageStatePath, { force: true }); + } + }, + { scope: "worker" }, + ], +}); + +test("powered-off tolerance preserves inheritance, explicit zero, and strict validation", async ({ + page, +}, testInfo) => { + let document = { + schemaVersion: 1, + revision: 0, + defaults: {}, + resourceTypes: {}, + resources: {}, + }; + const updates: Array> = []; + + await page.route("**/api/resources?**", async (route: Route) => { + const url = new URL(route.request().url()); + const pageNumber = Number(url.searchParams.get("page") ?? "1"); + const data = + pageNumber === 1 + ? [ + { + id: "vm:1567", + type: "vm", + name: "backup-window-vm", + displayName: "backup-window-vm", + status: "stopped", + lastSeen: "2026-07-24T12:00:00Z", + sources: ["proxmox"], + platformScopes: ["proxmox"], + platformType: "proxmox", + }, + ] + : []; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ + data, + meta: { + page: pageNumber, + limit: 100, + total: 1, + totalPages: 1, + }, + }), + }); + }); + + await page.route("**/api/alerts/intent-policies", async (route: Route) => { + if (route.request().method() === "GET") { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(document), + }); + return; + } + const update = route.request().postDataJSON() as Record; + updates.push(update); + document = { + ...update, + revision: Number(update.revision) + 1, + } as typeof document; + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify(document), + }); + }); + + await page.goto("/alerts/thresholds", { waitUntil: "domcontentloaded" }); + await page.getByRole("button", { name: "Configure policies" }).click(); + + const tolerance = page.getByRole("spinbutton", { + name: /VM \/ container powered-off tolerance/, + }); + await expect(tolerance).toHaveValue(""); + await expect( + page.getByText( + "Blank inherits the existing policy; 0 alerts on the first stopped observation.", + { exact: true }, + ), + ).toBeVisible(); + await expect(page.getByRole("combobox", { name: "Resource" })).toHaveValue( + "vm:1567", + ); + await tolerance.fill("300"); + await page.getByRole("button", { name: "Save defaults" }).click(); + await expect( + page.getByText("Default intent policy saved.", { exact: true }), + ).toBeVisible(); + expect(updates).toHaveLength(1); + expect(updates[0]).toMatchObject({ + resourceTypes: { + guest: { + "state.offline": { + graceSeconds: 300, + }, + }, + }, + }); + + const override = page.getByRole("spinbutton", { + name: /Grace override \(seconds\)/, + }); + await override.fill("1.5"); + await page.getByRole("button", { name: "Save override" }).click(); + await expect( + page.getByText("Grace override must be a whole number of seconds.", { + exact: true, + }), + ).toBeVisible(); + expect(updates).toHaveLength(1); + + await override.fill("0"); + await page.getByRole("button", { name: "Save override" }).click(); + await expect( + page.getByText("Resource intent override saved.", { exact: true }), + ).toBeVisible(); + expect(updates).toHaveLength(2); + const resourceRules = updates[1].resources as Record< + string, + Record + >; + expect(Object.values(resourceRules)[0]["state.offline"]).toEqual({ + graceSeconds: 0, + }); + + await testInfo.attach("powered-off-tolerance", { + body: await page.screenshot(), + contentType: "image/png", + }); +});