From 0966ae9594c07cd0e460d1fcb0d3641bc60fba96 Mon Sep 17 00:00:00 2001 From: "courtmanr@gmail.com" Date: Mon, 27 Jul 2026 10:15:48 +0100 Subject: [PATCH] Measure verified telemetry outcomes --- docs/PRIVACY.md | 12 ++ frontend-modern/public/docs/PRIVACY.md | 12 ++ .../src/api/__tests__/settings.test.ts | 9 +- frontend-modern/src/api/settings.ts | 7 ++ .../__tests__/useSystemSettingsState.test.ts | 9 +- internal/api/telemetry_pulse_intelligence.go | 67 +++++++++-- .../api/telemetry_pulse_intelligence_test.go | 81 ++++++++++++++ internal/telemetry/telemetry.go | 42 ++++++- internal/telemetry/telemetry_test.go | 16 ++- pkg/server/telemetry_pulse_intelligence.go | 30 ++--- .../telemetry_pulse_intelligence_test.go | 88 ++++++++++----- scripts/telemetry_adoption_report.py | 105 +++++++++++++++++- .../tests/test_telemetry_adoption_report.py | 70 ++++++++++++ 13 files changed, 488 insertions(+), 60 deletions(-) diff --git a/docs/PRIVACY.md b/docs/PRIVACY.md index c6031bc1ff..f5bb054ae7 100644 --- a/docs/PRIVACY.md +++ b/docs/PRIVACY.md @@ -163,6 +163,13 @@ Every field is listed below with the reason it exists. Nothing else is included | Pulse Intelligence approved action failures (execution) 30d | `1` | Count approved governed actions whose dispatched execution failed in the current 30-day telemetry window without sending action output, error text, command text, resource IDs, or actors | | Pulse Intelligence approved action failures (unverified) 30d | `1` | Count approved governed actions that executed but whose outcome verification was not confirmed in the current 30-day telemetry window without sending verification evidence, action output, command text, resource IDs, or actors | | Pulse Intelligence approved action stuck executing 30d | `1` | Count approved governed actions abandoned in the executing state in the current 30-day telemetry window without sending action output, command text, resource IDs, or actors | +| Pulse Intelligence approved action in flight 30d | `1` | Count approved governed actions still inside the legitimate execution window so the action-attempt total reconciles without sending action details | +| Pulse Intelligence approved action unclassified 30d | `1` | Count approved governed action attempts whose authoritative audit could not be classified so accounting gaps remain visible without sending action IDs or content | +| Pulse Intelligence approved action refusals plan stale 30d | `1` | Count pre-dispatch refusals caused by an expired, drifted, or legacy-unbound plan without sending plan or resource details | +| Pulse Intelligence approved action refusals: policy 30d | `1` | Count pre-dispatch refusals caused by an operator or policy safety control without sending policy, actor, resource, or command details | +| Pulse Intelligence approved action refusals: capability 30d | `1` | Count pre-dispatch refusals caused by a dry-run-only or unavailable capability without sending capability names, targets, or command details | +| Pulse Intelligence approved action refusals: other 30d | `1` | Count pre-dispatch refusals not covered by the fixed categories without sending raw error or action content | +| Pulse Intelligence verified finding resolutions 30d | `1` | Count approved Patrol-origin actions whose execution succeeded and whose linked finding postcondition was independently confirmed, without sending finding IDs, action IDs, evidence, resources, or fix details | | Pulse Intelligence approved action last failure reason 30d | `plan_drift` | See one fixed machine reason code for the most recent approved-action failure in the current 30-day telemetry window without sending error text, action output, command text, resource IDs, or actors | Telemetry schema v3 corrects the meaning of `notification_failures_7d`: v2 @@ -172,6 +179,11 @@ The adoption report keeps those schema cohorts separate, so legacy retry noise is not compared with current terminal-delivery failures. This correction adds no notification content or identity fields. +Telemetry schema v4 adds complete approved-action outcome accounting, fixed +pre-dispatch refusal categories, and an independently verified +action-to-finding resolution count. These remain aggregate counters and do not +send action, finding, resource, evidence, actor, or command identity or content. + #### Server-side handling and retention - Telemetry pings are stored on the Pulse license server only for aggregate install/use analysis. diff --git a/frontend-modern/public/docs/PRIVACY.md b/frontend-modern/public/docs/PRIVACY.md index c6031bc1ff..f5bb054ae7 100644 --- a/frontend-modern/public/docs/PRIVACY.md +++ b/frontend-modern/public/docs/PRIVACY.md @@ -163,6 +163,13 @@ Every field is listed below with the reason it exists. Nothing else is included | Pulse Intelligence approved action failures (execution) 30d | `1` | Count approved governed actions whose dispatched execution failed in the current 30-day telemetry window without sending action output, error text, command text, resource IDs, or actors | | Pulse Intelligence approved action failures (unverified) 30d | `1` | Count approved governed actions that executed but whose outcome verification was not confirmed in the current 30-day telemetry window without sending verification evidence, action output, command text, resource IDs, or actors | | Pulse Intelligence approved action stuck executing 30d | `1` | Count approved governed actions abandoned in the executing state in the current 30-day telemetry window without sending action output, command text, resource IDs, or actors | +| Pulse Intelligence approved action in flight 30d | `1` | Count approved governed actions still inside the legitimate execution window so the action-attempt total reconciles without sending action details | +| Pulse Intelligence approved action unclassified 30d | `1` | Count approved governed action attempts whose authoritative audit could not be classified so accounting gaps remain visible without sending action IDs or content | +| Pulse Intelligence approved action refusals plan stale 30d | `1` | Count pre-dispatch refusals caused by an expired, drifted, or legacy-unbound plan without sending plan or resource details | +| Pulse Intelligence approved action refusals: policy 30d | `1` | Count pre-dispatch refusals caused by an operator or policy safety control without sending policy, actor, resource, or command details | +| Pulse Intelligence approved action refusals: capability 30d | `1` | Count pre-dispatch refusals caused by a dry-run-only or unavailable capability without sending capability names, targets, or command details | +| Pulse Intelligence approved action refusals: other 30d | `1` | Count pre-dispatch refusals not covered by the fixed categories without sending raw error or action content | +| Pulse Intelligence verified finding resolutions 30d | `1` | Count approved Patrol-origin actions whose execution succeeded and whose linked finding postcondition was independently confirmed, without sending finding IDs, action IDs, evidence, resources, or fix details | | Pulse Intelligence approved action last failure reason 30d | `plan_drift` | See one fixed machine reason code for the most recent approved-action failure in the current 30-day telemetry window without sending error text, action output, command text, resource IDs, or actors | Telemetry schema v3 corrects the meaning of `notification_failures_7d`: v2 @@ -172,6 +179,11 @@ The adoption report keeps those schema cohorts separate, so legacy retry noise is not compared with current terminal-delivery failures. This correction adds no notification content or identity fields. +Telemetry schema v4 adds complete approved-action outcome accounting, fixed +pre-dispatch refusal categories, and an independently verified +action-to-finding resolution count. These remain aggregate counters and do not +send action, finding, resource, evidence, actor, or command identity or content. + #### Server-side handling and retention - Telemetry pings are stored on the Pulse license server only for aggregate install/use analysis. diff --git a/frontend-modern/src/api/__tests__/settings.test.ts b/frontend-modern/src/api/__tests__/settings.test.ts index c0354208a3..561ce5ebdb 100644 --- a/frontend-modern/src/api/__tests__/settings.test.ts +++ b/frontend-modern/src/api/__tests__/settings.test.ts @@ -7,7 +7,7 @@ vi.mock('@/utils/apiClient', () => ({ })); const mockTelemetryPreviewPayload = { - schema_version: 3, + schema_version: 4, sent_at: '2026-07-23T08:30:00Z', install_id: 'preview-install-id', version: '6.0.0', @@ -131,6 +131,13 @@ const mockTelemetryPreviewPayload = { pulse_intelligence_approved_action_failures_execution_30d: 0, pulse_intelligence_approved_action_failures_unverified_30d: 0, pulse_intelligence_approved_action_stuck_executing_30d: 0, + pulse_intelligence_approved_action_in_flight_30d: 0, + pulse_intelligence_approved_action_unclassified_30d: 0, + pulse_intelligence_approved_action_refusals_plan_stale_30d: 0, + pulse_intelligence_approved_action_refusals_policy_30d: 0, + pulse_intelligence_approved_action_refusals_capability_30d: 0, + pulse_intelligence_approved_action_refusals_other_30d: 0, + pulse_intelligence_verified_finding_resolutions_30d: 0, pulse_intelligence_approved_action_last_failure_reason_30d: undefined, } satisfies TelemetryPingPreview; diff --git a/frontend-modern/src/api/settings.ts b/frontend-modern/src/api/settings.ts index 4cafd8f716..90e154c871 100644 --- a/frontend-modern/src/api/settings.ts +++ b/frontend-modern/src/api/settings.ts @@ -132,6 +132,13 @@ export interface TelemetryPingPreview { pulse_intelligence_approved_action_failures_execution_30d: number; pulse_intelligence_approved_action_failures_unverified_30d: number; pulse_intelligence_approved_action_stuck_executing_30d: number; + pulse_intelligence_approved_action_in_flight_30d: number; + pulse_intelligence_approved_action_unclassified_30d: number; + pulse_intelligence_approved_action_refusals_plan_stale_30d: number; + pulse_intelligence_approved_action_refusals_policy_30d: number; + pulse_intelligence_approved_action_refusals_capability_30d: number; + pulse_intelligence_approved_action_refusals_other_30d: number; + pulse_intelligence_verified_finding_resolutions_30d: number; pulse_intelligence_approved_action_last_failure_reason_30d?: string; } diff --git a/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts b/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts index 1138bd9046..b96f419814 100644 --- a/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts +++ b/frontend-modern/src/components/Settings/__tests__/useSystemSettingsState.test.ts @@ -13,7 +13,7 @@ const flushAsync = async () => { const buildTelemetryPreviewPayload = ( overrides: Partial = {}, ): TelemetryPingPreview => ({ - schema_version: 3, + schema_version: 4, sent_at: '2026-07-23T08:30:00Z', install_id: 'preview-install-id', version: '6.0.0', @@ -137,6 +137,13 @@ const buildTelemetryPreviewPayload = ( pulse_intelligence_approved_action_failures_execution_30d: 0, pulse_intelligence_approved_action_failures_unverified_30d: 0, pulse_intelligence_approved_action_stuck_executing_30d: 0, + pulse_intelligence_approved_action_in_flight_30d: 0, + pulse_intelligence_approved_action_unclassified_30d: 0, + pulse_intelligence_approved_action_refusals_plan_stale_30d: 0, + pulse_intelligence_approved_action_refusals_policy_30d: 0, + pulse_intelligence_approved_action_refusals_capability_30d: 0, + pulse_intelligence_approved_action_refusals_other_30d: 0, + pulse_intelligence_verified_finding_resolutions_30d: 0, pulse_intelligence_approved_action_last_failure_reason_30d: undefined, ...overrides, }); diff --git a/internal/api/telemetry_pulse_intelligence.go b/internal/api/telemetry_pulse_intelligence.go index 2b8525d4a6..262d12ada3 100644 --- a/internal/api/telemetry_pulse_intelligence.go +++ b/internal/api/telemetry_pulse_intelligence.go @@ -7,6 +7,7 @@ import ( "github.com/rcourtman/pulse-go-rewrite/internal/telemetry" unifiedresources "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" + "github.com/rcourtman/pulse-go-rewrite/pkg/aicontracts" "github.com/rs/zerolog/log" ) @@ -76,7 +77,7 @@ func (r *Router) GetPulseIntelligenceActionTelemetry(since time.Time) telemetry. snapshot.ApprovedActionDecisions30d += len(approvedDecisionIDs) snapshot.ApprovedActionAttempts30d += len(approvedAttemptIDs) snapshot.ApprovedActionSuccesses30d += len(approvedSuccessIDs) - accumulatePulseIntelligenceApprovedActionFailures(&snapshot, store, orgID, approvedAttemptIDs, approvedSuccessIDs, recordsByID, time.Now().UTC()) + accumulatePulseIntelligenceApprovedActionOutcomes(&snapshot, store, orgID, approvedAttemptIDs, approvedSuccessIDs, recordsByID, time.Now().UTC()) } return snapshot @@ -92,15 +93,14 @@ const pulseIntelligenceStuckExecutingThreshold = time.Hour // executor misuses the reason-code field. var pulseIntelligenceReasonCodePattern = regexp.MustCompile(`^[a-z0-9_.-]{1,64}$`) -// accumulatePulseIntelligenceApprovedActionFailures attributes every approved -// attempt that is not a success to one cause bucket, and records the machine -// reason code of the most recent failure. -func accumulatePulseIntelligenceApprovedActionFailures(snapshot *telemetry.PulseIntelligenceActionSnapshot, store unifiedresources.ResourceStore, orgID string, attemptIDs, successIDs map[string]struct{}, recordsByID map[string]unifiedresources.ActionAuditRecord, now time.Time) { +// accumulatePulseIntelligenceApprovedActionOutcomes attributes every approved +// attempt to exactly one success, failure, stuck, in-flight, or unclassified +// bucket. It also records fixed refusal categories and the machine reason code +// of the most recent failure. +func accumulatePulseIntelligenceApprovedActionOutcomes(snapshot *telemetry.PulseIntelligenceActionSnapshot, store unifiedresources.ResourceStore, orgID string, attemptIDs, successIDs map[string]struct{}, recordsByID map[string]unifiedresources.ActionAuditRecord, now time.Time) { var lastFailureAt time.Time for actionID := range attemptIDs { - if _, ok := successIDs[actionID]; ok { - continue - } + _, isSuccess := successIDs[actionID] record, ok := recordsByID[actionID] if !ok { fetched, found, err := store.GetActionAudit(actionID) @@ -108,21 +108,46 @@ func accumulatePulseIntelligenceApprovedActionFailures(snapshot *telemetry.Pulse if err != nil { log.Warn().Err(err).Str("org_id", orgID).Msg("Unable to resolve action audit for failure-cause telemetry summary") } + // A success outcome is already accounted by its lifecycle event; + // only unresolved non-success attempts belong in unclassified. + if !isSuccess { + snapshot.ApprovedActionUnclassified30d++ + } continue } record = fetched } + if isSuccess { + if pulseIntelligenceVerifiedFindingResolution(record) { + snapshot.VerifiedFindingResolutions30d++ + } + continue + } cause, reason := pulseIntelligenceApprovedActionFailureCause(record, now) switch cause { case "pre_dispatch": snapshot.ApprovedActionFailuresPreDispatch30d++ + switch pulseIntelligenceApprovedActionRefusalCategory(reason) { + case "plan_stale": + snapshot.ApprovedActionRefusalsPlanStale30d++ + case "policy": + snapshot.ApprovedActionRefusalsPolicy30d++ + case "capability": + snapshot.ApprovedActionRefusalsCapability30d++ + default: + snapshot.ApprovedActionRefusalsOther30d++ + } case "execution": snapshot.ApprovedActionFailuresExecution30d++ case "unverified": snapshot.ApprovedActionFailuresUnverified30d++ case "stuck_executing": snapshot.ApprovedActionStuckExecuting30d++ + case "in_flight": + snapshot.ApprovedActionInFlight30d++ + continue default: + snapshot.ApprovedActionUnclassified30d++ continue } if record.UpdatedAt.After(lastFailureAt) { @@ -134,15 +159,15 @@ func accumulatePulseIntelligenceApprovedActionFailures(snapshot *telemetry.Pulse // pulseIntelligenceApprovedActionFailureCause classifies an approved attempt // that is not a verified success into a coarse cause bucket plus the specific -// machine reason code. A recently-executing record returns no cause: it is -// still in flight and may yet succeed. +// machine reason code. A recently-executing record is explicitly classified as +// in flight because it may yet succeed. func pulseIntelligenceApprovedActionFailureCause(record unifiedresources.ActionAuditRecord, now time.Time) (string, string) { switch record.State { case unifiedresources.ActionStateExecuting: if record.UpdatedAt.IsZero() || now.Sub(record.UpdatedAt) >= pulseIntelligenceStuckExecutingThreshold { return "stuck_executing", "stuck_executing" } - return "", "" + return "in_flight", "" case unifiedresources.ActionStateFailed: truth := unifiedresources.CanonicalActionResultV2(record) if truth.Execution.Status == unifiedresources.ActionExecutionNotRun { @@ -157,6 +182,26 @@ func pulseIntelligenceApprovedActionFailureCause(record unifiedresources.ActionA } } +func pulseIntelligenceApprovedActionRefusalCategory(reason string) string { + switch reason { + case "plan_drift", "action_plan_expired", "action_replan_required": + return "plan_stale" + case "resource_remediation_locked", "policy_authorization_expired", + "policy_authorization_invalid", "policy_authorization_revoked", "action_emergency_stop": + return "policy" + case "action_dry_run_only", "action_execution_unavailable": + return "capability" + default: + return "other" + } +} + +func pulseIntelligenceVerifiedFindingResolution(record unifiedresources.ActionAuditRecord) bool { + return isPatrolActionOrigin(record.Origin) && + pulseIntelligenceApprovedActionSuccess(record) && + patrolOutcomeForActionAudit(record) == aicontracts.OutcomeFixVerified +} + func pulseIntelligenceSanitizedReasonCode(code, fallback string) string { code = strings.TrimSpace(code) if pulseIntelligenceReasonCodePattern.MatchString(code) { diff --git a/internal/api/telemetry_pulse_intelligence_test.go b/internal/api/telemetry_pulse_intelligence_test.go index c149a4ca03..7b7e28271d 100644 --- a/internal/api/telemetry_pulse_intelligence_test.go +++ b/internal/api/telemetry_pulse_intelligence_test.go @@ -328,6 +328,24 @@ func TestGetPulseIntelligenceActionTelemetry_AttributesApprovedActionFailureCaus if got.ApprovedActionStuckExecuting30d != 1 { t.Fatalf("ApprovedActionStuckExecuting30d = %d, want 1", got.ApprovedActionStuckExecuting30d) } + if got.ApprovedActionInFlight30d != 1 { + t.Fatalf("ApprovedActionInFlight30d = %d, want 1", got.ApprovedActionInFlight30d) + } + if got.ApprovedActionUnclassified30d != 0 { + t.Fatalf("ApprovedActionUnclassified30d = %d, want 0", got.ApprovedActionUnclassified30d) + } + if got.ApprovedActionRefusalsPlanStale30d != 1 || + got.ApprovedActionRefusalsPolicy30d != 0 || + got.ApprovedActionRefusalsCapability30d != 0 || + got.ApprovedActionRefusalsOther30d != 0 { + t.Fatalf( + "refusal categories = %d/%d/%d/%d, want 1/0/0/0", + got.ApprovedActionRefusalsPlanStale30d, + got.ApprovedActionRefusalsPolicy30d, + got.ApprovedActionRefusalsCapability30d, + got.ApprovedActionRefusalsOther30d, + ) + } // The completed-unverified record is the most recent failure; the legacy // row carries no canonical reason code, so the sanitized fallback applies. if got.ApprovedActionLastFailureReason30d != "verification_unconfirmed" { @@ -335,6 +353,50 @@ func TestGetPulseIntelligenceActionTelemetry_AttributesApprovedActionFailureCaus } } +func TestGetPulseIntelligenceActionTelemetry_CountsOnlyIndependentPatrolFindingResolutions(t *testing.T) { + now := time.Now().UTC() + since := now.Add(-telemetry.PulseIntelligenceTelemetryWindow) + router := &Router{ + resourceHandlers: NewResourceHandlers(&config.Config{DataPath: t.TempDir()}), + } + store, err := router.resourceHandlers.getStore("default") + if err != nil { + t.Fatalf("getStore: %v", err) + } + approved := []unifiedresources.ActionApprovalRecord{{ + Outcome: unifiedresources.OutcomeApproved, Method: unifiedresources.MethodUI, + Timestamp: now.Add(-time.Hour), Actor: "operator", + }} + makeCompleted := func(id string, evidenceClass unifiedresources.ActionEvidenceClass, origin *unifiedresources.ActionOrigin) unifiedresources.ActionAuditRecord { + record := pulseTelemetryActionRecord(id, now.Add(-time.Hour), unifiedresources.ActionStateCompleted, true, approved) + truth := apiActionResultTruth(t, unifiedresources.ActionExecutionSucceeded, unifiedresources.ActionVerificationConfirmed, evidenceClass) + record.Result = &unifiedresources.ExecutionResult{ActionResultV2: &truth} + record.Origin = origin + return record + } + for _, record := range []unifiedresources.ActionAuditRecord{ + makeCompleted("linked-independent", unifiedresources.ActionEvidenceIndependent, &unifiedresources.ActionOrigin{ + Surface: "patrol", FindingID: "finding-1", InvestigationID: "investigation-1", + }), + makeCompleted("linked-agent-attested", unifiedresources.ActionEvidenceAgentAttested, &unifiedresources.ActionOrigin{ + Surface: "patrol", FindingID: "finding-2", InvestigationID: "investigation-2", + }), + makeCompleted("independent-without-finding", unifiedresources.ActionEvidenceIndependent, nil), + } { + if err := store.RecordActionAudit(record); err != nil { + t.Fatalf("RecordActionAudit(%s): %v", record.ID, err) + } + } + + got := router.GetPulseIntelligenceActionTelemetry(since) + if got.ApprovedActionAttempts30d != 3 || got.ApprovedActionSuccesses30d != 3 { + t.Fatalf("attempts/successes = %d/%d, want 3/3", got.ApprovedActionAttempts30d, got.ApprovedActionSuccesses30d) + } + if got.VerifiedFindingResolutions30d != 1 { + t.Fatalf("VerifiedFindingResolutions30d = %d, want 1", got.VerifiedFindingResolutions30d) + } +} + func TestGetPulseIntelligenceActionTelemetry_LastFailureReasonUsesCanonicalReasonCode(t *testing.T) { now := time.Now().UTC() since := now.Add(-telemetry.PulseIntelligenceTelemetryWindow) @@ -371,6 +433,25 @@ func TestGetPulseIntelligenceActionTelemetry_LastFailureReasonUsesCanonicalReaso } } +func TestPulseIntelligenceApprovedActionRefusalCategory(t *testing.T) { + tests := map[string]string{ + "plan_drift": "plan_stale", + "action_plan_expired": "plan_stale", + "action_replan_required": "plan_stale", + "resource_remediation_locked": "policy", + "policy_authorization_expired": "policy", + "action_emergency_stop": "policy", + "action_dry_run_only": "capability", + "action_execution_unavailable": "capability", + "future_reason": "other", + } + for reason, want := range tests { + if got := pulseIntelligenceApprovedActionRefusalCategory(reason); got != want { + t.Errorf("reason %q category = %q, want %q", reason, got, want) + } + } +} + func pulseTelemetryActionRecord( id string, createdAt time.Time, diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index d131d6fc16..c88e936b16 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -133,8 +133,9 @@ const ( // TelemetrySchemaVersion identifies the exact outbound payload contract. // Schema v3 makes notification_failures_7d a terminal-delivery count; - // schema v2 counted every unsuccessful retry attempt. - TelemetrySchemaVersion = 3 + // schema v4 adds complete approved-action outcome accounting, fixed + // pre-dispatch refusal categories, and verified finding-resolution linkage. + TelemetrySchemaVersion = 4 ) type installIDRecord struct { @@ -298,6 +299,13 @@ type Ping struct { PulseIntelligenceApprovedActionFailuresExecution30d int `json:"pulse_intelligence_approved_action_failures_execution_30d"` PulseIntelligenceApprovedActionFailuresUnverified30d int `json:"pulse_intelligence_approved_action_failures_unverified_30d"` PulseIntelligenceApprovedActionStuckExecuting30d int `json:"pulse_intelligence_approved_action_stuck_executing_30d"` + PulseIntelligenceApprovedActionInFlight30d int `json:"pulse_intelligence_approved_action_in_flight_30d"` + PulseIntelligenceApprovedActionUnclassified30d int `json:"pulse_intelligence_approved_action_unclassified_30d"` + PulseIntelligenceApprovedActionRefusalsPlanStale30d int `json:"pulse_intelligence_approved_action_refusals_plan_stale_30d"` + PulseIntelligenceApprovedActionRefusalsPolicy30d int `json:"pulse_intelligence_approved_action_refusals_policy_30d"` + PulseIntelligenceApprovedActionRefusalsCapability30d int `json:"pulse_intelligence_approved_action_refusals_capability_30d"` + PulseIntelligenceApprovedActionRefusalsOther30d int `json:"pulse_intelligence_approved_action_refusals_other_30d"` + PulseIntelligenceVerifiedFindingResolutions30d int `json:"pulse_intelligence_verified_finding_resolutions_30d"` PulseIntelligenceApprovedActionLastFailureReason30d string `json:"pulse_intelligence_approved_action_last_failure_reason_30d,omitempty"` } @@ -412,6 +420,13 @@ type Snapshot struct { PulseIntelligenceApprovedActionFailuresExecution30d int PulseIntelligenceApprovedActionFailuresUnverified30d int PulseIntelligenceApprovedActionStuckExecuting30d int + PulseIntelligenceApprovedActionInFlight30d int + PulseIntelligenceApprovedActionUnclassified30d int + PulseIntelligenceApprovedActionRefusalsPlanStale30d int + PulseIntelligenceApprovedActionRefusalsPolicy30d int + PulseIntelligenceApprovedActionRefusalsCapability30d int + PulseIntelligenceApprovedActionRefusalsOther30d int + PulseIntelligenceVerifiedFindingResolutions30d int PulseIntelligenceApprovedActionLastFailureReason30d string } @@ -441,6 +456,22 @@ type PulseIntelligenceActionSnapshot struct { // ApprovedActionStuckExecuting30d counts approved attempts still in the // executing state well past any legitimate dispatch window. ApprovedActionStuckExecuting30d int + // ApprovedActionInFlight30d counts approved attempts still inside the + // legitimate dispatch window. + ApprovedActionInFlight30d int + // ApprovedActionUnclassified30d counts approved attempts whose + // authoritative audit is missing or cannot be mapped to a terminal class. + ApprovedActionUnclassified30d int + // The refusal category counters partition ApprovedActionFailuresPreDispatch30d + // into stable, content-free operator diagnostics. + ApprovedActionRefusalsPlanStale30d int + ApprovedActionRefusalsPolicy30d int + ApprovedActionRefusalsCapability30d int + ApprovedActionRefusalsOther30d int + // VerifiedFindingResolutions30d counts completed, approved Patrol-origin + // actions whose postcondition was independently confirmed. No finding or + // action identity leaves the runtime. + VerifiedFindingResolutions30d int // ApprovedActionLastFailureReason30d is the machine reason code of the // most recent approved-action failure, sanitized to a closed code shape. ApprovedActionLastFailureReason30d string @@ -950,6 +981,13 @@ func applySnapshot(base Ping, fn SnapshotFunc) Ping { ping.PulseIntelligenceApprovedActionFailuresExecution30d = s.PulseIntelligenceApprovedActionFailuresExecution30d ping.PulseIntelligenceApprovedActionFailuresUnverified30d = s.PulseIntelligenceApprovedActionFailuresUnverified30d ping.PulseIntelligenceApprovedActionStuckExecuting30d = s.PulseIntelligenceApprovedActionStuckExecuting30d + ping.PulseIntelligenceApprovedActionInFlight30d = s.PulseIntelligenceApprovedActionInFlight30d + ping.PulseIntelligenceApprovedActionUnclassified30d = s.PulseIntelligenceApprovedActionUnclassified30d + ping.PulseIntelligenceApprovedActionRefusalsPlanStale30d = s.PulseIntelligenceApprovedActionRefusalsPlanStale30d + ping.PulseIntelligenceApprovedActionRefusalsPolicy30d = s.PulseIntelligenceApprovedActionRefusalsPolicy30d + ping.PulseIntelligenceApprovedActionRefusalsCapability30d = s.PulseIntelligenceApprovedActionRefusalsCapability30d + ping.PulseIntelligenceApprovedActionRefusalsOther30d = s.PulseIntelligenceApprovedActionRefusalsOther30d + ping.PulseIntelligenceVerifiedFindingResolutions30d = s.PulseIntelligenceVerifiedFindingResolutions30d ping.PulseIntelligenceApprovedActionLastFailureReason30d = s.PulseIntelligenceApprovedActionLastFailureReason30d return ping } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 4b963f8e9a..9bf66162b7 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -415,6 +415,13 @@ func TestApplySnapshot(t *testing.T) { PulseIntelligenceApprovedActionDecisions30d: 1, PulseIntelligenceApprovedActionAttempts30d: 1, PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceApprovedActionInFlight30d: 2, + PulseIntelligenceApprovedActionUnclassified30d: 3, + PulseIntelligenceApprovedActionRefusalsPlanStale30d: 4, + PulseIntelligenceApprovedActionRefusalsPolicy30d: 5, + PulseIntelligenceApprovedActionRefusalsCapability30d: 6, + PulseIntelligenceApprovedActionRefusalsOther30d: 7, + PulseIntelligenceVerifiedFindingResolutions30d: 8, } } @@ -501,7 +508,14 @@ func TestApplySnapshot(t *testing.T) { ping.PulseIntelligenceRejectedActionDecisions30d != 1 || ping.PulseIntelligenceApprovedActionDecisions30d != 1 || ping.PulseIntelligenceApprovedActionAttempts30d != 1 || - ping.PulseIntelligenceApprovedActionSuccesses30d != 1 { + ping.PulseIntelligenceApprovedActionSuccesses30d != 1 || + ping.PulseIntelligenceApprovedActionInFlight30d != 2 || + ping.PulseIntelligenceApprovedActionUnclassified30d != 3 || + ping.PulseIntelligenceApprovedActionRefusalsPlanStale30d != 4 || + ping.PulseIntelligenceApprovedActionRefusalsPolicy30d != 5 || + ping.PulseIntelligenceApprovedActionRefusalsCapability30d != 6 || + ping.PulseIntelligenceApprovedActionRefusalsOther30d != 7 || + ping.PulseIntelligenceVerifiedFindingResolutions30d != 8 { t.Fatalf("Pulse Intelligence counters not applied: %#v", ping) } if !ping.PulseIntelligenceExternalAgentEnabled || !ping.PulseIntelligenceExternalAgentUsed30d || diff --git a/pkg/server/telemetry_pulse_intelligence.go b/pkg/server/telemetry_pulse_intelligence.go index dbe5c8d31b..cc6978e68a 100644 --- a/pkg/server/telemetry_pulse_intelligence.go +++ b/pkg/server/telemetry_pulse_intelligence.go @@ -41,6 +41,13 @@ func applyPulseIntelligenceTelemetrySnapshot( snap.PulseIntelligenceApprovedActionFailuresExecution30d = actionSnapshot.ApprovedActionFailuresExecution30d snap.PulseIntelligenceApprovedActionFailuresUnverified30d = actionSnapshot.ApprovedActionFailuresUnverified30d snap.PulseIntelligenceApprovedActionStuckExecuting30d = actionSnapshot.ApprovedActionStuckExecuting30d + snap.PulseIntelligenceApprovedActionInFlight30d = actionSnapshot.ApprovedActionInFlight30d + snap.PulseIntelligenceApprovedActionUnclassified30d = actionSnapshot.ApprovedActionUnclassified30d + snap.PulseIntelligenceApprovedActionRefusalsPlanStale30d = actionSnapshot.ApprovedActionRefusalsPlanStale30d + snap.PulseIntelligenceApprovedActionRefusalsPolicy30d = actionSnapshot.ApprovedActionRefusalsPolicy30d + snap.PulseIntelligenceApprovedActionRefusalsCapability30d = actionSnapshot.ApprovedActionRefusalsCapability30d + snap.PulseIntelligenceApprovedActionRefusalsOther30d = actionSnapshot.ApprovedActionRefusalsOther30d + snap.PulseIntelligenceVerifiedFindingResolutions30d = actionSnapshot.VerifiedFindingResolutions30d snap.PulseIntelligenceApprovedActionLastFailureReason30d = actionSnapshot.ApprovedActionLastFailureReason30d applyPulseIntelligenceAdoptionSnapshot(snap) @@ -90,14 +97,14 @@ func applyPulseIntelligenceAdoptionSnapshot(snap *telemetry.Snapshot) { snap.PulseIntelligenceApprovedActionAttempts30d > 0 approvedExecutionActive := snap.PulseIntelligenceApprovedActionAttempts30d > 0 approvedSuccessActive := snap.PulseIntelligenceApprovedActionSuccesses30d > 0 - resolutionOutcomeActive := snap.PulseIntelligencePatrolResolvedFindings30d > 0 + verifiedFindingResolutionActive := snap.PulseIntelligenceVerifiedFindingResolutions30d > 0 patrolControlProof := telemetry.ClassifyPulseIntelligencePatrolControlProof(telemetry.PulseIntelligencePatrolControlProofInput{ PatrolControlStarterCount: snap.PulseIntelligencePatrolControlOperationsLoopStarterRequests30d, PatrolIssueEvidenceCount: patrolIssueEvidenceCount, ContextualCollaborationCount: contextualCollaborationCount, ApprovedDecisionCount: snap.PulseIntelligenceApprovedActionDecisions30d, RejectedDecisionCount: snap.PulseIntelligenceRejectedActionDecisions30d, - VerifiedOutcomeCount: snap.PulseIntelligenceApprovedActionSuccesses30d, + VerifiedOutcomeCount: snap.PulseIntelligenceVerifiedFindingResolutions30d, }) snap.PulseIntelligenceCompleteOperationsLoop30d = patrolIssueEvidenceActive && @@ -108,9 +115,7 @@ func applyPulseIntelligenceAdoptionSnapshot(snap *telemetry.Snapshot) { contextualCollaborationActive && approvedExecutionActive snap.PulseIntelligenceResolvedOperationsLoop30d = - resolutionOutcomeActive && - contextualCollaborationActive && - approvedSuccessActive + verifiedFindingResolutionActive snap.PulseIntelligencePatrolControlCompletedOperationsLoop30d = patrolControlProof.Completed snap.PulseIntelligencePatrolControlResolvedOperationsLoop30d = patrolControlProof.Resolved snap.PulseIntelligencePatrolControlPaidCompletedOperationsLoop30d = @@ -137,9 +142,8 @@ func applyPulseIntelligenceAdoptionSnapshot(snap *telemetry.Snapshot) { assistantCollaborationActive && approvedSuccessActive snap.PulseIntelligenceAssistantResolvedOperationsLoop30d = - resolutionOutcomeActive && - assistantCollaborationActive && - approvedSuccessActive + verifiedFindingResolutionActive && + assistantCollaborationActive snap.PulseIntelligenceExternalAgentOperationsLoop30d = patrolIssueEvidenceActive && externalAgentCollaborationActive && @@ -153,9 +157,8 @@ func applyPulseIntelligenceAdoptionSnapshot(snap *telemetry.Snapshot) { externalAgentCollaborationActive && approvedSuccessActive snap.PulseIntelligenceExternalAgentResolvedOperationsLoop30d = - resolutionOutcomeActive && - externalAgentCollaborationActive && - approvedSuccessActive + verifiedFindingResolutionActive && + externalAgentCollaborationActive snap.PulseIntelligenceMCPAdapterOperationsLoop30d = patrolIssueEvidenceActive && mcpAdapterCollaborationActive && @@ -169,9 +172,8 @@ func applyPulseIntelligenceAdoptionSnapshot(snap *telemetry.Snapshot) { mcpAdapterCollaborationActive && approvedSuccessActive snap.PulseIntelligenceMCPAdapterResolvedOperationsLoop30d = - resolutionOutcomeActive && - mcpAdapterCollaborationActive && - approvedSuccessActive + verifiedFindingResolutionActive && + mcpAdapterCollaborationActive snap.PulseIntelligenceLoopActive30d = snap.PulseIntelligenceAssistantAICalls30d > 0 || diff --git a/pkg/server/telemetry_pulse_intelligence_test.go b/pkg/server/telemetry_pulse_intelligence_test.go index a01dac4952..ec96f0955c 100644 --- a/pkg/server/telemetry_pulse_intelligence_test.go +++ b/pkg/server/telemetry_pulse_intelligence_test.go @@ -110,12 +110,19 @@ func TestApplyPulseIntelligenceTelemetrySnapshot_AggregatesContentFreeLoopCounts snap := telemetry.Snapshot{PaidLicense: true} actions := telemetry.PulseIntelligenceActionSnapshot{ - ActionPlans30d: 4, - ApprovalRequests30d: 2, - RejectedActionDecisions30d: 1, - ApprovedActionDecisions30d: 1, - ApprovedActionAttempts30d: 1, - ApprovedActionSuccesses30d: 1, + ActionPlans30d: 4, + ApprovalRequests30d: 2, + RejectedActionDecisions30d: 1, + ApprovedActionDecisions30d: 1, + ApprovedActionAttempts30d: 1, + ApprovedActionSuccesses30d: 1, + ApprovedActionInFlight30d: 2, + ApprovedActionUnclassified30d: 3, + ApprovedActionRefusalsPlanStale30d: 4, + ApprovedActionRefusalsPolicy30d: 5, + ApprovedActionRefusalsCapability30d: 6, + ApprovedActionRefusalsOther30d: 7, + VerifiedFindingResolutions30d: 1, } applyPulseIntelligenceTelemetrySnapshot(&snap, persistence, cfg, actions, now) @@ -168,8 +175,15 @@ func TestApplyPulseIntelligenceTelemetrySnapshot_AggregatesContentFreeLoopCounts snap.PulseIntelligenceRejectedActionDecisions30d != 1 || snap.PulseIntelligenceApprovedActionDecisions30d != 1 || snap.PulseIntelligenceApprovedActionAttempts30d != 1 || - snap.PulseIntelligenceApprovedActionSuccesses30d != 1 { - t.Fatalf("action telemetry = %#v, want action plans 4, approval requests 2, rejected decisions 1, approved decisions 1, approved attempts 1, approved successes 1", snap) + snap.PulseIntelligenceApprovedActionSuccesses30d != 1 || + snap.PulseIntelligenceApprovedActionInFlight30d != 2 || + snap.PulseIntelligenceApprovedActionUnclassified30d != 3 || + snap.PulseIntelligenceApprovedActionRefusalsPlanStale30d != 4 || + snap.PulseIntelligenceApprovedActionRefusalsPolicy30d != 5 || + snap.PulseIntelligenceApprovedActionRefusalsCapability30d != 6 || + snap.PulseIntelligenceApprovedActionRefusalsOther30d != 7 || + snap.PulseIntelligenceVerifiedFindingResolutions30d != 1 { + t.Fatalf("action telemetry counters were not copied: %#v", snap) } if snap.PulseIntelligenceOperationsLoopStarterRequests30d != 6 || snap.PulseIntelligenceAssistantOperationsLoopStarterRequests30d != 1 || @@ -192,7 +206,7 @@ func TestApplyPulseIntelligenceTelemetrySnapshot_AggregatesContentFreeLoopCounts t.Fatalf("Pulse Intelligence approved-execution loop should be active when the complete loop includes an approved action attempt: %#v", snap) } if !snap.PulseIntelligenceResolvedOperationsLoop30d { - t.Fatalf("Pulse Intelligence resolved operations loop should be active when contextual collaboration, approved success, and Patrol resolution all exist: %#v", snap) + t.Fatalf("Pulse Intelligence resolved operations loop should be active when a Patrol finding has a linked, independently verified action resolution: %#v", snap) } if !snap.PulseIntelligencePatrolControlResolvedOperationsLoop30d { t.Fatalf("Patrol-control resolved operations loop should be active when Patrol-control starter, issue evidence, contextual collaboration, approved decision, and verified outcome all exist: %#v", snap) @@ -289,6 +303,7 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_PatrolControlResolvedRequiresSta PulseIntelligenceAssistantContextAICalls30d: 1, PulseIntelligenceApprovedActionDecisions30d: 1, PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, PulseIntelligenceExternalAgentEnabled: true, PulseIntelligenceExternalAgentOperationsLoopReady: true, } @@ -327,9 +342,9 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_PatrolControlResolvedRequiresSta }, }, { - name: "approved decision without verified outcome", + name: "approved decision without linked verified outcome", mut: func(snap *telemetry.Snapshot) { - snap.PulseIntelligenceApprovedActionSuccesses30d = 0 + snap.PulseIntelligenceVerifiedFindingResolutions30d = 0 }, }, { @@ -396,6 +411,7 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_PatrolControlCompletedIncludesRe snap.PulseIntelligenceRejectedActionDecisions30d = 0 snap.PulseIntelligenceApprovedActionDecisions30d = 1 snap.PulseIntelligenceApprovedActionSuccesses30d = 1 + snap.PulseIntelligenceVerifiedFindingResolutions30d = 1 }, want: true, }, @@ -459,6 +475,7 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_PaidPatrolControlCohortsRequireP PulseIntelligenceAssistantContextAICalls30d: 1, PulseIntelligenceApprovedActionDecisions30d: 1, PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, PulseIntelligenceExternalAgentEnabled: true, PulseIntelligenceExternalAgentOperationsLoopReady: true, } @@ -711,13 +728,14 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_SourceSpecificLoopEvidence(t *te { name: "assistant collaboration stays first party", snap: telemetry.Snapshot{ - PulseIntelligencePatrolNewFindings30d: 1, - PulseIntelligencePatrolResolvedFindings30d: 1, - PulseIntelligenceAssistantContextAICalls30d: 1, - PulseIntelligenceApprovalRequests30d: 1, - PulseIntelligenceApprovedActionDecisions30d: 1, - PulseIntelligenceApprovedActionAttempts30d: 1, - PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligencePatrolNewFindings30d: 1, + PulseIntelligencePatrolResolvedFindings30d: 1, + PulseIntelligenceAssistantContextAICalls30d: 1, + PulseIntelligenceApprovalRequests30d: 1, + PulseIntelligenceApprovedActionDecisions30d: 1, + PulseIntelligenceApprovedActionAttempts30d: 1, + PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, }, wantAssistantLoop: true, wantAssistantResolved: true, @@ -732,6 +750,7 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_SourceSpecificLoopEvidence(t *te PulseIntelligenceApprovedActionDecisions30d: 1, PulseIntelligenceApprovedActionAttempts30d: 1, PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, }, wantExternalLoop: true, wantExternalResolved: true, @@ -739,13 +758,14 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_SourceSpecificLoopEvidence(t *te { name: "pulse mcp is the mcp subset and external-agent umbrella", snap: telemetry.Snapshot{ - PulseIntelligencePatrolNewFindings30d: 1, - PulseIntelligencePatrolResolvedFindings30d: 1, - PulseIntelligenceMCPAdapterUsed30d: true, - PulseIntelligenceApprovalRequests30d: 1, - PulseIntelligenceApprovedActionDecisions30d: 1, - PulseIntelligenceApprovedActionAttempts30d: 1, - PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligencePatrolNewFindings30d: 1, + PulseIntelligencePatrolResolvedFindings30d: 1, + PulseIntelligenceMCPAdapterUsed30d: true, + PulseIntelligenceApprovalRequests30d: 1, + PulseIntelligenceApprovedActionDecisions30d: 1, + PulseIntelligenceApprovedActionAttempts30d: 1, + PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, }, wantExternalLoop: true, wantExternalResolved: true, @@ -941,7 +961,21 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_CompleteLoopRequiresIssueEvidenc wantResolved: false, }, { - name: "resolved operations loop requires patrol resolution collaboration and approved success", + name: "resolved operations loop requires linked independently verified finding resolution", + snap: telemetry.Snapshot{ + PulseIntelligencePatrolResolvedFindings30d: 1, + PulseIntelligenceAssistantContextAICalls30d: 1, + PulseIntelligenceApprovedActionAttempts30d: 1, + PulseIntelligenceApprovedActionSuccesses30d: 1, + PulseIntelligenceVerifiedFindingResolutions30d: 1, + }, + wantActive: true, + wantComplete: true, + wantApproved: true, + wantResolved: true, + }, + { + name: "unlinked finding resolution and action success do not resolve", snap: telemetry.Snapshot{ PulseIntelligencePatrolResolvedFindings30d: 1, PulseIntelligenceAssistantContextAICalls30d: 1, @@ -951,7 +985,7 @@ func TestApplyPulseIntelligenceAdoptionSnapshot_CompleteLoopRequiresIssueEvidenc wantActive: true, wantComplete: true, wantApproved: true, - wantResolved: true, + wantResolved: false, }, { name: "patrol resolution without governed action decision does not complete or resolve the operations loop", diff --git a/scripts/telemetry_adoption_report.py b/scripts/telemetry_adoption_report.py index 9514c35671..009fcbffa0 100644 --- a/scripts/telemetry_adoption_report.py +++ b/scripts/telemetry_adoption_report.py @@ -314,6 +314,34 @@ PULSE_INTELLIGENCE_COUNT_FIELDS = ( "pulse_intelligence_approved_action_stuck_executing_30d", "Approved action attempts stuck executing 30d", ), + ( + "pulse_intelligence_approved_action_in_flight_30d", + "Approved action attempts still in flight 30d", + ), + ( + "pulse_intelligence_approved_action_unclassified_30d", + "Approved action attempts unclassified 30d", + ), + ( + "pulse_intelligence_approved_action_refusals_plan_stale_30d", + "Pre-dispatch refusals: stale or expired plan 30d", + ), + ( + "pulse_intelligence_approved_action_refusals_policy_30d", + "Pre-dispatch refusals: policy or operator safety control 30d", + ), + ( + "pulse_intelligence_approved_action_refusals_capability_30d", + "Pre-dispatch refusals: capability unavailable 30d", + ), + ( + "pulse_intelligence_approved_action_refusals_other_30d", + "Pre-dispatch refusals: other 30d", + ), + ( + "pulse_intelligence_verified_finding_resolutions_30d", + "Independently verified action-to-finding resolutions 30d", + ), ) PULSE_INTELLIGENCE_OUTCOME_COHORTS = ( ( @@ -1232,6 +1260,20 @@ def latest_rc_version(releases: Iterable[dict[str, Any]]) -> str | None: return str(latest["version"]) +def latest_target_release_version(releases: Iterable[dict[str, Any]]) -> str | None: + release_list = list(releases) + stable_releases = [ + release + for release in release_list + if not release.get("is_prerelease") + and version_channel(str(release.get("version") or "")) == "stable" + ] + if stable_releases: + latest = max(stable_releases, key=lambda release: str(release.get("published_at") or "")) + return str(latest["version"]) + return latest_rc_version(release_list) + + def fetch_rows_local(db_path: str, since_days: int) -> dict[str, Any]: conn = sqlite3.connect(db_path) conn.row_factory = sqlite3.Row @@ -1506,13 +1548,49 @@ def summarize_pulse_intelligence_value_loop( signal["free_installs"] += 1 signal["free_total"] += value + count_signal_list = list(count_signals.values()) + totals = {signal["field"]: int(signal["total"]) for signal in count_signal_list} + attempts = totals["pulse_intelligence_approved_action_attempts_30d"] + accounted = sum( + totals[field] + for field in ( + "pulse_intelligence_approved_action_successes_30d", + "pulse_intelligence_approved_action_failures_pre_dispatch_30d", + "pulse_intelligence_approved_action_failures_execution_30d", + "pulse_intelligence_approved_action_failures_unverified_30d", + "pulse_intelligence_approved_action_stuck_executing_30d", + "pulse_intelligence_approved_action_in_flight_30d", + "pulse_intelligence_approved_action_unclassified_30d", + ) + ) + pre_dispatch = totals["pulse_intelligence_approved_action_failures_pre_dispatch_30d"] + refusal_accounted = sum( + totals[field] + for field in ( + "pulse_intelligence_approved_action_refusals_plan_stale_30d", + "pulse_intelligence_approved_action_refusals_policy_30d", + "pulse_intelligence_approved_action_refusals_capability_30d", + "pulse_intelligence_approved_action_refusals_other_30d", + ) + ) + return { "window": "7d", "active_installs": active_installs, "paid_installs": paid_installs, "free_installs": free_installs, "boolean_signals": list(bool_signals.values()), - "count_signals": list(count_signals.values()), + "count_signals": count_signal_list, + "approved_action_outcome_accounting": { + "attempts": attempts, + "accounted": accounted, + "gap": max(0, attempts - accounted), + "overflow": max(0, accounted - attempts), + "pre_dispatch_refusals": pre_dispatch, + "refusal_categories_accounted": refusal_accounted, + "refusal_category_gap": max(0, pre_dispatch - refusal_accounted), + "refusal_category_overflow": max(0, refusal_accounted - pre_dispatch), + }, } @@ -2351,6 +2429,24 @@ def format_text(summary: dict[str, Any], repo: str, since_days: int) -> str: lines.extend(f" - {entry['label']}: {format_paid_count_split(entry)}" for entry in count_signals) else: lines.append(" - none") + accounting = pulse_loop.get("approved_action_outcome_accounting", {}) + lines.extend( + [ + "- approved-action outcome accounting:", + " - " + f"attempts {accounting.get('attempts', 0)}, " + f"accounted {accounting.get('accounted', 0)}, " + f"gap {accounting.get('gap', 0)}, " + f"overflow {accounting.get('overflow', 0)}", + " - " + f"pre-dispatch refusals {accounting.get('pre_dispatch_refusals', 0)}, " + f"categorized {accounting.get('refusal_categories_accounted', 0)}, " + f"gap {accounting.get('refusal_category_gap', 0)}, " + f"overflow {accounting.get('refusal_category_overflow', 0)}", + " - interpretation: non-zero gaps are expected from pre-schema-v4 rows; " + "schema v4 installs must reconcile exactly", + ] + ) pulse_cohorts = summary.get("pulse_intelligence_outcome_cohorts") if pulse_cohorts: @@ -2453,7 +2549,10 @@ def parse_args(argv: list[str]) -> argparse.Namespace: ) parser.add_argument( "--target-version", - help="release version to highlight for per-signal coverage; defaults to the latest published RC", + help=( + "release version to highlight for per-signal coverage; defaults to " + "the latest published stable release, falling back to the latest RC" + ), ) parser.add_argument( "--include-mock-fleet", @@ -2479,7 +2578,7 @@ def main(argv: list[str] | None = None) -> int: published_releases = fetch_published_releases(args.github_repo) published_versions = {release["version"] for release in published_releases} - target_version = args.target_version or latest_rc_version(published_releases) + target_version = args.target_version or latest_target_release_version(published_releases) source = ( fetch_rows_remote(args.ssh_host, args.db_path, args.since_days) if args.ssh_host diff --git a/scripts/tests/test_telemetry_adoption_report.py b/scripts/tests/test_telemetry_adoption_report.py index f1cebea3e7..3d9ab430cd 100644 --- a/scripts/tests/test_telemetry_adoption_report.py +++ b/scripts/tests/test_telemetry_adoption_report.py @@ -132,6 +132,76 @@ class TelemetryAdoptionReportTest(unittest.TestCase): "6.0.0-rc.2", ) + def test_latest_target_release_version_prefers_latest_stable_release(self) -> None: + self.assertEqual( + report.latest_target_release_version( + [ + { + "version": "6.1.2", + "is_prerelease": False, + "published_at": "2026-07-26T22:11:46Z", + }, + { + "version": "6.2.0-rc.1", + "is_prerelease": True, + "published_at": "2026-07-27T08:00:00Z", + }, + { + "version": "helm-chart-6.1.2", + "is_prerelease": False, + "published_at": "2026-07-27T09:00:00Z", + }, + ] + ), + "6.1.2", + ) + + def test_latest_target_release_version_falls_back_to_latest_rc(self) -> None: + self.assertEqual( + report.latest_target_release_version( + [ + { + "version": "6.2.0-rc.1", + "is_prerelease": True, + "published_at": "2026-07-27T08:00:00Z", + } + ] + ), + "6.2.0-rc.1", + ) + + def test_value_loop_reconciles_schema_v4_action_outcomes_and_refusals(self) -> None: + now = datetime.now(timezone.utc).replace(microsecond=0) + summary = report.summarize_pulse_intelligence_value_loop( + { + "install-a": { + "install_id": "install-a", + "received_at": now.strftime("%Y-%m-%d %H:%M:%S"), + "schema_version": 4, + "pulse_intelligence_approved_action_attempts_30d": 7, + "pulse_intelligence_approved_action_successes_30d": 1, + "pulse_intelligence_approved_action_failures_pre_dispatch_30d": 2, + "pulse_intelligence_approved_action_failures_execution_30d": 1, + "pulse_intelligence_approved_action_failures_unverified_30d": 1, + "pulse_intelligence_approved_action_stuck_executing_30d": 1, + "pulse_intelligence_approved_action_in_flight_30d": 1, + "pulse_intelligence_approved_action_unclassified_30d": 0, + "pulse_intelligence_approved_action_refusals_plan_stale_30d": 1, + "pulse_intelligence_approved_action_refusals_policy_30d": 1, + "pulse_intelligence_verified_finding_resolutions_30d": 1, + } + }, + now=now, + ) + accounting = summary["approved_action_outcome_accounting"] + self.assertEqual(accounting["attempts"], 7) + self.assertEqual(accounting["accounted"], 7) + self.assertEqual(accounting["gap"], 0) + self.assertEqual(accounting["overflow"], 0) + self.assertEqual(accounting["pre_dispatch_refusals"], 2) + self.assertEqual(accounting["refusal_categories_accounted"], 2) + self.assertEqual(accounting["refusal_category_gap"], 0) + def test_security_docs_use_current_agent_install_surface(self) -> None: security_docs = ( REPO_ROOT / "SECURITY.md",