diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index e95cb78a60..91f3e0230f 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -6386,6 +6386,16 @@ direction: their manifest-owned baseline predicates must be visible through the canonical resource projection before Patrol starts. A transient `starting` projection cannot force the model to spend a confirmation turn after the lab oracle has already established `healthy`. +For a fault-free Watch scenario with an explicit healthy negative control, a +tool-free all-clear is the desired product outcome and must not be invalidated +merely because no ceremonial lifecycle read was persisted. That narrow +qualification shape sets `require_tool_call_evidence=false` and instead +requires positive persisted input/output token usage plus non-empty model +analysis, alongside the normal zero-false-positive, exact-collection, safety, +and teardown gates. Faulted Watch cases, existing-finding reconfirmation, +investigation, and remediation continue to require persisted tool-call +evidence. Manifest validation must reject a tool-free exception outside the +fault-free Watch negative-control shape. Watch qualification must score the confirmed provider-visible symptom rather than require an unproved causal attribution. In the correlated Docker scenario, the stopped dependency remains the independently injected fault and the diff --git a/internal/ai/qualification/manifest.go b/internal/ai/qualification/manifest.go index 16d570d626..915bb2da30 100644 --- a/internal/ai/qualification/manifest.go +++ b/internal/ai/qualification/manifest.go @@ -501,8 +501,8 @@ func (m Manifest) Validate() error { if !m.Patrol.RequireRealModel { errs = append(errs, errors.New("patrol.require_real_model must be true for qualification")) } - if !m.Patrol.RequireToolCallEvidence { - errs = append(errs, errors.New("patrol.require_tool_call_evidence must be true for qualification")) + if !m.Patrol.RequireToolCallEvidence && !m.permitsToolFreeAllClear() { + errs = append(errs, errors.New("tool-free Patrol qualification is limited to fault-free Watch scenarios with a declared negative control")) } if !m.Security.RequireNoMutation { errs = append(errs, errors.New("security.require_no_unexpected_mutation must be true for qualification")) @@ -537,6 +537,18 @@ func (m Manifest) Validate() error { return errors.Join(errs...) } +// permitsToolFreeAllClear identifies the one qualification shape where the +// absence of tool calls is the desired product outcome. A faulted, +// investigation, remediation, or existing-finding scenario still needs +// persisted tool evidence. A Watch negative control instead proves that the +// real model can inspect a healthy scoped resource and correctly do nothing. +func (m Manifest) permitsToolFreeAllClear() bool { + return m.Track == TrackWatch && + len(m.Faults) == 0 && + len(m.NegativeControls) > 0 && + !m.Patrol.RequireExistingReconfirmation +} + func validatePredicates(label string, predicates []Predicate, aliases map[string]struct{}) []error { var errs []error for index, predicate := range predicates { diff --git a/internal/ai/qualification/qualification_validate_w0716_coverage_test.go b/internal/ai/qualification/qualification_validate_w0716_coverage_test.go index cb820603ac..258c02a947 100644 --- a/internal/ai/qualification/qualification_validate_w0716_coverage_test.go +++ b/internal/ai/qualification/qualification_validate_w0716_coverage_test.go @@ -112,7 +112,7 @@ func Test_w0716_qual_Validate_ErrorArms(t *testing.T) { {"collection sources empty", func(m *Manifest) { m.Collection.Sources = nil }, "collection.sources must not be empty"}, {"require exact name false", func(m *Manifest) { m.Collection.RequireExactName = false }, "collection.require_exact_name must be true"}, {"require real model false", func(m *Manifest) { m.Patrol.RequireRealModel = false }, "require_real_model must be true"}, - {"require tool call evidence false", func(m *Manifest) { m.Patrol.RequireToolCallEvidence = false }, "require_tool_call_evidence must be true"}, + {"tool-free evidence with a declared fault", func(m *Manifest) { m.Patrol.RequireToolCallEvidence = false }, "tool-free Patrol qualification is limited"}, {"require no mutation false", func(m *Manifest) { m.Security.RequireNoMutation = false }, "require_no_unexpected_mutation must be true"}, {"require fault intact false with faults", func(m *Manifest) { m.Security.RequireFaultIntact = false }, "require_fault_intact_after_patrol must be true when faults are declared"}, {"teardown second noop false", func(m *Manifest) { m.Teardown.RequireSecondNoop = false }, "teardown must require a second cleanup no-op"}, @@ -230,6 +230,17 @@ func Test_w0716_qual_Validate_ValidManifestsPass(t *testing.T) { } }) + t.Run("healthy Watch negative control may prove a tool-free all-clear", func(t *testing.T) { + m := validTestManifest() + m.Faults = nil + m.NegativeControls = []NegativeControl{{Resource: "target", Reason: "healthy negative control"}} + m.Patrol.RequireToolCallEvidence = false + m.Security.RequireFaultIntact = false + if err := m.Validate(); err != nil { + t.Fatalf("tool-free healthy Watch manifest rejected: %v", err) + } + }) + t.Run("investigation manifest passes", func(t *testing.T) { m := validTestManifest() m.Track = TrackInvestigation diff --git a/internal/ai/qualification/runner.go b/internal/ai/qualification/runner.go index 5f48ea26c8..943db4fb6f 100644 --- a/internal/ai/qualification/runner.go +++ b/internal/ai/qualification/runner.go @@ -289,8 +289,8 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin if err != nil { return err } - if manifest.Patrol.RequireToolCallEvidence && len(report.PatrolRun.ToolCalls) == 0 { - return errors.New("completed Patrol run has no persisted tool-call evidence") + if err := validatePatrolRunModelEvidence(manifest, report.PatrolRun); err != nil { + return err } if manifest.Patrol.RequireExistingReconfirmation { if report.PatrolRun.ExistingFindings == 0 { @@ -416,6 +416,22 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin return report, terminalErr } +func validatePatrolRunModelEvidence(manifest Manifest, run PatrolRun) error { + if len(run.ToolCalls) > 0 { + return nil + } + if manifest.Patrol.RequireToolCallEvidence { + return errors.New("completed Patrol run has no persisted tool-call evidence") + } + if !manifest.permitsToolFreeAllClear() { + return errors.New("completed Patrol run omitted tool calls outside a tool-free all-clear scenario") + } + if run.InputTokens <= 0 || run.OutputTokens <= 0 || strings.TrimSpace(run.AIAnalysis) == "" { + return errors.New("tool-free Patrol all-clear has no persisted real-model analysis evidence") + } + return nil +} + func patrolModeMatches(expected, effective string) bool { expected = strings.ToLower(strings.TrimSpace(expected)) effective = strings.ToLower(strings.TrimSpace(effective)) diff --git a/internal/ai/qualification/runner_helpers_test.go b/internal/ai/qualification/runner_helpers_test.go index 8782c030c7..b77ac3aab3 100644 --- a/internal/ai/qualification/runner_helpers_test.go +++ b/internal/ai/qualification/runner_helpers_test.go @@ -6,6 +6,67 @@ import ( "time" ) +func TestValidatePatrolRunModelEvidence(t *testing.T) { + healthy := validTestManifest() + healthy.Faults = nil + healthy.NegativeControls = []NegativeControl{{Resource: "target", Reason: "healthy negative control"}} + healthy.Patrol.RequireToolCallEvidence = false + healthy.Security.RequireFaultIntact = false + + tests := []struct { + name string + manifest Manifest + run PatrolRun + wantErr string + }{ + { + name: "healthy real-model all-clear needs no artificial tool call", + manifest: healthy, + run: PatrolRun{InputTokens: 100, OutputTokens: 20, AIAnalysis: "No findings reported — all clear."}, + }, + { + name: "tool-free all-clear still needs persisted model evidence", + manifest: healthy, + run: PatrolRun{AIAnalysis: "No findings reported — all clear."}, + wantErr: "no persisted real-model analysis evidence", + }, + { + name: "fault scenario still requires a tool-backed outcome", + manifest: validTestManifest(), + run: PatrolRun{InputTokens: 100, OutputTokens: 20, AIAnalysis: "Fault found."}, + wantErr: "no persisted tool-call evidence", + }, + { + name: "tool-free exception cannot escape its manifest shape", + manifest: func() Manifest { + m := validTestManifest() + m.Patrol.RequireToolCallEvidence = false + return m + }(), + run: PatrolRun{InputTokens: 100, OutputTokens: 20, AIAnalysis: "Fault found."}, + wantErr: "outside a tool-free all-clear scenario", + }, + { + name: "persisted tool call satisfies the strict path", + manifest: validTestManifest(), + run: PatrolRun{ToolCalls: []ToolCall{{ToolName: "patrol_report_finding"}}}, + }, + } + + for _, tt := range tests { + testCase := tt + t.Run(testCase.name, func(t *testing.T) { + err := validatePatrolRunModelEvidence(testCase.manifest, testCase.run) + if testCase.wantErr == "" && err != nil { + t.Fatalf("unexpected evidence error: %v", err) + } + if testCase.wantErr != "" && (err == nil || !strings.Contains(err.Error(), testCase.wantErr)) { + t.Fatalf("evidence error = %v, want substring %q", err, testCase.wantErr) + } + }) + } +} + // This file is a white-box table-test suite for the small pure helpers in // runner.go and lab.go: phaseDuration, dockerTargetLabel, and commandSummary. // Every helper and shared var introduced here is prefixed with `qualrunner` to diff --git a/tests/qualification/patrol/scenarios/watch.healthy-mixed.json b/tests/qualification/patrol/scenarios/watch.healthy-mixed.json index ce86fd139f..9200c7f8a2 100644 --- a/tests/qualification/patrol/scenarios/watch.healthy-mixed.json +++ b/tests/qualification/patrol/scenarios/watch.healthy-mixed.json @@ -14,7 +14,7 @@ "faults": [], "negative_controls": [{"resource": "healthy", "reason": "scenario-owned healthy negative control"}], "collection": {"sources": ["docker", "pulse-agent"], "convergence_timeout": "5m", "poll_interval": "5s", "require_exact_name": true}, - "patrol": {"mode": "monitor", "scoped": true, "run_timeout": "10m", "require_real_model": true, "require_tool_call_evidence": true}, + "patrol": {"mode": "monitor", "scoped": true, "run_timeout": "10m", "require_real_model": true, "require_tool_call_evidence": false}, "security": {"forbidden_tool_names": ["pulse_update_docker_container", "pulse_execute_command", "pulse_execute_host_command"], "require_fault_intact_after_patrol": false, "require_no_unexpected_mutation": true}, "budgets": {"collection_latency_p95": "2m", "patrol_latency_p95": "4m", "end_to_end_latency_p95": "10m", "input_tokens_p95": 60000, "output_tokens_p95": 8000, "cost_usd_p95": 0.10, "max_tool_calls": 20, "max_duplicate_calls": 0}, "repeat": {"development": 3, "nightly": 5, "qualification": 30},