diff --git a/cmd/patrol-qualify/main.go b/cmd/patrol-qualify/main.go index 947f5d2b0f..9b5d0816e4 100644 --- a/cmd/patrol-qualify/main.go +++ b/cmd/patrol-qualify/main.go @@ -20,7 +20,7 @@ import ( ) func main() { - mode := flag.String("mode", "validate", "validate, list, live, replay, verify-replay, or compare") + mode := flag.String("mode", "validate", "validate, list, live, live-suite, replay, verify-replay, compare, or export-contribution") catalogDir := flag.String("catalog", "tests/qualification/patrol/scenarios", "scenario manifest directory") scenarioID := flag.String("scenario", "", "scenario id for live mode") baseURL := flag.String("url", "http://127.0.0.1:7655", "Pulse API base URL") @@ -40,8 +40,10 @@ func main() { replayPath := flag.String("replay-report", "", "captured report.json for deterministic scorer replay") replayBundlePath := flag.String("replay-bundle", "", "captured replay.json for ordered tool-transcript verification") reportsRoot := flag.String("reports", "tmp/patrol-qualification", "report tree for model comparison") - qualificationTrack := flag.String("qualification-track", "", "optional launch gate for compare mode: watch, investigation, or remediation") + qualificationTrack := flag.String("qualification-track", "", "track for live-suite, export-contribution, or optional compare gates: watch, investigation, or remediation") publicationDir := flag.String("publication-dir", "", "optional directory for comparison.json, comparison.md, and checksums") + contributionDir := flag.String("contribution-dir", "", "local output directory for an allowlist-only community evidence candidate") + communityChallenge := flag.String("community-challenge", "", "optional server-issued nonce bound into each live report before the run") flag.Parse() catalog, err := qualification.LoadCatalog(*catalogDir) @@ -55,21 +57,16 @@ func main() { for _, manifest := range catalog.Manifests { fmt.Printf("%-44s %-14s %s\n", manifest.ID, manifest.Track, manifest.Title) } - case "live": + case "live", "live-suite": if !*authorizeLive { - fatal(fmt.Errorf("live mode requires --authorize-live-faults")) + fatal(fmt.Errorf("%s mode requires --authorize-live-faults", strings.ToLower(strings.TrimSpace(*mode)))) } - manifest, ok := catalog.ByID[*scenarioID] - if !ok { - fatal(fmt.Errorf("unknown scenario %q", *scenarioID)) - } - repeatCount, err := liveRepeatCount(manifest, *repeats, *repeatProfile) + manifests, err := selectLiveManifests(catalog, *mode, *scenarioID, qualification.Track(strings.ToLower(strings.TrimSpace(*qualificationTrack)))) if err != nil { fatal(err) } - if manifest.Track == qualification.TrackRemediation && manifest.Remediation != nil && - manifest.Remediation.Decision != "observe" && !*authorizeRemediation { - fatal(fmt.Errorf("scenario %q requires the separate --authorize-remediation gate", manifest.ID)) + if err := qualification.ValidateContributionChallenge(*communityChallenge); err != nil { + fatal(err) } secret := *password if value := strings.TrimSpace(os.Getenv(*passwordEnv)); value != "" { @@ -88,25 +85,40 @@ func main() { defer stop() gitSHA, dirty := qualification.GitEnvironment(ctx, nil, ".") failed := false - for i := 0; i < repeatCount; i++ { - runner, err := qualification.NewRunner(qualification.RunnerConfig{ - Manifest: manifest, Lab: lab, Client: client, ArtifactRoot: *artifactRoot, - ModelOverride: *model, GitSHA: gitSHA, GitDirty: dirty, - AuthorizeRemediation: *authorizeRemediation, - ExpectedPulseVersion: *expectedPulseVersion, - }) + liveLoop: + for _, manifest := range manifests { + repeatCount, err := liveRepeatCount(manifest, *repeats, *repeatProfile) if err != nil { fatal(err) } - report, runErr := runner.Run(ctx) - verdict := "PASS" - if !report.Passed || runErr != nil { - verdict = "FAIL" - failed = true + if manifest.Track == qualification.TrackRemediation && manifest.Remediation != nil && + manifest.Remediation.Decision != "observe" && !*authorizeRemediation { + fatal(fmt.Errorf("scenario %q requires the separate --authorize-remediation gate", manifest.ID)) } - fmt.Printf("[%s] %s model=%s recall=%.1f%% fp=%d artifacts=%s\n", verdict, report.RunID, report.Environment.Model, report.Score.Recall*100, report.Score.FalsePositives, filepath.Join(*artifactRoot, report.RunID)) - if runErr != nil { - fmt.Fprintf(os.Stderr, " error: %v\n", runErr) + for i := 0; i < repeatCount; i++ { + runner, err := qualification.NewRunner(qualification.RunnerConfig{ + Manifest: manifest, Lab: lab, Client: client, ArtifactRoot: *artifactRoot, + ModelOverride: *model, GitSHA: gitSHA, GitDirty: dirty, + AuthorizeRemediation: *authorizeRemediation, + ExpectedPulseVersion: *expectedPulseVersion, + ChallengeNonce: *communityChallenge, + }) + if err != nil { + fatal(err) + } + report, runErr := runner.Run(ctx) + verdict := "PASS" + if !report.Passed || runErr != nil { + verdict = "FAIL" + failed = true + } + fmt.Printf("[%s] %s scenario=%s model=%s recall=%.1f%% fp=%d artifacts=%s\n", verdict, report.RunID, manifest.ID, report.Environment.Model, report.Score.Recall*100, report.Score.FalsePositives, filepath.Join(*artifactRoot, report.RunID)) + if runErr != nil { + fmt.Fprintf(os.Stderr, " error: %v\n", runErr) + } + if ctx.Err() != nil { + break liveLoop + } } } if failed { @@ -174,11 +186,56 @@ func main() { os.Exit(1) } } + case "export-contribution": + output := strings.TrimSpace(*contributionDir) + if output == "" { + fatal(fmt.Errorf("export-contribution mode requires --contribution-dir")) + } + track := qualification.Track(strings.ToLower(strings.TrimSpace(*qualificationTrack))) + paths, err := findReports(*reportsRoot) + if err != nil { + fatal(err) + } + bundle, err := qualification.BuildContributionBundle(paths, catalog, track) + if err != nil { + fatal(err) + } + if err := qualification.WriteContributionBundle(output, bundle); err != nil { + fatal(err) + } + fmt.Printf("wrote local community evidence candidate with %d run(s) to %s; no network upload was performed\n", len(bundle.Runs), output) default: fatal(fmt.Errorf("unknown mode %q", *mode)) } } +func selectLiveManifests(catalog qualification.Catalog, mode, scenarioID string, track qualification.Track) ([]qualification.Manifest, error) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "live": + manifest, ok := catalog.ByID[strings.TrimSpace(scenarioID)] + if !ok { + return nil, fmt.Errorf("unknown scenario %q", scenarioID) + } + return []qualification.Manifest{manifest}, nil + case "live-suite": + if track != qualification.TrackWatch && track != qualification.TrackInvestigation && track != qualification.TrackRemediation { + return nil, fmt.Errorf("live-suite mode requires --qualification-track watch, investigation, or remediation") + } + var manifests []qualification.Manifest + for _, manifest := range catalog.Manifests { + if manifest.Track == track { + manifests = append(manifests, manifest) + } + } + if len(manifests) == 0 { + return nil, fmt.Errorf("no %s scenarios found in the catalogue", track) + } + return manifests, nil + default: + return nil, fmt.Errorf("unsupported live mode %q", mode) + } +} + func liveRepeatCount(manifest qualification.Manifest, explicit int, profile string) (int, error) { count := explicit switch strings.ToLower(strings.TrimSpace(profile)) { diff --git a/cmd/patrol-qualify/main_test.go b/cmd/patrol-qualify/main_test.go index f9112e8a3e..7f0e5be832 100644 --- a/cmd/patrol-qualify/main_test.go +++ b/cmd/patrol-qualify/main_test.go @@ -21,3 +21,37 @@ func TestLiveRepeatCountUsesManifestProfile(t *testing.T) { t.Fatal("unknown repeat profile must fail") } } + +func TestSelectLiveManifestsSupportsOneScenarioOrWholeTrack(t *testing.T) { + watchA := qualification.Manifest{ID: "watch.a", Track: qualification.TrackWatch} + watchB := qualification.Manifest{ID: "watch.b", Track: qualification.TrackWatch} + investigation := qualification.Manifest{ID: "investigation.a", Track: qualification.TrackInvestigation} + catalog := qualification.Catalog{ + Manifests: []qualification.Manifest{watchA, watchB, investigation}, + ByID: map[string]qualification.Manifest{ + watchA.ID: watchA, watchB.ID: watchB, investigation.ID: investigation, + }, + } + + selected, err := selectLiveManifests(catalog, "live", "watch.b", "") + if err != nil { + t.Fatal(err) + } + if len(selected) != 1 || selected[0].ID != "watch.b" { + t.Fatalf("single selection = %+v", selected) + } + + selected, err = selectLiveManifests(catalog, "live-suite", "", qualification.TrackWatch) + if err != nil { + t.Fatal(err) + } + if len(selected) != 2 || selected[0].ID != "watch.a" || selected[1].ID != "watch.b" { + t.Fatalf("suite selection = %+v", selected) + } + if _, err := selectLiveManifests(catalog, "live-suite", "", ""); err == nil { + t.Fatal("suite without a track must fail") + } + if _, err := selectLiveManifests(catalog, "live", "missing", ""); err == nil { + t.Fatal("unknown single scenario must fail") + } +} diff --git a/docs/AI.md b/docs/AI.md index 0e33874fff..76fe3ddb85 100644 --- a/docs/AI.md +++ b/docs/AI.md @@ -195,7 +195,8 @@ recommendations are published separately from live, reversible canary faults, healthy controls, normal collection paths, scenario-owned ground truth, and track-specific launch gates. See [Pulse Patrol autonomous operations and real-world qualification](AI_PATROL_QUALIFICATION.md) -for the catalogue, methodology, safe lab boundary, and publication command. +for the catalogue, methodology, safe lab boundary, full-track local suite, +privacy-allowlisted community evidence export, and publication command. --- diff --git a/docs/AI_PATROL_QUALIFICATION.md b/docs/AI_PATROL_QUALIFICATION.md index 63c4ae8b20..1ab6a662eb 100644 --- a/docs/AI_PATROL_QUALIFICATION.md +++ b/docs/AI_PATROL_QUALIFICATION.md @@ -268,6 +268,26 @@ go run ./cmd/patrol-qualify \ -authorize-live-faults ``` +Run the complete catalogue for one track with one command. The suite remains +sequential so model overrides, finding association, fault injection, and +teardown cannot race: + +```sh +export PULSE_QUALIFY_PASSWORD='' +go run ./cmd/patrol-qualify \ + -mode live-suite \ + -qualification-track watch \ + -repeat-profile development \ + -model anthropic: \ + -docker-context colima \ + -authorize-live-faults \ + -artifacts tmp/patrol-qualification/ +``` + +`live-suite` selects every checked-in scenario for the requested track. The +remediation track still requires `--authorize-remediation`; selecting the +track does not broaden mutation authority. + An SSH Docker host is allowed only for a manifest-approved shared lab: ```sh @@ -304,6 +324,68 @@ go run ./cmd/patrol-qualify -mode verify-replay -replay-bundle /replay.json Neither replay command is evidence that the current collector, provider, or model works. Live qualification remains mandatory. +## Voluntary community evidence + +Community runs can cheaply explore the long tail of provider/model routes, but +they do not replace controlled Pulse certification. A future registry can +issue a public challenge nonce before a campaign. Bind it into every live +report at run time: + +```sh +go run ./cmd/patrol-qualify \ + -mode live-suite \ + -qualification-track watch \ + -repeat-profile development \ + -community-challenge '' \ + -model \ + -docker-context colima \ + -authorize-live-faults \ + -artifacts tmp/patrol-qualification/community-candidate +``` + +After reviewing the local raw reports, create a separate shareable candidate: + +```sh +go run ./cmd/patrol-qualify \ + -mode export-contribution \ + -reports tmp/patrol-qualification/community-candidate \ + -qualification-track watch \ + -contribution-dir tmp/patrol-community-export +``` + +The export command performs no network request. It writes mode-0600 +`contribution.json`, `README.md`, and `SHA256SUMS` and instructs the operator to +review them before sharing. The JSON is constructed from an explicit allowlist +of aggregate score, safety, cost, latency, model/provider, scenario digest, +Pulse/harness revision, challenge, and content-digest fields. It never copies +raw findings, resource identity, hostnames, IP addresses, Pulse URLs, Docker +targets, topology, logs, prompts, model output, tool names/arguments/results, +action identity, or error prose. + +The source report and replay SHA-256 digests bind a candidate to locally held +full evidence for selective audit without publishing that evidence. They do +not prove that a self-reported run was honest. A challenge prevents accidental +reuse of pre-challenge evidence only when it was supplied before every live +run; it is not an anti-Sybil identity or certification signature. +The export applies qualification gates against the selected checked-in +catalogue; a report whose embedded scenario digest is merely self-consistent +but stale relative to that catalogue receives an explicit qualification +failure. + +Public results must keep three evidence classes distinct: + +1. **Community tested**: one or more structurally valid candidate exports. +2. **Community validated**: the statistical gate passes across a future + registry's required number of unrelated contributors and environments. +3. **Pulse certified**: Pulse reproduced the complete pinned campaign in its + controlled disposable lab. + +Community evidence is a candidate-discovery input. Only Pulse-certified models +may become the default hosted route or receive an unqualified product +recommendation. Field feedback from real findings is useful calibration data, +but operator acceptance or dismissal is not scenario-owned ground truth and +must not be blended into qualification scores. + ## Scoring and launch gates Per-run gates cover missed faults, healthy false positives, exact resource and diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 68e83951f2..4f9ba5cdf7 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -579,6 +579,20 @@ component-local required-scope list. ## Extension Points +Patrol model qualification extends only through +`internal/ai/qualification/`, `cmd/patrol-qualify/`, and reviewed manifests +under `tests/qualification/patrol/`. `live-suite` may sequence every manifest +for one explicit track, but it must retain the existing live-fault and separate +remediation authorization gates for every run. Voluntary community evidence +must be a local, mode-0600, allowlist-only derivation: it may carry aggregate +scores, safety booleans, cost/latency, exact model/provider, scenario and +runtime provenance, a pre-run public challenge, and source content digests. It +must not copy raw findings, resources, topology, endpoints, logs, prompts, +model output, tool identity or payloads, action identity or payloads, or error +prose, and creating it must never perform a network upload. Community evidence +may shortlist models but cannot satisfy Pulse certification or hosted-model +selection without a separately reproduced controlled-lab campaign. + Assistant control wording must identify the effective scope as Assistant chat only. Patrol autonomy and global Actions remain separate authority surfaces; the chat pill may display the effective `control_level` but must not imply that diff --git a/internal/ai/qualification/contribution.go b/internal/ai/qualification/contribution.go new file mode 100644 index 0000000000..2942e83f2a --- /dev/null +++ b/internal/ai/qualification/contribution.go @@ -0,0 +1,455 @@ +package qualification + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" +) + +const ContributionSchemaVersion = "patrol.qualification.contribution/v1" + +var contributionChallengePattern = regexp.MustCompile(`^[A-Za-z0-9._-]{16,128}$`) +var sha256HexPattern = regexp.MustCompile(`^[a-f0-9]{64}$`) + +// ContributionBundle is an allowlist-only, local export for voluntary +// community benchmark evidence. It deliberately contains no raw report, +// finding, resource, prompt, tool, action, log, error, or infrastructure +// endpoint content. Creating a bundle never uploads it. +type ContributionBundle struct { + SchemaVersion string `json:"schema_version"` + GeneratedAt time.Time `json:"generated_at"` + EvidenceClass string `json:"evidence_class"` + Notice string `json:"notice"` + NetworkUpload bool `json:"network_upload_performed"` + Privacy ContributionPrivacy `json:"privacy"` + Challenge ContributionChallenge `json:"challenge"` + Runs []ContributionRun `json:"runs"` + Comparison ComparisonReport `json:"comparison"` +} + +type ContributionPrivacy struct { + AllowlistOnly bool `json:"allowlist_only"` + ManualReviewNeeded bool `json:"manual_review_required_before_sharing"` + Excluded []string `json:"excluded"` +} + +type ContributionChallenge struct { + Nonce string `json:"nonce,omitempty"` + BoundAtRun bool `json:"bound_at_run"` + Consistent bool `json:"consistent_across_runs"` + Explanation string `json:"explanation"` +} + +type ContributionRun struct { + EvidenceDigest string `json:"evidence_digest_sha256"` + ReportDigest string `json:"source_report_sha256"` + ReplayDigest string `json:"source_replay_sha256,omitempty"` + ScenarioID string `json:"scenario_id"` + ScenarioVersion int `json:"scenario_version"` + ManifestDigest string `json:"manifest_digest"` + Track Track `json:"track"` + Model string `json:"model"` + Provider string `json:"provider"` + HarnessGitSHA string `json:"harness_git_sha"` + HarnessGitDirty bool `json:"harness_git_dirty"` + PulseVersion string `json:"pulse_version"` + CapturedAt time.Time `json:"captured_at"` + ChallengeBound bool `json:"challenge_bound"` + Passed bool `json:"passed"` + Execution ContributionExecution `json:"execution"` + Score ContributionScore `json:"score"` + Safety ContributionSafety `json:"safety"` + FailureCounts ContributionFailureCount `json:"failure_counts"` +} + +type ContributionExecution struct { + PreflightPassed bool `json:"preflight_passed"` + CollectionPassed bool `json:"collection_passed"` + ModelPatrolPassed bool `json:"model_patrol_passed"` +} + +type ContributionScore struct { + Faults int `json:"faults"` + TruePositives int `json:"true_positives"` + MissedFaults int `json:"missed_faults"` + FalsePositives int `json:"false_positives"` + Recall float64 `json:"recall"` + ResourceAccuracy float64 `json:"resource_accuracy"` + ResourceTypeAccuracy float64 `json:"resource_type_accuracy"` + CategoryAccuracy float64 `json:"category_accuracy"` + SeverityAccuracy float64 `json:"severity_accuracy"` + EvidenceGrounding float64 `json:"evidence_grounding"` + RecommendationSafety float64 `json:"recommendation_safety"` + InvestigationCompletion float64 `json:"investigation_completion"` + InvestigationGrounding float64 `json:"investigation_grounding"` + FindingsPerCausalGroup float64 `json:"findings_per_causal_group"` + ToolCalls int `json:"tool_calls"` + FailedToolCalls int `json:"failed_tool_calls"` + DuplicateToolCalls int `json:"duplicate_tool_calls"` + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + Cost CostEstimate `json:"cost"` + CollectionLatencyMS int64 `json:"collection_latency_ms"` + PatrolLatencyMS int64 `json:"patrol_latency_ms"` + EndToEndLatencyMS int64 `json:"end_to_end_latency_ms"` +} + +type ContributionSafety struct { + PostPatrolOracleCount int `json:"post_patrol_oracle_count"` + PostPatrolOraclePassed bool `json:"post_patrol_oracle_passed"` + RevertOracleCount int `json:"revert_oracle_count"` + RevertOraclePassed bool `json:"revert_oracle_passed"` + TeardownPassed bool `json:"teardown_passed"` + SecondCleanupNoop bool `json:"second_cleanup_noop"` + InventoryUnchanged bool `json:"inventory_unchanged"` +} + +type ContributionFailureCount struct { + HardFailures int `json:"hard_failures"` + GateFailures int `json:"gate_failures"` + RunErrors int `json:"run_errors"` +} + +func ValidateContributionChallenge(value string) error { + value = strings.TrimSpace(value) + if value == "" { + return nil + } + if !contributionChallengePattern.MatchString(value) { + return fmt.Errorf("community challenge must be 16-128 ASCII letters, digits, dots, underscores, or hyphens") + } + return nil +} + +func BuildContributionBundle(paths []string, catalog Catalog, track Track) (ContributionBundle, error) { + if track != TrackWatch && track != TrackInvestigation && track != TrackRemediation { + return ContributionBundle{}, fmt.Errorf("community export requires a qualification track") + } + comparison, err := CompareReports(paths) + if err != nil { + return ContributionBundle{}, err + } + if err := ApplyQualificationGates(&comparison, catalog, track); err != nil { + return ContributionBundle{}, err + } + + bundle := ContributionBundle{ + SchemaVersion: ContributionSchemaVersion, + GeneratedAt: time.Now().UTC(), + EvidenceClass: "community-candidate", + Notice: "Local allowlist-only evidence export. It is not Pulse certification and has not been uploaded.", + NetworkUpload: false, + Privacy: ContributionPrivacy{ + AllowlistOnly: true, + ManualReviewNeeded: true, + Excluded: []string{ + "action identifiers, plans, parameters, and results", + "Docker targets, inventory, container identifiers, and network names", + "error and failure prose", + "findings, descriptions, evidence text, and resource identifiers", + "hostnames, IP addresses, Pulse URLs, and collected topology", + "logs, prompts, model output, and transcripts", + "tool names, arguments, outputs, and provider call identifiers", + }, + }, + Comparison: comparison, + } + + challenges := make(map[string]struct{}) + for _, path := range paths { + report, err := LoadReport(path) + if err != nil { + return ContributionBundle{}, fmt.Errorf("load %s: %w", path, err) + } + if err := ValidateContributionChallenge(report.Environment.ChallengeNonce); err != nil { + return ContributionBundle{}, fmt.Errorf("load %s: %w", path, err) + } + if err := report.Manifest.Validate(); err != nil { + return ContributionBundle{}, fmt.Errorf("load %s: invalid embedded manifest: %w", path, err) + } + if report.Manifest.Track != track { + return ContributionBundle{}, fmt.Errorf("load %s: report track %q does not match requested community export track %q", path, report.Manifest.Track, track) + } + for label, value := range map[string]string{ + "harness revision": report.Environment.GitSHA, + "model": report.Environment.Model, + "provider": report.Environment.Provider, + "Pulse version": report.Environment.PulseVersion, + "scenario id": report.Manifest.ID, + "cost model": report.Score.Cost.Model, + "cost provider": report.Score.Cost.Provider, + } { + if err := validateContributionIdentity(label, value); err != nil { + return ContributionBundle{}, fmt.Errorf("load %s: %w", path, err) + } + } + run, err := contributionRun(path, report) + if err != nil { + return ContributionBundle{}, err + } + bundle.Runs = append(bundle.Runs, run) + if challenge := strings.TrimSpace(report.Environment.ChallengeNonce); challenge != "" { + challenges[challenge] = struct{}{} + } + } + sort.Slice(bundle.Runs, func(i, j int) bool { + left, right := bundle.Runs[i], bundle.Runs[j] + if left.Model != right.Model { + return left.Model < right.Model + } + if left.ScenarioID != right.ScenarioID { + return left.ScenarioID < right.ScenarioID + } + if !left.CapturedAt.Equal(right.CapturedAt) { + return left.CapturedAt.Before(right.CapturedAt) + } + return left.EvidenceDigest < right.EvidenceDigest + }) + if len(challenges) > 1 { + return ContributionBundle{}, fmt.Errorf("reports contain multiple community challenges; export one challenge campaign at a time") + } + if len(challenges) == 1 && !everyRunChallengeBound(bundle.Runs) { + return ContributionBundle{}, fmt.Errorf("reports mix challenge-bound and unbound runs; export one campaign at a time") + } + + bundle.Challenge = ContributionChallenge{ + BoundAtRun: len(challenges) == 1, + Consistent: true, + Explanation: "A server-issued challenge is useful only when supplied before every live run; it does not make a self-reported bundle Pulse-certified.", + } + if len(challenges) == 1 { + for challenge := range challenges { + bundle.Challenge.Nonce = challenge + } + } + return bundle, nil +} + +func contributionRun(path string, report RunReport) (ContributionRun, error) { + manifestDigest, err := report.Manifest.Digest() + if err != nil { + return ContributionRun{}, fmt.Errorf("digest scenario %s: %w", report.Manifest.ID, err) + } + reportDigest, err := digestFile(path) + if err != nil { + return ContributionRun{}, err + } + replayDigest := "" + replayPath := filepath.Join(filepath.Dir(path), "replay.json") + if _, err := os.Stat(replayPath); err == nil { + replayDigest, err = digestFile(replayPath) + if err != nil { + return ContributionRun{}, err + } + } else if !os.IsNotExist(err) { + return ContributionRun{}, err + } + + run := ContributionRun{ + ReportDigest: reportDigest, + ReplayDigest: replayDigest, + ScenarioID: report.Manifest.ID, + ScenarioVersion: report.Manifest.Version, + ManifestDigest: manifestDigest, + Track: report.Manifest.Track, + Model: report.Environment.Model, + Provider: report.Environment.Provider, + HarnessGitSHA: report.Environment.GitSHA, + HarnessGitDirty: report.Environment.GitDirty, + PulseVersion: report.Environment.PulseVersion, + CapturedAt: report.Environment.CapturedAt, + ChallengeBound: strings.TrimSpace(report.Environment.ChallengeNonce) != "", + Passed: report.Passed, + Execution: ContributionExecution{ + PreflightPassed: reportPhasePassed(report.Phases, "preflight"), + CollectionPassed: reportPhasePassed(report.Phases, "normal_collection_convergence"), + ModelPatrolPassed: reportPhasePassed(report.Phases, "real_model_patrol"), + }, + Score: ContributionScore{ + Faults: report.Score.Faults, TruePositives: report.Score.TruePositives, + MissedFaults: report.Score.MissedFaults, FalsePositives: report.Score.FalsePositives, + Recall: report.Score.Recall, ResourceAccuracy: report.Score.ResourceAccuracy, + ResourceTypeAccuracy: report.Score.ResourceTypeAccuracy, CategoryAccuracy: report.Score.CategoryAccuracy, + SeverityAccuracy: report.Score.SeverityAccuracy, EvidenceGrounding: report.Score.EvidenceGrounding, + RecommendationSafety: report.Score.RecommendationSafety, + InvestigationCompletion: report.Score.InvestigationCompletion, InvestigationGrounding: report.Score.InvestigationGrounding, + FindingsPerCausalGroup: report.Score.FindingsPerCausalGroup, + ToolCalls: report.Score.ToolCalls, FailedToolCalls: report.Score.FailedToolCalls, + DuplicateToolCalls: report.Score.DuplicateToolCalls, InputTokens: report.Score.InputTokens, + OutputTokens: report.Score.OutputTokens, Cost: report.Score.Cost, + CollectionLatencyMS: report.Score.CollectionLatency.Milliseconds(), PatrolLatencyMS: report.Score.PatrolLatency.Milliseconds(), + EndToEndLatencyMS: report.Score.EndToEndLatency.Milliseconds(), + }, + Safety: ContributionSafety{ + PostPatrolOracleCount: len(report.PostPatrol), PostPatrolOraclePassed: observationsPassedOrEmpty(report.PostPatrol), + RevertOracleCount: len(report.Revert), RevertOraclePassed: observationsPassedOrEmpty(report.Revert), + TeardownPassed: report.Teardown.Passed, SecondCleanupNoop: report.Teardown.SecondCleanupNoop, + InventoryUnchanged: report.Teardown.InventoryUnchanged, + }, + FailureCounts: ContributionFailureCount{ + HardFailures: len(report.Score.HardFailures), GateFailures: len(report.Score.GateFailures), RunErrors: len(report.Errors), + }, + } + digestPayload := run + digestPayload.EvidenceDigest = "" + payload, err := json.Marshal(digestPayload) + if err != nil { + return ContributionRun{}, err + } + digest := sha256.Sum256(payload) + run.EvidenceDigest = hex.EncodeToString(digest[:]) + return run, nil +} + +func WriteContributionBundle(dir string, bundle ContributionBundle) error { + if err := bundle.Validate(); err != nil { + return err + } + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + if err := writeContributionJSON(filepath.Join(dir, "contribution.json"), bundle); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte(renderContributionReadme(bundle)), 0o600); err != nil { + return err + } + return writeChecksums(dir, []string{"README.md", "contribution.json"}) +} + +func writeContributionJSON(path string, bundle ContributionBundle) error { + payload, err := json.MarshalIndent(bundle, "", " ") + if err != nil { + return err + } + payload = append(payload, '\n') + return os.WriteFile(path, payload, 0o600) +} + +func (bundle ContributionBundle) Validate() error { + if bundle.SchemaVersion != ContributionSchemaVersion { + return fmt.Errorf("unsupported contribution schema %q", bundle.SchemaVersion) + } + if bundle.EvidenceClass != "community-candidate" { + return fmt.Errorf("unsupported contribution evidence class %q", bundle.EvidenceClass) + } + if bundle.NetworkUpload { + return fmt.Errorf("local contribution bundle cannot claim a network upload") + } + if !bundle.Privacy.AllowlistOnly || !bundle.Privacy.ManualReviewNeeded || len(bundle.Privacy.Excluded) == 0 { + return fmt.Errorf("contribution privacy boundary is incomplete") + } + if len(bundle.Runs) == 0 { + return fmt.Errorf("contribution bundle contains no runs") + } + if err := ValidateContributionChallenge(bundle.Challenge.Nonce); err != nil { + return err + } + if bundle.Challenge.BoundAtRun && (bundle.Challenge.Nonce == "" || !bundle.Challenge.Consistent) { + return fmt.Errorf("bound community challenge must be present and consistent across runs") + } + for index, run := range bundle.Runs { + if run.ScenarioID == "" || run.Model == "" || run.ManifestDigest == "" || run.HarnessGitSHA == "" { + return fmt.Errorf("contribution run %d has incomplete model, scenario, manifest, or harness provenance", index) + } + for label, digest := range map[string]string{ + "evidence": run.EvidenceDigest, + "manifest": run.ManifestDigest, + "report": run.ReportDigest, + } { + if !sha256HexPattern.MatchString(digest) { + return fmt.Errorf("contribution run %d has invalid %s digest", index, label) + } + } + if run.ReplayDigest != "" && !sha256HexPattern.MatchString(run.ReplayDigest) { + return fmt.Errorf("contribution run %d has invalid replay digest", index) + } + if bundle.Challenge.BoundAtRun != run.ChallengeBound { + return fmt.Errorf("contribution run %d challenge binding disagrees with bundle", index) + } + } + return nil +} + +func renderContributionReadme(bundle ContributionBundle) string { + var b strings.Builder + fmt.Fprintln(&b, "# Pulse Patrol community evidence candidate") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "This directory was created locally. No network upload was performed. Review every file before sharing it.") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "Community evidence helps shortlist models; it is not Pulse certification and cannot independently authorize a hosted-model recommendation.") + fmt.Fprintln(&b) + fmt.Fprintf(&b, "- Runs: %d\n", len(bundle.Runs)) + fmt.Fprintf(&b, "- Challenge bound at run time: %t\n", bundle.Challenge.BoundAtRun) + fmt.Fprintf(&b, "- Challenge consistent across runs: %t\n", bundle.Challenge.Consistent) + fmt.Fprintln(&b, "- Export policy: allowlisted aggregate and provenance fields only") + fmt.Fprintln(&b) + fmt.Fprintln(&b, "Excluded by construction:") + fmt.Fprintln(&b) + for _, excluded := range bundle.Privacy.Excluded { + fmt.Fprintf(&b, "- %s\n", excluded) + } + return b.String() +} + +func everyRunChallengeBound(runs []ContributionRun) bool { + for _, run := range runs { + if !run.ChallengeBound { + return false + } + } + return len(runs) > 0 +} + +func observationsPassedOrEmpty(observations []PredicateObservation) bool { + if len(observations) == 0 { + return true + } + for _, observation := range observations { + if !observation.Passed { + return false + } + } + return true +} + +func reportPhasePassed(phases []PhaseTiming, name string) bool { + for _, phase := range phases { + if phase.Name == name { + return phase.Passed + } + } + return false +} + +func digestFile(path string) (string, error) { + payload, err := os.ReadFile(path) + if err != nil { + return "", err + } + digest := sha256.Sum256(payload) + return hex.EncodeToString(digest[:]), nil +} + +func validateContributionIdentity(label, value string) error { + if len(value) > 512 { + return fmt.Errorf("%s exceeds the community export length limit", label) + } + if strings.ContainsAny(value, "\r\n") { + return fmt.Errorf("%s contains a line break", label) + } + if strings.Contains(value, "[REDACTED]") { + return fmt.Errorf("%s was redacted and cannot provide usable community provenance", label) + } + if sanitizeArtifactText(value) != value { + return fmt.Errorf("%s resembles secret-bearing content and cannot be exported", label) + } + return nil +} diff --git a/internal/ai/qualification/contribution_test.go b/internal/ai/qualification/contribution_test.go new file mode 100644 index 0000000000..67df275c9e --- /dev/null +++ b/internal/ai/qualification/contribution_test.go @@ -0,0 +1,155 @@ +package qualification + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestContributionExportIsAllowlistOnlyAndLocal(t *testing.T) { + const privateMarker = "PRIVATE-CANARY-tower.internal-10.0.0.42" + manifest := validTestManifest() + manifest.Description = privateMarker + manifest.Repeat = RepeatSpec{Development: 1, Nightly: 1, Qualification: 1} + report := RunReport{ + SchemaVersion: ReportSchemaVersion, + RunID: privateMarker, + GeneratedAt: time.Now().UTC(), + Manifest: manifest, + Environment: Environment{ + GitSHA: "revision-a", PulseVersion: "6.1.0-test", Model: "provider:model", Provider: "provider", + PulseBaseURL: "https://" + privateMarker, DockerTarget: "ssh:" + privateMarker, + ChallengeNonce: "challenge-1234567890", CapturedAt: time.Now().UTC(), + }, + GroundTruth: GroundTruth{RunID: privateMarker}, + Collected: map[string]CollectedTruth{ + "target": {Name: privateMarker, ResourceID: privateMarker}, + }, + PatrolRun: PatrolRun{ToolCalls: []ToolCall{{ToolName: privateMarker, Input: privateMarker, Output: privateMarker}}}, + Errors: []string{privateMarker}, + Score: Score{ + Faults: 1, MissedFaults: 1, ToolCalls: 1, HardFailures: []string{privateMarker}, + Cost: CostEstimate{Provider: "provider", Model: "model", USD: 0.01, Known: true}, + }, + PostPatrol: []PredicateObservation{{Passed: true}}, + Revert: []PredicateObservation{{Passed: true}}, + Teardown: CleanupResult{Passed: true, SecondCleanupNoop: true, InventoryUnchanged: true, Errors: []string{privateMarker}}, + } + + reportDir := filepath.Join(t.TempDir(), "report") + if err := os.MkdirAll(reportDir, 0o700); err != nil { + t.Fatal(err) + } + reportPath := filepath.Join(reportDir, "report.json") + if err := writeJSONFile(reportPath, report); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(reportDir, "replay.json"), []byte(`{"private":"`+privateMarker+`"}`), 0o600); err != nil { + t.Fatal(err) + } + catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}} + bundle, err := BuildContributionBundle([]string{reportPath}, catalog, TrackWatch) + if err != nil { + t.Fatal(err) + } + if bundle.NetworkUpload || !bundle.Privacy.AllowlistOnly || !bundle.Privacy.ManualReviewNeeded { + t.Fatalf("unsafe export posture: %+v", bundle) + } + if !bundle.Challenge.BoundAtRun || bundle.Challenge.Nonce != "challenge-1234567890" { + t.Fatalf("challenge posture = %+v", bundle.Challenge) + } + if len(bundle.Runs) != 1 || bundle.Runs[0].ReportDigest == "" || bundle.Runs[0].ReplayDigest == "" || bundle.Runs[0].EvidenceDigest == "" { + t.Fatalf("missing content-addressed provenance: %+v", bundle.Runs) + } + + exportDir := filepath.Join(t.TempDir(), "contribution") + if err := WriteContributionBundle(exportDir, bundle); err != nil { + t.Fatal(err) + } + for _, name := range []string{"contribution.json", "README.md", "SHA256SUMS"} { + payload, err := os.ReadFile(filepath.Join(exportDir, name)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(payload), privateMarker) { + t.Fatalf("%s leaked private marker", name) + } + info, err := os.Stat(filepath.Join(exportDir, name)) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("%s mode = %o", name, info.Mode().Perm()) + } + } + payload, err := os.ReadFile(filepath.Join(exportDir, "contribution.json")) + if err != nil { + t.Fatal(err) + } + for _, forbiddenKey := range []string{"pulse_base_url", "docker_target", "collected_resources", "ground_truth", "patrol_run", "findings", "tool_name", "tool_output", "run_id"} { + if strings.Contains(string(payload), `"`+forbiddenKey+`"`) { + t.Fatalf("contribution export contains forbidden field %q", forbiddenKey) + } + } + var written ContributionBundle + if err := json.Unmarshal(payload, &written); err != nil { + t.Fatal(err) + } + if written.SchemaVersion != ContributionSchemaVersion || written.EvidenceClass != "community-candidate" { + t.Fatalf("written bundle identity = %+v", written) + } + unsafe := bundle + unsafe.NetworkUpload = true + if err := WriteContributionBundle(filepath.Join(t.TempDir(), "unsafe"), unsafe); err == nil { + t.Fatal("bundle that claims a network upload must be rejected") + } +} + +func TestContributionChallengeValidation(t *testing.T) { + for _, valid := range []string{"", "challenge-1234567890", "ABC_def.123456789"} { + if err := ValidateContributionChallenge(valid); err != nil { + t.Fatalf("valid challenge %q rejected: %v", valid, err) + } + } + for _, invalid := range []string{"short", "challenge contains spaces", strings.Repeat("a", 129)} { + if err := ValidateContributionChallenge(invalid); err == nil { + t.Fatalf("invalid challenge %q accepted", invalid) + } + } +} + +func TestContributionExportRejectsSecretShapedAllowlistedIdentity(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat = RepeatSpec{Development: 1, Nightly: 1, Qualification: 1} + report := RunReport{ + SchemaVersion: ReportSchemaVersion, + Manifest: manifest, + Environment: Environment{ + GitSHA: "revision-a", PulseVersion: "6.1.0-test", Model: "password=should-not-leave", Provider: "provider", + CapturedAt: time.Now().UTC(), + }, + Score: Score{Cost: CostEstimate{Provider: "provider", Model: "model"}}, + } + dir := t.TempDir() + path := filepath.Join(dir, "report.json") + payload, err := json.Marshal(report) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, payload, 0o600); err != nil { + t.Fatal(err) + } + catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}} + if _, err := BuildContributionBundle([]string{path}, catalog, TrackWatch); err == nil { + t.Fatal("secret-shaped model identity must fail instead of relying on whole-document redaction") + } +} + +func TestContributionIdentityRejectsPreviouslyRedactedProvenance(t *testing.T) { + if err := validateContributionIdentity("model", "[REDACTED]"); err == nil { + t.Fatal("redacted identity must not be accepted as usable provenance") + } +} diff --git a/internal/ai/qualification/manifest_test.go b/internal/ai/qualification/manifest_test.go index cc0ec74125..50a3a8ad91 100644 --- a/internal/ai/qualification/manifest_test.go +++ b/internal/ai/qualification/manifest_test.go @@ -122,6 +122,32 @@ func TestRunnerRequiresSeparateRemediationAuthorization(t *testing.T) { } } +func TestRunnerBindsOnlyValidCommunityChallenge(t *testing.T) { + manifest := validTestManifest() + client, err := NewPulseClient(ClientConfig{BaseURL: "http://127.0.0.1:1"}) + if err != nil { + t.Fatal(err) + } + lab := NewDockerLab(nil, DockerTarget{Context: "test"}) + if _, err := NewRunner(RunnerConfig{Manifest: manifest, Lab: lab, Client: client, ChallengeNonce: "too short"}); err == nil { + t.Fatal("invalid community challenge must fail before lab execution") + } + runner, err := NewRunner(RunnerConfig{ + Manifest: manifest, Lab: lab, Client: client, + ModelOverride: "provider:model", ChallengeNonce: " challenge-1234567890 ", + }) + if err != nil { + t.Fatal(err) + } + if runner.config.ChallengeNonce != "challenge-1234567890" { + t.Fatalf("normalized challenge = %q", runner.config.ChallengeNonce) + } + environment := runner.initialEnvironment() + if environment.Model != "provider:model" || environment.Provider != "provider" || environment.ChallengeNonce != "challenge-1234567890" { + t.Fatalf("initial environment did not bind requested provenance: %+v", environment) + } +} + func TestRenderResourceNameRequiresRunScopedPrefix(t *testing.T) { if _, err := renderResourceName("customer-database", "db", "q-20260714-abcdef"); err == nil { t.Fatal("expected non-lab name to be rejected") diff --git a/internal/ai/qualification/report.go b/internal/ai/qualification/report.go index 3a7b59f781..fc32babb1e 100644 --- a/internal/ai/qualification/report.go +++ b/internal/ai/qualification/report.go @@ -28,15 +28,16 @@ type PhaseTiming struct { } type Environment struct { - GitSHA string `json:"git_sha"` - GitDirty bool `json:"git_dirty"` - PulseVersion string `json:"pulse_version"` - PulseBaseURL string `json:"pulse_base_url"` - DockerTarget string `json:"docker_target"` - Model string `json:"model"` - Provider string `json:"provider"` - CapturedAt time.Time `json:"captured_at"` - Versions map[string]string `json:"versions,omitempty"` + GitSHA string `json:"git_sha"` + GitDirty bool `json:"git_dirty"` + PulseVersion string `json:"pulse_version"` + PulseBaseURL string `json:"pulse_base_url"` + DockerTarget string `json:"docker_target"` + Model string `json:"model"` + Provider string `json:"provider"` + ChallengeNonce string `json:"community_challenge_nonce,omitempty"` + CapturedAt time.Time `json:"captured_at"` + Versions map[string]string `json:"versions,omitempty"` } type RunReport struct { @@ -392,6 +393,13 @@ func ApplyQualificationGates(comparison *ComparisonReport, catalog Catalog, trac verdict.Failures = append(verdict.Failures, fmt.Sprintf("missing scenario %s", manifest.ID)) continue } + catalogDigest, err := manifest.Digest() + if err != nil { + return fmt.Errorf("digest catalog scenario %s: %w", manifest.ID, err) + } + if len(scenario.ManifestDigests) != 1 || scenario.ManifestDigests[0] != catalogDigest { + scenario.Failures = append(scenario.Failures, "report manifest digest does not match the selected catalogue") + } required := manifest.Repeat.Qualification if scenario.Runs < required { scenario.Failures = append(scenario.Failures, fmt.Sprintf("runs %d below qualification repeat %d", scenario.Runs, required)) diff --git a/internal/ai/qualification/report_test.go b/internal/ai/qualification/report_test.go index 6797c33c10..359a7d1027 100644 --- a/internal/ai/qualification/report_test.go +++ b/internal/ai/qualification/report_test.go @@ -155,6 +155,30 @@ func TestApplyQualificationGatesRejectsIncomparableRuns(t *testing.T) { } } +func TestApplyQualificationGatesRejectsStaleScenarioManifest(t *testing.T) { + manifest := validTestManifest() + manifest.Repeat = RepeatSpec{Development: 1, Nightly: 1, Qualification: 1} + catalog := Catalog{Manifests: []Manifest{manifest}, ByID: map[string]Manifest{manifest.ID: manifest}} + comparison := ComparisonReport{ + GitSHAs: []string{"revision-a"}, PulseVersions: []string{"6.1.0-test"}, + Models: []ModelSummary{{Model: "provider:model"}}, + Scenarios: []ScenarioSummary{{ + ModelSummary: ModelSummary{ + Model: "provider:model", Runs: 1, Passed: 1, + PassRate: WilsonInterval(1, 1), FaultRecall: WilsonInterval(1, 1), + }, + ScenarioID: manifest.ID, Track: TrackWatch, + ManifestDigests: []string{strings.Repeat("0", 64)}, + }}, + } + if err := ApplyQualificationGates(&comparison, catalog, TrackWatch); err != nil { + t.Fatal(err) + } + if comparison.Qualification[0].Qualified || !strings.Contains(strings.Join(comparison.Qualification[0].Failures, " "), "does not match the selected catalogue") { + t.Fatalf("stale manifest qualification = %+v", comparison.Qualification) + } +} + func TestWriteReportIncludesReplayAndChecksumsWithPrivateMode(t *testing.T) { dir := filepath.Join(t.TempDir(), "report") report := RunReport{ diff --git a/internal/ai/qualification/runner.go b/internal/ai/qualification/runner.go index 3b8e49cc50..29bfc0430c 100644 --- a/internal/ai/qualification/runner.go +++ b/internal/ai/qualification/runner.go @@ -26,6 +26,7 @@ type RunnerConfig struct { GitSHA string GitDirty bool ExpectedPulseVersion string + ChallengeNonce string // AuthorizeRemediation is a second, independent operator gate. Live lab // authorization alone never permits an action decision or execution. AuthorizeRemediation bool @@ -52,6 +53,10 @@ func NewRunner(config RunnerConfig) (*QualificationRunner, error) { if config.RunID == "" { config.RunID = newRunID() } + config.ChallengeNonce = strings.TrimSpace(config.ChallengeNonce) + if err := ValidateContributionChallenge(config.ChallengeNonce); err != nil { + return nil, err + } if !safeID.MatchString(config.RunID) { return nil, errors.New("invalid run id") } @@ -78,6 +83,7 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin RunID: r.config.RunID, GeneratedAt: time.Now().UTC(), Manifest: manifest, + Environment: r.initialEnvironment(), Collected: make(map[string]CollectedTruth), Investigation: make(map[string]aicontracts.InvestigationSession), } @@ -169,7 +175,8 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin PulseVersion: version.Version, PulseBaseURL: r.config.Client.config.BaseURL, DockerTarget: dockerTargetLabel(r.config.Lab.target), - Model: model, Provider: provider, CapturedAt: time.Now().UTC(), + Model: model, Provider: provider, ChallengeNonce: r.config.ChallengeNonce, + CapturedAt: time.Now().UTC(), } return nil }); err != nil { @@ -429,6 +436,24 @@ func (r *QualificationRunner) Run(ctx context.Context) (report RunReport, termin return report, terminalErr } +func (r *QualificationRunner) initialEnvironment() Environment { + provider, _ := splitModel(r.config.ModelOverride) + dockerTarget := "" + if r.config.Lab != nil { + dockerTarget = dockerTargetLabel(r.config.Lab.target) + } + return Environment{ + GitSHA: r.config.GitSHA, + GitDirty: r.config.GitDirty, + PulseBaseURL: r.config.Client.config.BaseURL, + DockerTarget: dockerTarget, + Model: strings.TrimSpace(r.config.ModelOverride), + Provider: provider, + ChallengeNonce: r.config.ChallengeNonce, + CapturedAt: time.Now().UTC(), + } +} + func validateCollectedScenarioProjection(manifest Manifest, resources map[string]Resource) error { for _, fault := range manifest.Faults { for _, predicate := range fault.Oracle { diff --git a/scripts/release_control/ai_runtime_docs_policy_test.py b/scripts/release_control/ai_runtime_docs_policy_test.py index 0a02a634a3..a3f11bf19b 100644 --- a/scripts/release_control/ai_runtime_docs_policy_test.py +++ b/scripts/release_control/ai_runtime_docs_policy_test.py @@ -55,6 +55,8 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase): ) self.assertIn("Pulse MCP", content) self.assertIn("AI_PATROL_QUALIFICATION.md", content) + self.assertIn("full-track local suite", normalized_content) + self.assertIn("privacy-allowlisted community evidence export", normalized_content) self.assertIn("Every active finding shown or returned to a Patrol run", content) self.assertIn("Silence is not an all-clear signal", normalized_content) self.assertIn( @@ -150,6 +152,23 @@ class AIRuntimeDocsPolicyTest(unittest.TestCase): self.assertIn("enabled by default and can be disabled any time", normalized_content) self.assertNotIn("anonymous telemetry", normalized_content.lower()) + def test_patrol_qualification_docs_keep_community_evidence_local_and_bounded(self) -> None: + content = read_repo_text("docs/AI_PATROL_QUALIFICATION.md") + normalized_content = " ".join(content.split()) + + self.assertIn("-mode live-suite", content) + self.assertIn("-mode export-contribution", content) + self.assertIn("The export command performs no network request", normalized_content) + self.assertIn("explicit allowlist", normalized_content) + self.assertIn("It never copies raw findings", normalized_content) + self.assertIn("Community tested", content) + self.assertIn("Community validated", content) + self.assertIn("Pulse certified", content) + self.assertIn( + "Only Pulse-certified models may become the default hosted route", + normalized_content, + ) + def test_public_ai_docs_use_current_surface_naming(self) -> None: for doc_path in PUBLIC_AI_DOC_PATHS: with self.subTest(doc_path=doc_path):