From 2b271e3e20085fa120b71f43dd5420de3acd92ea Mon Sep 17 00:00:00 2001 From: rcourtman Date: Tue, 14 Jul 2026 18:49:25 +0100 Subject: [PATCH] Repair disposable dependency qualification fixtures --- .../v6/internal/subsystems/ai-runtime.md | 6 +++++ internal/ai/qualification/manifest_test.go | 27 +++++++++++++++++++ .../investigation.docker-dependency.json | 2 +- .../watch.correlated-dependency.json | 2 +- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index d2b562ecb7..158d2351a8 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -5674,6 +5674,12 @@ tool result is authoritative and the remaining text-only provider turn is only for operator-facing summary prose. That turn must use a bounded summary system instruction rather than resending the full detection and investigation prompt; it may not invent findings, evidence, actions, verification, or remediation. +Docker dependency qualification fixtures must use only executables present in +the catalogue's pinned image and must reach a healthy dependency/client +baseline before fault injection. The Watch correlation and Pro investigation +fixtures use Alpine's `nc` applet for the disposable HTTP dependency; a missing +optional daemon must fail baseline convergence and never be scored as a model +miss. Persisted Patrol tool inputs must retain complete structured finding calls up to the bounded 16 KiB record limit so normal evidence-rich findings remain deterministically replayable. If any captured input is nevertheless incomplete diff --git a/internal/ai/qualification/manifest_test.go b/internal/ai/qualification/manifest_test.go index 717049fe48..d754487d57 100644 --- a/internal/ai/qualification/manifest_test.go +++ b/internal/ai/qualification/manifest_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "os" "path/filepath" + "strings" "testing" ) @@ -22,6 +23,32 @@ func TestLoadCatalogValidatesCheckedInScenarios(t *testing.T) { } } +func TestDependencyScenariosUsePinnedImageExecutables(t *testing.T) { + catalog, err := LoadCatalog(filepath.Join("..", "..", "..", "tests", "qualification", "patrol", "scenarios")) + if err != nil { + t.Fatal(err) + } + for _, id := range []string{"watch.correlated-dependency", "investigation.docker-dependency"} { + manifest, ok := catalog.ByID[id] + if !ok { + t.Fatalf("catalog missing %s", id) + } + var command string + for _, resource := range manifest.Resources { + if resource.Alias == "dependency" { + command = strings.Join(resource.Command, " ") + break + } + } + if !strings.Contains(command, "nc -l -p 8080") { + t.Fatalf("%s dependency does not use the pinned Alpine nc applet: %q", id, command) + } + if strings.Contains(command, "httpd") { + t.Fatalf("%s dependency requires optional httpd: %q", id, command) + } + } +} + func TestLoadManifestRejectsTrailingJSON(t *testing.T) { path := filepath.Join(t.TempDir(), "scenario.json") payload, err := json.Marshal(validTestManifest()) diff --git a/tests/qualification/patrol/scenarios/investigation.docker-dependency.json b/tests/qualification/patrol/scenarios/investigation.docker-dependency.json index bc0ed58a82..d87ac8998b 100644 --- a/tests/qualification/patrol/scenarios/investigation.docker-dependency.json +++ b/tests/qualification/patrol/scenarios/investigation.docker-dependency.json @@ -10,7 +10,7 @@ "tags": ["docker", "investigation", "proposal", "live"], "lab": {"driver": "docker", "profile": "docker-canary-v1", "image": "alpine:3.20", "allow_pull": false, "shared_host_ok": true}, "resources": [ - {"alias": "dependency", "kind": "container", "name": "pulse-qual-${run_id}-dependency", "command": ["/bin/sh", "-c", "mkdir -p /www; echo ok >/www/index.html; exec httpd -f -p 8080 -h /www"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://127.0.0.1:8080"], "health_every": "2s"}, + {"alias": "dependency", "kind": "container", "name": "pulse-qual-${run_id}-dependency", "command": ["/bin/sh", "-c", "while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: 3\\r\\n\\r\\nok\\n' | nc -l -p 8080; done"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://127.0.0.1:8080"], "health_every": "2s"}, {"alias": "client", "kind": "container", "name": "pulse-qual-${run_id}-client", "command": ["/bin/sh", "-c", "while true; do sleep 3600; done"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://pulse-qual-${run_id}-dependency:8080"], "health_every": "2s"} ], "baseline": [{"probe": "docker.health", "target": "dependency", "operator": "eq", "value": "healthy", "timeout": "30s"}, {"probe": "docker.health", "target": "client", "operator": "eq", "value": "healthy", "timeout": "30s"}], diff --git a/tests/qualification/patrol/scenarios/watch.correlated-dependency.json b/tests/qualification/patrol/scenarios/watch.correlated-dependency.json index a1b2f29752..53182f68a3 100644 --- a/tests/qualification/patrol/scenarios/watch.correlated-dependency.json +++ b/tests/qualification/patrol/scenarios/watch.correlated-dependency.json @@ -10,7 +10,7 @@ "tags": ["docker", "correlation", "deduplication", "live"], "lab": {"driver": "docker", "profile": "docker-canary-v1", "image": "alpine:3.20", "allow_pull": false, "shared_host_ok": true}, "resources": [ - {"alias": "dependency", "kind": "container", "name": "pulse-qual-${run_id}-dependency", "command": ["/bin/sh", "-c", "mkdir -p /www; echo ok >/www/index.html; exec httpd -f -p 8080 -h /www"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://127.0.0.1:8080"], "health_every": "2s"}, + {"alias": "dependency", "kind": "container", "name": "pulse-qual-${run_id}-dependency", "command": ["/bin/sh", "-c", "while true; do printf 'HTTP/1.1 200 OK\\r\\nContent-Length: 3\\r\\n\\r\\nok\\n' | nc -l -p 8080; done"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://127.0.0.1:8080"], "health_every": "2s"}, {"alias": "client", "kind": "container", "name": "pulse-qual-${run_id}-client", "command": ["/bin/sh", "-c", "while true; do sleep 3600; done"], "healthcheck": ["wget", "-q", "-T", "1", "-O", "-", "http://pulse-qual-${run_id}-dependency:8080"], "health_every": "2s"} ], "baseline": [{"probe": "docker.health", "target": "dependency", "operator": "eq", "value": "healthy", "timeout": "30s"}, {"probe": "docker.health", "target": "client", "operator": "eq", "value": "healthy", "timeout": "30s"}],