From 48cf66323769bd2b85e01486bc09a6569b868976 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Sat, 27 Jun 2026 14:53:48 +0100 Subject: [PATCH] Strengthen evidence and impact guidance in patrol finding tool The patrol_report_finding tool descriptions for evidence, impact, and recommendation were too permissive, causing the LLM to frequently omit them. Future patrol runs will produce better-scaffolded findings. 1. Rewrite tool descriptions to emphasize evidence as a trust anchor that should always be included, and impact as expected whenever the data supports it. 2. Add 'Authoring Evidence' section to the patrol system prompt with concrete examples, matching the existing 'Authoring Impact' section. 3. Add test verifying trust scaffolding guidance is in the system prompt. Addresses checklist L48-50 (trust scaffolding). --- internal/ai/patrol_ai.go | 8 ++++++++ internal/ai/patrol_ai_more_test.go | 17 +++++++++++++++++ internal/ai/tools/tools_patrol.go | 14 +++++++++----- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index ab89dc5f1..026e2d959 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -953,6 +953,14 @@ Every finding you report should answer "what specifically happens if the operato - Do NOT echo severity or category. The operator already sees "warning" or "capacity"; impact must add information. - Do NOT fabricate consequences when you genuinely do not know. Leave ` + "`impact`" + ` empty rather than inventing one. The frontend will render an explicit "Impact not assessed" placeholder, which is more useful than guessed copy. +## Authoring Evidence (what you checked) + +Every finding must include the ` + "`evidence`" + ` field in patrol_report_finding. This is the trust anchor that lets the operator verify your conclusion independently. + +- Include the specific metric values, command outputs, log entries, or tool results that led to the finding. "ZFS pool 'data' has 7 checksum errors (checked via zpool status)" is good. "Storage issue detected" is not. +- If you used pulse_metrics, pulse_storage, pulse_read, or any investigation tool, reference the key data points from those calls. +- The operator should be able to look at your evidence and confirm: yes, this is real, I can see the same thing. + ## Final Summary Format After completing your investigation, write a concise summary using this structure: diff --git a/internal/ai/patrol_ai_more_test.go b/internal/ai/patrol_ai_more_test.go index 44d6beb3a..e92f6a8b6 100644 --- a/internal/ai/patrol_ai_more_test.go +++ b/internal/ai/patrol_ai_more_test.go @@ -268,6 +268,23 @@ func TestGetPatrolSystemPrompt_ModeSwitch(t *testing.T) { } } +func TestGetPatrolSystemPrompt_IncludesTrustScaffoldingGuidance(t *testing.T) { + svc := &Service{cfg: &config.AIConfig{PatrolAutonomyLevel: config.PatrolAutonomyMonitor}} + ps := NewPatrolService(svc, nil) + prompt := ps.getPatrolSystemPrompt() + + required := []string{ + "Authoring Impact", + "Authoring Evidence", + "trust anchor", + } + for _, want := range required { + if !strings.Contains(prompt, want) { + t.Fatalf("patrol system prompt missing %q", want) + } + } +} + func TestRunAIAnalysis_EarlyErrors(t *testing.T) { t.Run("nil service", func(t *testing.T) { ps := NewPatrolService(nil, nil) diff --git a/internal/ai/tools/tools_patrol.go b/internal/ai/tools/tools_patrol.go index c2adc8fab..7bb8bb101 100644 --- a/internal/ai/tools/tools_patrol.go +++ b/internal/ai/tools/tools_patrol.go @@ -60,16 +60,20 @@ Returns: {"ok": true, "finding_id": "...", "is_new": true/false} on success.`, Description: "Detailed description of the issue found", }, "impact": { - Type: "string", - Description: "Operator-facing consequence-if-ignored statement: what specifically happens, and to which workloads or jobs, if the operator does nothing. Concrete operational consequences only — do not echo severity or category. Leave empty if the consequence is genuinely unknown rather than fabricating one.", + Type: "string", + Description: "Operator-facing consequence if ignored: what happens to which workloads, jobs, or data if the operator does nothing. " + + "Base this on the data you gathered, not on the severity level. Most infrastructure issues have knowable consequences — " + + "provide this whenever the data supports it. Omit only if the consequence is truly unknowable from available information.", }, "recommendation": { Type: "string", - Description: "Specific actionable recommendation for the operator", + Description: "Specific, actionable next step the operator should take. Include verification steps where possible.", }, "evidence": { - Type: "string", - Description: "Specific data/metrics/commands that support this finding", + Type: "string", + Description: "Specific data points, metric values, command outputs, or tool results that led to this finding. " + + "This is the trust anchor: the operator uses it to verify your conclusion independently. " + + "Always include the key evidence — if you gathered enough data to create this finding, you have evidence to report.", }, }, Required: []string{"key", "severity", "category", "resource_id", "resource_name", "resource_type", "title", "description"},