From 87aafb86c4bba27293f53de040418c661a8e68e5 Mon Sep 17 00:00:00 2001 From: rcourtman Date: Wed, 1 Apr 2026 23:09:32 +0100 Subject: [PATCH] Retry Patrol with provider-derived seed budgets --- .../v6/internal/subsystems/ai-runtime.md | 6 + internal/ai/patrol_ai.go | 516 ++++++++++++------ internal/ai/patrol_ai_more_test.go | 86 +++ internal/ai/patrol_triage.go | 138 +++-- 4 files changed, 553 insertions(+), 193 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index 11c3a109e..cc10cced6 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -87,6 +87,12 @@ governed floor is ready. `internal/ai/` is the live backend AI engine. It owns chat execution, Patrol orchestration, findings generation, investigation support, quickstart and provider selection, remediation flow, and cost persistence. +That Patrol runtime ownership includes seed-context admission control. +`internal/ai/patrol_ai.go` must build Patrol and triage prompts from +canonical seed sections, size them against the runtime budget model, and when +a provider reports a smaller real context window than the static model map, +reassemble the same canonical sections under tighter provider-derived budgets +instead of hard-failing or truncating ad hoc prompt strings. That same backend runtime ownership also includes bounded Patrol and investigation read models. `internal/ai/patrol_history_persistence.go` and `internal/ai/proxmox/events.go` must cap persisted-history loads and diff --git a/internal/ai/patrol_ai.go b/internal/ai/patrol_ai.go index 5434f95c1..10a6c4498 100644 --- a/internal/ai/patrol_ai.go +++ b/internal/ai/patrol_ai.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "fmt" + "regexp" "sort" "strconv" "strings" @@ -46,8 +47,19 @@ const ( patrolTurnsPer50Devices = 5 patrolQuickMinTurns = 10 patrolQuickMaxTurns = 30 + patrolRetrySeedBudget1 = 16_000 + patrolRetrySeedBudget2 = 8_000 + patrolRetrySeedBudget3 = 4_000 ) +var patrolContextWindowPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)"n_ctx"\s*:\s*(\d+)`), + regexp.MustCompile(`(?i)available context size\s*\((\d+)\s*tokens?\)`), + regexp.MustCompile(`(?i)maximum context length[^0-9]*(\d+)`), + regexp.MustCompile(`(?i)context window[^0-9]*(\d+)`), + regexp.MustCompile(`(?i)context length[^0-9]*(\d+)`), +} + // CleanThinkingTokens removes model-specific thinking markers from AI responses. // Different AI models use different markers for their internal reasoning: // - DeepSeek: <｜end▁of▁thinking｜> or similar unicode variants @@ -242,7 +254,9 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti } // Phase 2: Build focused seed context from triage results - seedContext, seededFindingIDs := p.buildTriageSeedContextState(triageResult, snap, scope, guestIntel) + seedSections, seededFindingIDs := p.buildTriageSeedSectionsState(triageResult, snap, scope, guestIntel) + seedBudget := p.calculateSeedBudget() + seedContext := p.assembleSeedWithinBudget(seedSections, seedBudget) if strings.TrimSpace(seedContext) == "" { return nil, nil } @@ -302,159 +316,218 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti defer executor.SetPatrolFindingCreator(nil) // Clear after run // Execute the agentic patrol loop - var contentBuffer strings.Builder var inputTokens, outputTokens int + type patrolStreamAttempt struct { + response *PatrolStreamResponse + finalContent string + toolCalls []ToolCallRecord + rawToolOutputs []string + } - // Tool call collection - var toolCallsMu sync.Mutex - pendingToolCalls := make(map[string]ToolCallRecord) - var pendingToolOrder []string - anonToolCounter := 0 - var completedToolCalls []ToolCallRecord - var rawToolOutputs []string + executePatrol := func(prompt string) (*patrolStreamAttempt, error) { + var contentBuffer strings.Builder - chatResp, chatErr := cs.ExecutePatrolStream(ctx, PatrolExecuteRequest{ - Prompt: seedContext, - SystemPrompt: p.getPatrolSystemPromptForTriage(), - SessionID: "patrol-main", - UseCase: "patrol", - MaxTurns: maxTurns, - }, func(event ChatStreamEvent) { - switch event.Type { - case "content": - var contentData struct { - Text string `json:"text"` - } - if json.Unmarshal(event.Data, &contentData) == nil && contentData.Text != "" { - contentBuffer.WriteString(contentData.Text) - if !noStream { - p.appendStreamContent(contentData.Text) + var toolCallsMu sync.Mutex + pendingToolCalls := make(map[string]ToolCallRecord) + var pendingToolOrder []string + anonToolCounter := 0 + var completedToolCalls []ToolCallRecord + var rawToolOutputs []string + + chatResp, chatErr := cs.ExecutePatrolStream(ctx, PatrolExecuteRequest{ + Prompt: prompt, + SystemPrompt: p.getPatrolSystemPromptForTriage(), + SessionID: "patrol-main", + UseCase: "patrol", + MaxTurns: maxTurns, + }, func(event ChatStreamEvent) { + switch event.Type { + case "content": + var contentData struct { + Text string `json:"text"` } - } - case "thinking": - var thinkingData struct { - Text string `json:"text"` - } - if json.Unmarshal(event.Data, &thinkingData) == nil && thinkingData.Text != "" { - if !noStream { - p.broadcast(PatrolStreamEvent{ - Type: "thinking", - Content: thinkingData.Text, - }) + if json.Unmarshal(event.Data, &contentData) == nil && contentData.Text != "" { + contentBuffer.WriteString(contentData.Text) + if !noStream { + p.appendStreamContent(contentData.Text) + } } - } - case "tool_start": - var data struct { - ID string `json:"id"` - Name string `json:"name"` - Input string `json:"input"` - RawInput string `json:"raw_input"` - } - if json.Unmarshal(event.Data, &data) == nil { - if data.ID == "" { - anonToolCounter++ - data.ID = fmt.Sprintf("patrol-anon-%d", anonToolCounter) + case "thinking": + var thinkingData struct { + Text string `json:"text"` } - if !noStream { - p.broadcast(PatrolStreamEvent{ - Type: "tool_start", - ToolID: data.ID, - ToolName: data.Name, - ToolInput: data.Input, - ToolRawInput: data.RawInput, - }) + if json.Unmarshal(event.Data, &thinkingData) == nil && thinkingData.Text != "" { + if !noStream { + p.broadcast(PatrolStreamEvent{ + Type: "thinking", + Content: thinkingData.Text, + }) + } } - input := data.Input - if data.RawInput != "" { - input = data.RawInput + case "tool_start": + var data struct { + ID string `json:"id"` + Name string `json:"name"` + Input string `json:"input"` + RawInput string `json:"raw_input"` } - toolCallsMu.Lock() - pendingToolOrder = append(pendingToolOrder, data.ID) - pendingToolCalls[data.ID] = ToolCallRecord{ - ID: data.ID, - ToolName: data.Name, - Input: truncateString(input, MaxToolInputSize), - StartTime: time.Now().UnixMilli(), - } - toolCallsMu.Unlock() - } - case "tool_end": - var data struct { - ID string `json:"id"` - Name string `json:"name"` - Input string `json:"input"` - RawInput string `json:"raw_input"` - Output string `json:"output"` - Success bool `json:"success"` - } - if json.Unmarshal(event.Data, &data) == nil { - if data.ID == "" { - if len(pendingToolOrder) > 0 { - data.ID = pendingToolOrder[0] - pendingToolOrder = pendingToolOrder[1:] - } else { + if json.Unmarshal(event.Data, &data) == nil { + if data.ID == "" { anonToolCounter++ - data.ID = fmt.Sprintf("patrol-anon-end-%d", anonToolCounter) + data.ID = fmt.Sprintf("patrol-anon-%d", anonToolCounter) } - } else if len(pendingToolOrder) > 0 { - for i, id := range pendingToolOrder { - if id == data.ID { - pendingToolOrder = append(pendingToolOrder[:i], pendingToolOrder[i+1:]...) - break - } + if !noStream { + p.broadcast(PatrolStreamEvent{ + Type: "tool_start", + ToolID: data.ID, + ToolName: data.Name, + ToolInput: data.Input, + ToolRawInput: data.RawInput, + }) } - } - if !noStream { - success := data.Success - p.broadcast(PatrolStreamEvent{ - Type: "tool_end", - ToolID: data.ID, - ToolName: data.Name, - ToolInput: data.Input, - ToolRawInput: data.RawInput, - ToolOutput: data.Output, - ToolSuccess: &success, - }) - } - toolCallsMu.Lock() - if pending, ok := pendingToolCalls[data.ID]; ok { - now := time.Now().UnixMilli() input := data.Input if data.RawInput != "" { input = data.RawInput } - if input != "" { - pending.Input = truncateString(input, MaxToolInputSize) - } - pending.Output = truncateString(data.Output, MaxToolOutputSize) - pending.Success = data.Success - pending.EndTime = now - pending.Duration = now - pending.StartTime - completedToolCalls = append(completedToolCalls, pending) - rawToolOutputs = append(rawToolOutputs, data.Output) - delete(pendingToolCalls, data.ID) - } else { - now := time.Now().UnixMilli() - input := data.Input - if data.RawInput != "" { - input = data.RawInput - } - completedToolCalls = append(completedToolCalls, ToolCallRecord{ + toolCallsMu.Lock() + pendingToolOrder = append(pendingToolOrder, data.ID) + pendingToolCalls[data.ID] = ToolCallRecord{ ID: data.ID, ToolName: data.Name, Input: truncateString(input, MaxToolInputSize), - Output: truncateString(data.Output, MaxToolOutputSize), - Success: data.Success, - StartTime: now, - EndTime: now, - Duration: 0, - }) - rawToolOutputs = append(rawToolOutputs, data.Output) + StartTime: time.Now().UnixMilli(), + } + toolCallsMu.Unlock() } - toolCallsMu.Unlock() + case "tool_end": + var data struct { + ID string `json:"id"` + Name string `json:"name"` + Input string `json:"input"` + RawInput string `json:"raw_input"` + Output string `json:"output"` + Success bool `json:"success"` + } + if json.Unmarshal(event.Data, &data) == nil { + if data.ID == "" { + if len(pendingToolOrder) > 0 { + data.ID = pendingToolOrder[0] + pendingToolOrder = pendingToolOrder[1:] + } else { + anonToolCounter++ + data.ID = fmt.Sprintf("patrol-anon-end-%d", anonToolCounter) + } + } else if len(pendingToolOrder) > 0 { + for i, id := range pendingToolOrder { + if id == data.ID { + pendingToolOrder = append(pendingToolOrder[:i], pendingToolOrder[i+1:]...) + break + } + } + } + if !noStream { + success := data.Success + p.broadcast(PatrolStreamEvent{ + Type: "tool_end", + ToolID: data.ID, + ToolName: data.Name, + ToolInput: data.Input, + ToolRawInput: data.RawInput, + ToolOutput: data.Output, + ToolSuccess: &success, + }) + } + toolCallsMu.Lock() + if pending, ok := pendingToolCalls[data.ID]; ok { + now := time.Now().UnixMilli() + input := data.Input + if data.RawInput != "" { + input = data.RawInput + } + if input != "" { + pending.Input = truncateString(input, MaxToolInputSize) + } + pending.Output = truncateString(data.Output, MaxToolOutputSize) + pending.Success = data.Success + pending.EndTime = now + pending.Duration = now - pending.StartTime + completedToolCalls = append(completedToolCalls, pending) + rawToolOutputs = append(rawToolOutputs, data.Output) + delete(pendingToolCalls, data.ID) + } else { + now := time.Now().UnixMilli() + input := data.Input + if data.RawInput != "" { + input = data.RawInput + } + completedToolCalls = append(completedToolCalls, ToolCallRecord{ + ID: data.ID, + ToolName: data.Name, + Input: truncateString(input, MaxToolInputSize), + Output: truncateString(data.Output, MaxToolOutputSize), + Success: data.Success, + StartTime: now, + EndTime: now, + Duration: 0, + }) + rawToolOutputs = append(rawToolOutputs, data.Output) + } + toolCallsMu.Unlock() + } + } + }) + if chatErr != nil { + return nil, chatErr + } + + finalContent := chatResp.Content + if finalContent == "" { + finalContent = contentBuffer.String() + } + + toolCallsMu.Lock() + collectedToolCalls := append([]ToolCallRecord(nil), completedToolCalls...) + collectedRawOutputs := append([]string(nil), rawToolOutputs...) + toolCallsMu.Unlock() + + return &patrolStreamAttempt{ + response: chatResp, + finalContent: finalContent, + toolCalls: collectedToolCalls, + rawToolOutputs: collectedRawOutputs, + }, nil + } + + attempt, chatErr := executePatrol(seedContext) + if chatErr != nil && isPatrolContextWindowError(chatErr) { + for _, retryBudget := range patrolSeedRetryBudgets(chatErr) { + if retryBudget >= seedBudget { + continue + } + + retrySeedContext := p.assembleSeedWithinBudget(seedSections, retryBudget) + if strings.TrimSpace(retrySeedContext) == "" || retrySeedContext == seedContext { + continue + } + + log.Warn(). + Int("previous_seed_budget_tokens", seedBudget). + Int("retry_seed_budget_tokens", retryBudget). + Int("previous_seed_tokens", chat.EstimateTokens(seedContext)). + Int("retry_seed_tokens", chat.EstimateTokens(retrySeedContext)). + Msg("AI Patrol: Retrying patrol analysis with tighter provider-derived seed budget") + + seedBudget = retryBudget + seedContext = retrySeedContext + attempt, chatErr = executePatrol(seedContext) + if chatErr == nil { + break + } + if !isPatrolContextWindowError(chatErr) { + break } } - }) + } if chatErr != nil { if !noStream { @@ -464,13 +537,10 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti return nil, fmt.Errorf("agentic patrol failed: %w", chatErr) } - finalContent := chatResp.Content - if finalContent == "" { - finalContent = contentBuffer.String() - } - inputTokens = chatResp.InputTokens - outputTokens = chatResp.OutputTokens - p.recordPatrolUsage(chatResp.InputTokens, chatResp.OutputTokens) + finalContent := attempt.finalContent + inputTokens = attempt.response.InputTokens + outputTokens = attempt.response.OutputTokens + p.recordPatrolUsage(attempt.response.InputTokens, attempt.response.OutputTokens) // Clean thinking tokens finalContent = CleanThinkingTokens(finalContent) @@ -482,6 +552,9 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti Int("findings_resolved", adapter.getResolvedCount()). Msg("AI Patrol: Agentic patrol analysis complete") + var toolCallsMu sync.Mutex + completedToolCalls := append([]ToolCallRecord(nil), attempt.toolCalls...) + rawToolOutputs := append([]string(nil), attempt.rawToolOutputs...) p.ensureInvestigationToolCall(ctx, executor, &toolCallsMu, &completedToolCalls, &rawToolOutputs, noStream) // Broadcast completion @@ -1162,6 +1235,16 @@ func (p *PatrolService) buildTriageSeedContextState( scope *PatrolScope, guestIntel map[string]*GuestIntelligence, ) (string, []string) { + sections, seededFindingIDs := p.buildTriageSeedSectionsState(triage, snap, scope, guestIntel) + return p.assembleSeedWithinBudget(sections, p.calculateSeedBudget()), seededFindingIDs +} + +func (p *PatrolService) buildTriageSeedSectionsState( + triage *TriageResult, + snap patrolRuntimeState, + scope *PatrolScope, + guestIntel map[string]*GuestIntelligence, +) ([]seedSection, []string) { p.mu.RLock() cfg := p.config p.mu.RUnlock() @@ -1179,24 +1262,31 @@ func (p *PatrolService) buildTriageSeedContextState( sections := []seedSection{ // P0 — always include. - {priority: 0, name: "triage_briefing", content: FormatTriageBriefing(triage)}, + {priority: 0, name: "triage_overview", content: formatTriageOverviewSection(triage)}, {priority: 0, name: "findings", content: findingsCtx}, {priority: 0, name: "health_alerts", content: p.seedHealthAndAlertsState(snap, flaggedSet, cfg, now)}, {priority: 0, name: "scope", content: buildScopeSection(scope, sortedScopedIDs(flaggedSet))}, - // P1 — flagged resource details only. + // P2 — triage already preserves the flagged set, so these sections can + // summarize under tighter provider-derived retry budgets. { - priority: 1, + priority: 2, + name: "triage_flags", + content: formatTriageFlagsSection(triage), + summary: formatTriageFlagsSummary(triage), + }, + { + priority: 2, name: "flagged_inventory", content: p.seedResourceInventoryState(snap, flaggedSet, cfg, now, false, guestIntel), summary: p.seedResourceInventorySummaryState(snap, flaggedSet, cfg, now, guestIntel), }, + + // P3 — healthy rollup is useful context but lowest-value on retries. + {priority: 3, name: "triage_healthy", content: formatTriageHealthySummarySection(triage)}, } - budget := p.calculateSeedBudget() - result := p.assembleSeedWithinBudget(sections, budget) - - return result, seededFindingIDs + return sections, seededFindingIDs } // buildSeedContext produces the infrastructure state context for the agentic patrol loop. @@ -1204,6 +1294,11 @@ func (p *PatrolService) buildTriageSeedContextState( // connection health, and baselines/trends so the model can analyze without tool calls. // Tools remain available for targeted deep-dives. func (p *PatrolService) buildSeedContextState(snap patrolRuntimeState, scope *PatrolScope, guestIntel map[string]*GuestIntelligence) (string, []string) { + sections, seededFindingIDs := p.buildSeedSectionsState(snap, scope, guestIntel) + return p.assembleSeedWithinBudget(sections, p.calculateSeedBudget()), seededFindingIDs +} + +func (p *PatrolService) buildSeedSectionsState(snap patrolRuntimeState, scope *PatrolScope, guestIntel map[string]*GuestIntelligence) ([]seedSection, []string) { p.mu.RLock() cfg := p.config p.mu.RUnlock() @@ -1239,19 +1334,12 @@ func (p *PatrolService) buildSeedContextState(snap patrolRuntimeState, scope *Pa {priority: 4, name: "pmg_snapshot", content: p.seedPMGSnapshotStringState(snap, scopedSet, cfg, intel.isQuiet)}, } - budget := p.calculateSeedBudget() - result := p.assembleSeedWithinBudget(sections, budget) - - return result, seededFindingIDs + return sections, seededFindingIDs } func (p *PatrolService) calculateSeedBudget() int { const ( - systemPromptEstimate = 4_000 - toolEstimate = 8_000 - outputReserve = 8_000 - historyReserve = 16_000 - minimumSeedBudget = 16_000 + defaultContextWindow = 128_000 ) model := "" @@ -1262,6 +1350,34 @@ func (p *PatrolService) calculateSeedBudget() int { } contextWindow := providers.ContextWindowTokens(model) + if contextWindow <= 0 { + contextWindow = defaultContextWindow + } + budget := calculateSeedBudgetForContextWindow(contextWindow) + + log.Debug(). + Str("model", model). + Int("context_window_tokens", contextWindow). + Int("seed_budget_tokens", budget). + Msg("AI Patrol: Calculated seed context token budget") + + return budget +} + +func calculateSeedBudgetForContextWindow(contextWindow int) int { + const ( + systemPromptEstimate = 4_000 + toolEstimate = 8_000 + outputReserve = 8_000 + historyReserve = 16_000 + minimumSeedBudget = 16_000 + defaultContextWindow = 128_000 + ) + + if contextWindow <= 0 { + contextWindow = defaultContextWindow + } + budget := contextWindow - systemPromptEstimate - toolEstimate - outputReserve - historyReserve // Clamp floor so small-context models aren't forced beyond practical capacity. @@ -1273,15 +1389,91 @@ func (p *PatrolService) calculateSeedBudget() int { budget = floor } - log.Debug(). - Str("model", model). - Int("context_window_tokens", contextWindow). - Int("seed_budget_tokens", budget). - Msg("AI Patrol: Calculated seed context token budget") - return budget } +func isPatrolContextWindowError(err error) bool { + if err == nil { + return false + } + + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "exceed_context_size_error") || + strings.Contains(msg, "exceeds the available context size") || + strings.Contains(msg, "maximum context length") || + strings.Contains(msg, "context window") || + strings.Contains(msg, "context length") || + strings.Contains(msg, "n_ctx") +} + +func patrolSeedRetryBudgets(err error) []int { + if err == nil { + return []int{patrolRetrySeedBudget1, patrolRetrySeedBudget2, patrolRetrySeedBudget3} + } + + contextWindow := extractPatrolContextWindow(err) + if contextWindow <= 0 { + return []int{patrolRetrySeedBudget1, patrolRetrySeedBudget2, patrolRetrySeedBudget3} + } + + safeBudget := calculateSeedBudgetForContextWindow(contextWindow) + return uniquePositiveInts( + safeBudget, + maxInt(1_000, safeBudget/2), + maxInt(1_000, safeBudget/4), + ) +} + +func extractPatrolContextWindow(err error) int { + if err == nil { + return 0 + } + + msg := err.Error() + for _, pattern := range patrolContextWindowPatterns { + matches := pattern.FindStringSubmatch(msg) + if len(matches) < 2 { + continue + } + nctx, convErr := strconv.Atoi(matches[1]) + if convErr == nil && nctx > 0 { + return nctx + } + } + + return 0 +} + +func uniquePositiveInts(values ...int) []int { + result := make([]int, 0, len(values)) + seen := make(map[int]struct{}, len(values)) + for _, value := range values { + if value <= 0 { + continue + } + if _, exists := seen[value]; exists { + continue + } + seen[value] = struct{}{} + result = append(result, value) + } + return result +} + +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + func (p *PatrolService) assembleSeedWithinBudget(sections []seedSection, budgetTokens int) string { if len(sections) == 0 { return "" diff --git a/internal/ai/patrol_ai_more_test.go b/internal/ai/patrol_ai_more_test.go index 689d014fb..96abb0f8a 100644 --- a/internal/ai/patrol_ai_more_test.go +++ b/internal/ai/patrol_ai_more_test.go @@ -3,12 +3,15 @@ package ai import ( "context" "encoding/json" + "fmt" + "reflect" "strings" "sync" "testing" "time" "github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline" + "github.com/rcourtman/pulse-go-rewrite/internal/ai/chat" "github.com/rcourtman/pulse-go-rewrite/internal/ai/cost" "github.com/rcourtman/pulse-go-rewrite/internal/ai/memory" "github.com/rcourtman/pulse-go-rewrite/internal/ai/tools" @@ -262,6 +265,89 @@ func TestRunAIAnalysis_EarlyErrors(t *testing.T) { }) } +func TestPatrolSeedRetryBudgets_UsesProviderContextWindow(t *testing.T) { + err := fmt.Errorf("API error (400): {\"error\":{\"message\":\"request exceeds the available context size (8192 tokens)\",\"type\":\"exceed_context_size_error\",\"n_ctx\":8192}}") + + got := patrolSeedRetryBudgets(err) + want := []int{4096, 2048, 1024} + if !reflect.DeepEqual(got, want) { + t.Fatalf("patrolSeedRetryBudgets() = %v, want %v", got, want) + } +} + +func TestRunAIAnalysis_RetriesWithProviderDerivedSeedBudget(t *testing.T) { + persistence := config.NewConfigPersistence(t.TempDir()) + svc := NewService(persistence, nil) + svc.cfg = &config.AIConfig{Enabled: true, PatrolModel: "mock:model"} + svc.provider = &mockProvider{} + + executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{}) + promptTokens := make([]int, 0, 2) + mockCS := &mockChatService{ + executor: executor, + executePatrolStreamFunc: func(ctx context.Context, req PatrolExecuteRequest, callback ChatStreamCallback) (*PatrolStreamResponse, error) { + promptTokens = append(promptTokens, chat.EstimateTokens(req.Prompt)) + if len(promptTokens) == 1 { + return nil, fmt.Errorf("API error (400): {\"error\":{\"message\":\"request exceeds the available context size (4096 tokens)\",\"type\":\"exceed_context_size_error\",\"n_ctx\":4096}}") + } + return &PatrolStreamResponse{ + Content: "analysis complete", + InputTokens: 8, + OutputTokens: 4, + }, nil + }, + } + svc.SetChatService(mockCS) + + ps := NewPatrolService(svc, nil) + ps.SetConfig(PatrolConfig{ + Enabled: true, + AnalyzeNodes: true, + AnalyzeGuests: true, + }) + + state := models.StateSnapshot{ + Nodes: []models.Node{ + {ID: "node-1", Name: "pve-1", Status: "online", CPU: 0.20, Memory: models.Memory{Usage: 30.0}}, + }, + } + for i := 0; i < 600; i++ { + state.VMs = append(state.VMs, models.VM{ + ID: fmt.Sprintf("vm-%d", i), + VMID: 100 + i, + Name: fmt.Sprintf("flagged-guest-%03d-%s", i, strings.Repeat("x", 96)), + Node: "pve-1", + Status: "running", + CPU: 0.95, + Memory: models.Memory{Usage: 91.0}, + Disk: models.Disk{Usage: 15.0}, + }) + } + + triage := ps.runDeterministicTriageState(context.Background(), patrolRuntimeStateForTest(ps, state), nil, nil) + seed, _ := ps.buildTriageSeedContextState(triage, patrolRuntimeStateForTest(ps, state), nil, nil) + if got := chat.EstimateTokens(seed); got <= 2048 { + t.Fatalf("expected initial triage seed to exceed retry budget, got %d tokens", got) + } + + result, err := ps.runAIAnalysisState(context.Background(), patrolRuntimeStateForTest(ps, state), &PatrolScope{NoStream: true}) + if err != nil { + t.Fatalf("runAIAnalysisState returned error: %v", err) + } + if result == nil || result.Response != "analysis complete" { + t.Fatalf("unexpected analysis result: %#v", result) + } + if len(promptTokens) != 2 { + t.Fatalf("expected 2 patrol attempts, got %d", len(promptTokens)) + } + if promptTokens[1] >= promptTokens[0] { + t.Fatalf("expected retry prompt to shrink, got %d then %d tokens", promptTokens[0], promptTokens[1]) + } + if promptTokens[1] > 2048 { + t.Fatalf("expected retry prompt to honor provider-derived budget, got %d tokens", promptTokens[1]) + } +} + func TestSeedResourceInventory_QuietSummary(t *testing.T) { ps := NewPatrolService(nil, nil) cfg := DefaultPatrolConfig() diff --git a/internal/ai/patrol_triage.go b/internal/ai/patrol_triage.go index 904025535..0cb98a5f4 100644 --- a/internal/ai/patrol_triage.go +++ b/internal/ai/patrol_triage.go @@ -453,12 +453,23 @@ func FormatTriageBriefing(triage *TriageResult) string { } var sb strings.Builder - sb.WriteString("# Deterministic Triage Results\n") + sb.WriteString(formatTriageOverviewSection(triage)) + sb.WriteString(formatTriageFlagsSection(triage)) + sb.WriteString(formatTriageHealthySummarySection(triage)) + + return sb.String() +} + +func formatTriageOverviewSection(triage *TriageResult) string { + if triage == nil { + return "# Deterministic Triage Results\nNo triage data available.\n" + } scanned := triage.Summary.TotalNodes + triage.Summary.TotalGuests + triage.Summary.TotalStorage + triage.Summary.TotalDocker + triage.Summary.TotalTrueNAS + triage.Summary.TotalPBS + triage.Summary.TotalPMG - sb.WriteString(fmt.Sprintf( - "Scanned %d resources: %d nodes, %d guests, %d storage resources (%d pools, %d physical disks), %d docker hosts, %d TrueNAS systems, %d PBS, %d PMG.\n\n", + + return fmt.Sprintf( + "# Deterministic Triage Results\nScanned %d resources: %d nodes, %d guests, %d storage resources (%d pools, %d physical disks), %d docker hosts, %d TrueNAS systems, %d PBS, %d PMG.\n\n", scanned, triage.Summary.TotalNodes, triage.Summary.TotalGuests, @@ -469,36 +480,73 @@ func FormatTriageBriefing(triage *TriageResult) string { triage.Summary.TotalTrueNAS, triage.Summary.TotalPBS, triage.Summary.TotalPMG, - )) + ) +} - if len(triage.Flags) > 0 { - flags := append([]TriageFlag(nil), triage.Flags...) - sort.Slice(flags, func(i, j int) bool { - ri := triageSeverityRank(flags[i].Severity) - rj := triageSeverityRank(flags[j].Severity) - if ri != rj { - return ri > rj - } - if flags[i].ResourceType != flags[j].ResourceType { - return flags[i].ResourceType < flags[j].ResourceType - } - return flags[i].ResourceName < flags[j].ResourceName - }) +func formatTriageFlagsSection(triage *TriageResult) string { + flags := sortedTriageFlags(triage) + if len(flags) == 0 { + return "" + } - sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags))) - sb.WriteString("| Resource | Type | Flag | Severity | Detail |\n") - sb.WriteString("|----------|------|------|----------|--------|\n") - for _, flag := range flags { - resource := triageResourceName(flag.ResourceName, flag.ResourceID) - sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n", - triageTableEscape(resource), - triageTableEscape(flag.ResourceType), - triageTableEscape(triageFlagLabel(flag)), - triageTableEscape(flag.Severity), - triageTableEscape(flag.Reason), - )) + var sb strings.Builder + sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags))) + sb.WriteString("| Resource | Type | Flag | Severity | Detail |\n") + sb.WriteString("|----------|------|------|----------|--------|\n") + for _, flag := range flags { + resource := triageResourceName(flag.ResourceName, flag.ResourceID) + sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n", + triageTableEscape(resource), + triageTableEscape(flag.ResourceType), + triageTableEscape(triageFlagLabel(flag)), + triageTableEscape(flag.Severity), + triageTableEscape(flag.Reason), + )) + } + sb.WriteString("\n") + return sb.String() +} + +func formatTriageFlagsSummary(triage *TriageResult) string { + flags := sortedTriageFlags(triage) + if len(flags) == 0 { + return "" + } + + critical := 0 + warning := 0 + watch := 0 + entries := make([]string, 0, triageMinInt(len(flags), 8)) + for i, flag := range flags { + switch strings.ToLower(flag.Severity) { + case "critical": + critical++ + case "warning": + warning++ + case "watch": + watch++ } - sb.WriteString("\n") + if i < 8 { + resource := triageResourceName(flag.ResourceName, flag.ResourceID) + entries = append(entries, fmt.Sprintf("%s (%s, %s)", resource, triageFlagLabel(flag), flag.Severity)) + } + } + + var sb strings.Builder + sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags))) + sb.WriteString(fmt.Sprintf("Severity mix: critical=%d, warning=%d, watch=%d.\n", critical, warning, watch)) + sb.WriteString("Prioritize: ") + sb.WriteString(strings.Join(entries, "; ")) + if len(flags) > len(entries) { + sb.WriteString(fmt.Sprintf("; +%d more flagged resources", len(flags)-len(entries))) + } + sb.WriteString(".\n\n") + return sb.String() +} + +func formatTriageHealthySummarySection(triage *TriageResult) string { + if triage == nil { + return "" } healthyGuests := triage.Summary.TotalGuests - triageUniqueFlaggedByTypes(triage.Flags, "vm", "system-container") @@ -532,6 +580,8 @@ func FormatTriageBriefing(triage *TriageResult) string { } totalHealthy := healthyNodes + healthyGuests + healthyStorage + healthyDocker + healthyTrueNAS + healthyPBS + healthyPMG + + var sb strings.Builder sb.WriteString(fmt.Sprintf("## Healthy Resources (%d)\n", totalHealthy)) sb.WriteString(fmt.Sprintf("Nodes: %d healthy\n", healthyNodes)) sb.WriteString(fmt.Sprintf("Guests: %d running, %d stopped\n", triage.Summary.RunningGuests, triage.Summary.StoppedGuests)) @@ -543,10 +593,36 @@ func FormatTriageBriefing(triage *TriageResult) string { sb.WriteString(fmt.Sprintf("TrueNAS: %d systems\n", healthyTrueNAS)) sb.WriteString(fmt.Sprintf("PBS: %d instances\n", healthyPBS)) sb.WriteString(fmt.Sprintf("PMG: %d instances\n", healthyPMG)) - return sb.String() } +func triageMinInt(a, b int) int { + if a < b { + return a + } + return b +} + +func sortedTriageFlags(triage *TriageResult) []TriageFlag { + if triage == nil || len(triage.Flags) == 0 { + return nil + } + + flags := append([]TriageFlag(nil), triage.Flags...) + sort.Slice(flags, func(i, j int) bool { + ri := triageSeverityRank(flags[i].Severity) + rj := triageSeverityRank(flags[j].Severity) + if ri != rj { + return ri > rj + } + if flags[i].ResourceType != flags[j].ResourceType { + return flags[i].ResourceType < flags[j].ResourceType + } + return flags[i].ResourceName < flags[j].ResourceName + }) + return flags +} + func triageBuildSummaryState(snap patrolRuntimeState, flaggedIDs map[string]bool) TriageSummary { counts := patrolRuntimeCountResources(snap) storagePools := patrolStoragePoolRows(snap, nil)