mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-07-09 16:00:59 +00:00
Retry Patrol with provider-derived seed budgets
This commit is contained in:
rcourtman
parent
e86494668a
commit
87aafb86c4
4 changed files with 553 additions and 193 deletions
|
|
@ -87,6 +87,12 @@ governed floor is ready.
|
|||
`internal/ai/` is the live backend AI engine. It owns chat execution, Patrol
|
||||
orchestration, findings generation, investigation support, quickstart and
|
||||
provider selection, remediation flow, and cost persistence.
|
||||
That Patrol runtime ownership includes seed-context admission control.
|
||||
`internal/ai/patrol_ai.go` must build Patrol and triage prompts from
|
||||
canonical seed sections, size them against the runtime budget model, and when
|
||||
a provider reports a smaller real context window than the static model map,
|
||||
reassemble the same canonical sections under tighter provider-derived budgets
|
||||
instead of hard-failing or truncating ad hoc prompt strings.
|
||||
That same backend runtime ownership also includes bounded Patrol and
|
||||
investigation read models. `internal/ai/patrol_history_persistence.go` and
|
||||
`internal/ai/proxmox/events.go` must cap persisted-history loads and
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
|
@ -46,8 +47,19 @@ const (
|
|||
patrolTurnsPer50Devices = 5
|
||||
patrolQuickMinTurns = 10
|
||||
patrolQuickMaxTurns = 30
|
||||
patrolRetrySeedBudget1 = 16_000
|
||||
patrolRetrySeedBudget2 = 8_000
|
||||
patrolRetrySeedBudget3 = 4_000
|
||||
)
|
||||
|
||||
var patrolContextWindowPatterns = []*regexp.Regexp{
|
||||
regexp.MustCompile(`(?i)"n_ctx"\s*:\s*(\d+)`),
|
||||
regexp.MustCompile(`(?i)available context size\s*\((\d+)\s*tokens?\)`),
|
||||
regexp.MustCompile(`(?i)maximum context length[^0-9]*(\d+)`),
|
||||
regexp.MustCompile(`(?i)context window[^0-9]*(\d+)`),
|
||||
regexp.MustCompile(`(?i)context length[^0-9]*(\d+)`),
|
||||
}
|
||||
|
||||
// CleanThinkingTokens removes model-specific thinking markers from AI responses.
|
||||
// Different AI models use different markers for their internal reasoning:
|
||||
// - DeepSeek: <｜end▁of▁thinking｜> or similar unicode variants
|
||||
|
|
@ -242,7 +254,9 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti
|
|||
}
|
||||
|
||||
// Phase 2: Build focused seed context from triage results
|
||||
seedContext, seededFindingIDs := p.buildTriageSeedContextState(triageResult, snap, scope, guestIntel)
|
||||
seedSections, seededFindingIDs := p.buildTriageSeedSectionsState(triageResult, snap, scope, guestIntel)
|
||||
seedBudget := p.calculateSeedBudget()
|
||||
seedContext := p.assembleSeedWithinBudget(seedSections, seedBudget)
|
||||
if strings.TrimSpace(seedContext) == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
@ -302,159 +316,218 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti
|
|||
defer executor.SetPatrolFindingCreator(nil) // Clear after run
|
||||
|
||||
// Execute the agentic patrol loop
|
||||
var contentBuffer strings.Builder
|
||||
var inputTokens, outputTokens int
|
||||
type patrolStreamAttempt struct {
|
||||
response *PatrolStreamResponse
|
||||
finalContent string
|
||||
toolCalls []ToolCallRecord
|
||||
rawToolOutputs []string
|
||||
}
|
||||
|
||||
// Tool call collection
|
||||
var toolCallsMu sync.Mutex
|
||||
pendingToolCalls := make(map[string]ToolCallRecord)
|
||||
var pendingToolOrder []string
|
||||
anonToolCounter := 0
|
||||
var completedToolCalls []ToolCallRecord
|
||||
var rawToolOutputs []string
|
||||
executePatrol := func(prompt string) (*patrolStreamAttempt, error) {
|
||||
var contentBuffer strings.Builder
|
||||
|
||||
chatResp, chatErr := cs.ExecutePatrolStream(ctx, PatrolExecuteRequest{
|
||||
Prompt: seedContext,
|
||||
SystemPrompt: p.getPatrolSystemPromptForTriage(),
|
||||
SessionID: "patrol-main",
|
||||
UseCase: "patrol",
|
||||
MaxTurns: maxTurns,
|
||||
}, func(event ChatStreamEvent) {
|
||||
switch event.Type {
|
||||
case "content":
|
||||
var contentData struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if json.Unmarshal(event.Data, &contentData) == nil && contentData.Text != "" {
|
||||
contentBuffer.WriteString(contentData.Text)
|
||||
if !noStream {
|
||||
p.appendStreamContent(contentData.Text)
|
||||
var toolCallsMu sync.Mutex
|
||||
pendingToolCalls := make(map[string]ToolCallRecord)
|
||||
var pendingToolOrder []string
|
||||
anonToolCounter := 0
|
||||
var completedToolCalls []ToolCallRecord
|
||||
var rawToolOutputs []string
|
||||
|
||||
chatResp, chatErr := cs.ExecutePatrolStream(ctx, PatrolExecuteRequest{
|
||||
Prompt: prompt,
|
||||
SystemPrompt: p.getPatrolSystemPromptForTriage(),
|
||||
SessionID: "patrol-main",
|
||||
UseCase: "patrol",
|
||||
MaxTurns: maxTurns,
|
||||
}, func(event ChatStreamEvent) {
|
||||
switch event.Type {
|
||||
case "content":
|
||||
var contentData struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
}
|
||||
case "thinking":
|
||||
var thinkingData struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if json.Unmarshal(event.Data, &thinkingData) == nil && thinkingData.Text != "" {
|
||||
if !noStream {
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "thinking",
|
||||
Content: thinkingData.Text,
|
||||
})
|
||||
if json.Unmarshal(event.Data, &contentData) == nil && contentData.Text != "" {
|
||||
contentBuffer.WriteString(contentData.Text)
|
||||
if !noStream {
|
||||
p.appendStreamContent(contentData.Text)
|
||||
}
|
||||
}
|
||||
}
|
||||
case "tool_start":
|
||||
var data struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input string `json:"input"`
|
||||
RawInput string `json:"raw_input"`
|
||||
}
|
||||
if json.Unmarshal(event.Data, &data) == nil {
|
||||
if data.ID == "" {
|
||||
anonToolCounter++
|
||||
data.ID = fmt.Sprintf("patrol-anon-%d", anonToolCounter)
|
||||
case "thinking":
|
||||
var thinkingData struct {
|
||||
Text string `json:"text"`
|
||||
}
|
||||
if !noStream {
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "tool_start",
|
||||
ToolID: data.ID,
|
||||
ToolName: data.Name,
|
||||
ToolInput: data.Input,
|
||||
ToolRawInput: data.RawInput,
|
||||
})
|
||||
if json.Unmarshal(event.Data, &thinkingData) == nil && thinkingData.Text != "" {
|
||||
if !noStream {
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "thinking",
|
||||
Content: thinkingData.Text,
|
||||
})
|
||||
}
|
||||
}
|
||||
input := data.Input
|
||||
if data.RawInput != "" {
|
||||
input = data.RawInput
|
||||
case "tool_start":
|
||||
var data struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input string `json:"input"`
|
||||
RawInput string `json:"raw_input"`
|
||||
}
|
||||
toolCallsMu.Lock()
|
||||
pendingToolOrder = append(pendingToolOrder, data.ID)
|
||||
pendingToolCalls[data.ID] = ToolCallRecord{
|
||||
ID: data.ID,
|
||||
ToolName: data.Name,
|
||||
Input: truncateString(input, MaxToolInputSize),
|
||||
StartTime: time.Now().UnixMilli(),
|
||||
}
|
||||
toolCallsMu.Unlock()
|
||||
}
|
||||
case "tool_end":
|
||||
var data struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input string `json:"input"`
|
||||
RawInput string `json:"raw_input"`
|
||||
Output string `json:"output"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
if json.Unmarshal(event.Data, &data) == nil {
|
||||
if data.ID == "" {
|
||||
if len(pendingToolOrder) > 0 {
|
||||
data.ID = pendingToolOrder[0]
|
||||
pendingToolOrder = pendingToolOrder[1:]
|
||||
} else {
|
||||
if json.Unmarshal(event.Data, &data) == nil {
|
||||
if data.ID == "" {
|
||||
anonToolCounter++
|
||||
data.ID = fmt.Sprintf("patrol-anon-end-%d", anonToolCounter)
|
||||
data.ID = fmt.Sprintf("patrol-anon-%d", anonToolCounter)
|
||||
}
|
||||
} else if len(pendingToolOrder) > 0 {
|
||||
for i, id := range pendingToolOrder {
|
||||
if id == data.ID {
|
||||
pendingToolOrder = append(pendingToolOrder[:i], pendingToolOrder[i+1:]...)
|
||||
break
|
||||
}
|
||||
if !noStream {
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "tool_start",
|
||||
ToolID: data.ID,
|
||||
ToolName: data.Name,
|
||||
ToolInput: data.Input,
|
||||
ToolRawInput: data.RawInput,
|
||||
})
|
||||
}
|
||||
}
|
||||
if !noStream {
|
||||
success := data.Success
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "tool_end",
|
||||
ToolID: data.ID,
|
||||
ToolName: data.Name,
|
||||
ToolInput: data.Input,
|
||||
ToolRawInput: data.RawInput,
|
||||
ToolOutput: data.Output,
|
||||
ToolSuccess: &success,
|
||||
})
|
||||
}
|
||||
toolCallsMu.Lock()
|
||||
if pending, ok := pendingToolCalls[data.ID]; ok {
|
||||
now := time.Now().UnixMilli()
|
||||
input := data.Input
|
||||
if data.RawInput != "" {
|
||||
input = data.RawInput
|
||||
}
|
||||
if input != "" {
|
||||
pending.Input = truncateString(input, MaxToolInputSize)
|
||||
}
|
||||
pending.Output = truncateString(data.Output, MaxToolOutputSize)
|
||||
pending.Success = data.Success
|
||||
pending.EndTime = now
|
||||
pending.Duration = now - pending.StartTime
|
||||
completedToolCalls = append(completedToolCalls, pending)
|
||||
rawToolOutputs = append(rawToolOutputs, data.Output)
|
||||
delete(pendingToolCalls, data.ID)
|
||||
} else {
|
||||
now := time.Now().UnixMilli()
|
||||
input := data.Input
|
||||
if data.RawInput != "" {
|
||||
input = data.RawInput
|
||||
}
|
||||
completedToolCalls = append(completedToolCalls, ToolCallRecord{
|
||||
toolCallsMu.Lock()
|
||||
pendingToolOrder = append(pendingToolOrder, data.ID)
|
||||
pendingToolCalls[data.ID] = ToolCallRecord{
|
||||
ID: data.ID,
|
||||
ToolName: data.Name,
|
||||
Input: truncateString(input, MaxToolInputSize),
|
||||
Output: truncateString(data.Output, MaxToolOutputSize),
|
||||
Success: data.Success,
|
||||
StartTime: now,
|
||||
EndTime: now,
|
||||
Duration: 0,
|
||||
})
|
||||
rawToolOutputs = append(rawToolOutputs, data.Output)
|
||||
StartTime: time.Now().UnixMilli(),
|
||||
}
|
||||
toolCallsMu.Unlock()
|
||||
}
|
||||
toolCallsMu.Unlock()
|
||||
case "tool_end":
|
||||
var data struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Input string `json:"input"`
|
||||
RawInput string `json:"raw_input"`
|
||||
Output string `json:"output"`
|
||||
Success bool `json:"success"`
|
||||
}
|
||||
if json.Unmarshal(event.Data, &data) == nil {
|
||||
if data.ID == "" {
|
||||
if len(pendingToolOrder) > 0 {
|
||||
data.ID = pendingToolOrder[0]
|
||||
pendingToolOrder = pendingToolOrder[1:]
|
||||
} else {
|
||||
anonToolCounter++
|
||||
data.ID = fmt.Sprintf("patrol-anon-end-%d", anonToolCounter)
|
||||
}
|
||||
} else if len(pendingToolOrder) > 0 {
|
||||
for i, id := range pendingToolOrder {
|
||||
if id == data.ID {
|
||||
pendingToolOrder = append(pendingToolOrder[:i], pendingToolOrder[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !noStream {
|
||||
success := data.Success
|
||||
p.broadcast(PatrolStreamEvent{
|
||||
Type: "tool_end",
|
||||
ToolID: data.ID,
|
||||
ToolName: data.Name,
|
||||
ToolInput: data.Input,
|
||||
ToolRawInput: data.RawInput,
|
||||
ToolOutput: data.Output,
|
||||
ToolSuccess: &success,
|
||||
})
|
||||
}
|
||||
toolCallsMu.Lock()
|
||||
if pending, ok := pendingToolCalls[data.ID]; ok {
|
||||
now := time.Now().UnixMilli()
|
||||
input := data.Input
|
||||
if data.RawInput != "" {
|
||||
input = data.RawInput
|
||||
}
|
||||
if input != "" {
|
||||
pending.Input = truncateString(input, MaxToolInputSize)
|
||||
}
|
||||
pending.Output = truncateString(data.Output, MaxToolOutputSize)
|
||||
pending.Success = data.Success
|
||||
pending.EndTime = now
|
||||
pending.Duration = now - pending.StartTime
|
||||
completedToolCalls = append(completedToolCalls, pending)
|
||||
rawToolOutputs = append(rawToolOutputs, data.Output)
|
||||
delete(pendingToolCalls, data.ID)
|
||||
} else {
|
||||
now := time.Now().UnixMilli()
|
||||
input := data.Input
|
||||
if data.RawInput != "" {
|
||||
input = data.RawInput
|
||||
}
|
||||
completedToolCalls = append(completedToolCalls, ToolCallRecord{
|
||||
ID: data.ID,
|
||||
ToolName: data.Name,
|
||||
Input: truncateString(input, MaxToolInputSize),
|
||||
Output: truncateString(data.Output, MaxToolOutputSize),
|
||||
Success: data.Success,
|
||||
StartTime: now,
|
||||
EndTime: now,
|
||||
Duration: 0,
|
||||
})
|
||||
rawToolOutputs = append(rawToolOutputs, data.Output)
|
||||
}
|
||||
toolCallsMu.Unlock()
|
||||
}
|
||||
}
|
||||
})
|
||||
if chatErr != nil {
|
||||
return nil, chatErr
|
||||
}
|
||||
|
||||
finalContent := chatResp.Content
|
||||
if finalContent == "" {
|
||||
finalContent = contentBuffer.String()
|
||||
}
|
||||
|
||||
toolCallsMu.Lock()
|
||||
collectedToolCalls := append([]ToolCallRecord(nil), completedToolCalls...)
|
||||
collectedRawOutputs := append([]string(nil), rawToolOutputs...)
|
||||
toolCallsMu.Unlock()
|
||||
|
||||
return &patrolStreamAttempt{
|
||||
response: chatResp,
|
||||
finalContent: finalContent,
|
||||
toolCalls: collectedToolCalls,
|
||||
rawToolOutputs: collectedRawOutputs,
|
||||
}, nil
|
||||
}
|
||||
|
||||
attempt, chatErr := executePatrol(seedContext)
|
||||
if chatErr != nil && isPatrolContextWindowError(chatErr) {
|
||||
for _, retryBudget := range patrolSeedRetryBudgets(chatErr) {
|
||||
if retryBudget >= seedBudget {
|
||||
continue
|
||||
}
|
||||
|
||||
retrySeedContext := p.assembleSeedWithinBudget(seedSections, retryBudget)
|
||||
if strings.TrimSpace(retrySeedContext) == "" || retrySeedContext == seedContext {
|
||||
continue
|
||||
}
|
||||
|
||||
log.Warn().
|
||||
Int("previous_seed_budget_tokens", seedBudget).
|
||||
Int("retry_seed_budget_tokens", retryBudget).
|
||||
Int("previous_seed_tokens", chat.EstimateTokens(seedContext)).
|
||||
Int("retry_seed_tokens", chat.EstimateTokens(retrySeedContext)).
|
||||
Msg("AI Patrol: Retrying patrol analysis with tighter provider-derived seed budget")
|
||||
|
||||
seedBudget = retryBudget
|
||||
seedContext = retrySeedContext
|
||||
attempt, chatErr = executePatrol(seedContext)
|
||||
if chatErr == nil {
|
||||
break
|
||||
}
|
||||
if !isPatrolContextWindowError(chatErr) {
|
||||
break
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
if chatErr != nil {
|
||||
if !noStream {
|
||||
|
|
@ -464,13 +537,10 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti
|
|||
return nil, fmt.Errorf("agentic patrol failed: %w", chatErr)
|
||||
}
|
||||
|
||||
finalContent := chatResp.Content
|
||||
if finalContent == "" {
|
||||
finalContent = contentBuffer.String()
|
||||
}
|
||||
inputTokens = chatResp.InputTokens
|
||||
outputTokens = chatResp.OutputTokens
|
||||
p.recordPatrolUsage(chatResp.InputTokens, chatResp.OutputTokens)
|
||||
finalContent := attempt.finalContent
|
||||
inputTokens = attempt.response.InputTokens
|
||||
outputTokens = attempt.response.OutputTokens
|
||||
p.recordPatrolUsage(attempt.response.InputTokens, attempt.response.OutputTokens)
|
||||
|
||||
// Clean thinking tokens
|
||||
finalContent = CleanThinkingTokens(finalContent)
|
||||
|
|
@ -482,6 +552,9 @@ func (p *PatrolService) runAIAnalysisState(ctx context.Context, snap patrolRunti
|
|||
Int("findings_resolved", adapter.getResolvedCount()).
|
||||
Msg("AI Patrol: Agentic patrol analysis complete")
|
||||
|
||||
var toolCallsMu sync.Mutex
|
||||
completedToolCalls := append([]ToolCallRecord(nil), attempt.toolCalls...)
|
||||
rawToolOutputs := append([]string(nil), attempt.rawToolOutputs...)
|
||||
p.ensureInvestigationToolCall(ctx, executor, &toolCallsMu, &completedToolCalls, &rawToolOutputs, noStream)
|
||||
|
||||
// Broadcast completion
|
||||
|
|
@ -1162,6 +1235,16 @@ func (p *PatrolService) buildTriageSeedContextState(
|
|||
scope *PatrolScope,
|
||||
guestIntel map[string]*GuestIntelligence,
|
||||
) (string, []string) {
|
||||
sections, seededFindingIDs := p.buildTriageSeedSectionsState(triage, snap, scope, guestIntel)
|
||||
return p.assembleSeedWithinBudget(sections, p.calculateSeedBudget()), seededFindingIDs
|
||||
}
|
||||
|
||||
func (p *PatrolService) buildTriageSeedSectionsState(
|
||||
triage *TriageResult,
|
||||
snap patrolRuntimeState,
|
||||
scope *PatrolScope,
|
||||
guestIntel map[string]*GuestIntelligence,
|
||||
) ([]seedSection, []string) {
|
||||
p.mu.RLock()
|
||||
cfg := p.config
|
||||
p.mu.RUnlock()
|
||||
|
|
@ -1179,24 +1262,31 @@ func (p *PatrolService) buildTriageSeedContextState(
|
|||
|
||||
sections := []seedSection{
|
||||
// P0 — always include.
|
||||
{priority: 0, name: "triage_briefing", content: FormatTriageBriefing(triage)},
|
||||
{priority: 0, name: "triage_overview", content: formatTriageOverviewSection(triage)},
|
||||
{priority: 0, name: "findings", content: findingsCtx},
|
||||
{priority: 0, name: "health_alerts", content: p.seedHealthAndAlertsState(snap, flaggedSet, cfg, now)},
|
||||
{priority: 0, name: "scope", content: buildScopeSection(scope, sortedScopedIDs(flaggedSet))},
|
||||
|
||||
// P1 — flagged resource details only.
|
||||
// P2 — triage already preserves the flagged set, so these sections can
|
||||
// summarize under tighter provider-derived retry budgets.
|
||||
{
|
||||
priority: 1,
|
||||
priority: 2,
|
||||
name: "triage_flags",
|
||||
content: formatTriageFlagsSection(triage),
|
||||
summary: formatTriageFlagsSummary(triage),
|
||||
},
|
||||
{
|
||||
priority: 2,
|
||||
name: "flagged_inventory",
|
||||
content: p.seedResourceInventoryState(snap, flaggedSet, cfg, now, false, guestIntel),
|
||||
summary: p.seedResourceInventorySummaryState(snap, flaggedSet, cfg, now, guestIntel),
|
||||
},
|
||||
|
||||
// P3 — healthy rollup is useful context but lowest-value on retries.
|
||||
{priority: 3, name: "triage_healthy", content: formatTriageHealthySummarySection(triage)},
|
||||
}
|
||||
|
||||
budget := p.calculateSeedBudget()
|
||||
result := p.assembleSeedWithinBudget(sections, budget)
|
||||
|
||||
return result, seededFindingIDs
|
||||
return sections, seededFindingIDs
|
||||
}
|
||||
|
||||
// buildSeedContext produces the infrastructure state context for the agentic patrol loop.
|
||||
|
|
@ -1204,6 +1294,11 @@ func (p *PatrolService) buildTriageSeedContextState(
|
|||
// connection health, and baselines/trends so the model can analyze without tool calls.
|
||||
// Tools remain available for targeted deep-dives.
|
||||
func (p *PatrolService) buildSeedContextState(snap patrolRuntimeState, scope *PatrolScope, guestIntel map[string]*GuestIntelligence) (string, []string) {
|
||||
sections, seededFindingIDs := p.buildSeedSectionsState(snap, scope, guestIntel)
|
||||
return p.assembleSeedWithinBudget(sections, p.calculateSeedBudget()), seededFindingIDs
|
||||
}
|
||||
|
||||
func (p *PatrolService) buildSeedSectionsState(snap patrolRuntimeState, scope *PatrolScope, guestIntel map[string]*GuestIntelligence) ([]seedSection, []string) {
|
||||
p.mu.RLock()
|
||||
cfg := p.config
|
||||
p.mu.RUnlock()
|
||||
|
|
@ -1239,19 +1334,12 @@ func (p *PatrolService) buildSeedContextState(snap patrolRuntimeState, scope *Pa
|
|||
{priority: 4, name: "pmg_snapshot", content: p.seedPMGSnapshotStringState(snap, scopedSet, cfg, intel.isQuiet)},
|
||||
}
|
||||
|
||||
budget := p.calculateSeedBudget()
|
||||
result := p.assembleSeedWithinBudget(sections, budget)
|
||||
|
||||
return result, seededFindingIDs
|
||||
return sections, seededFindingIDs
|
||||
}
|
||||
|
||||
func (p *PatrolService) calculateSeedBudget() int {
|
||||
const (
|
||||
systemPromptEstimate = 4_000
|
||||
toolEstimate = 8_000
|
||||
outputReserve = 8_000
|
||||
historyReserve = 16_000
|
||||
minimumSeedBudget = 16_000
|
||||
defaultContextWindow = 128_000
|
||||
)
|
||||
|
||||
model := ""
|
||||
|
|
@ -1262,6 +1350,34 @@ func (p *PatrolService) calculateSeedBudget() int {
|
|||
}
|
||||
|
||||
contextWindow := providers.ContextWindowTokens(model)
|
||||
if contextWindow <= 0 {
|
||||
contextWindow = defaultContextWindow
|
||||
}
|
||||
budget := calculateSeedBudgetForContextWindow(contextWindow)
|
||||
|
||||
log.Debug().
|
||||
Str("model", model).
|
||||
Int("context_window_tokens", contextWindow).
|
||||
Int("seed_budget_tokens", budget).
|
||||
Msg("AI Patrol: Calculated seed context token budget")
|
||||
|
||||
return budget
|
||||
}
|
||||
|
||||
func calculateSeedBudgetForContextWindow(contextWindow int) int {
|
||||
const (
|
||||
systemPromptEstimate = 4_000
|
||||
toolEstimate = 8_000
|
||||
outputReserve = 8_000
|
||||
historyReserve = 16_000
|
||||
minimumSeedBudget = 16_000
|
||||
defaultContextWindow = 128_000
|
||||
)
|
||||
|
||||
if contextWindow <= 0 {
|
||||
contextWindow = defaultContextWindow
|
||||
}
|
||||
|
||||
budget := contextWindow - systemPromptEstimate - toolEstimate - outputReserve - historyReserve
|
||||
|
||||
// Clamp floor so small-context models aren't forced beyond practical capacity.
|
||||
|
|
@ -1273,15 +1389,91 @@ func (p *PatrolService) calculateSeedBudget() int {
|
|||
budget = floor
|
||||
}
|
||||
|
||||
log.Debug().
|
||||
Str("model", model).
|
||||
Int("context_window_tokens", contextWindow).
|
||||
Int("seed_budget_tokens", budget).
|
||||
Msg("AI Patrol: Calculated seed context token budget")
|
||||
|
||||
return budget
|
||||
}
|
||||
|
||||
func isPatrolContextWindowError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
msg := strings.ToLower(err.Error())
|
||||
return strings.Contains(msg, "exceed_context_size_error") ||
|
||||
strings.Contains(msg, "exceeds the available context size") ||
|
||||
strings.Contains(msg, "maximum context length") ||
|
||||
strings.Contains(msg, "context window") ||
|
||||
strings.Contains(msg, "context length") ||
|
||||
strings.Contains(msg, "n_ctx")
|
||||
}
|
||||
|
||||
func patrolSeedRetryBudgets(err error) []int {
|
||||
if err == nil {
|
||||
return []int{patrolRetrySeedBudget1, patrolRetrySeedBudget2, patrolRetrySeedBudget3}
|
||||
}
|
||||
|
||||
contextWindow := extractPatrolContextWindow(err)
|
||||
if contextWindow <= 0 {
|
||||
return []int{patrolRetrySeedBudget1, patrolRetrySeedBudget2, patrolRetrySeedBudget3}
|
||||
}
|
||||
|
||||
safeBudget := calculateSeedBudgetForContextWindow(contextWindow)
|
||||
return uniquePositiveInts(
|
||||
safeBudget,
|
||||
maxInt(1_000, safeBudget/2),
|
||||
maxInt(1_000, safeBudget/4),
|
||||
)
|
||||
}
|
||||
|
||||
func extractPatrolContextWindow(err error) int {
|
||||
if err == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
msg := err.Error()
|
||||
for _, pattern := range patrolContextWindowPatterns {
|
||||
matches := pattern.FindStringSubmatch(msg)
|
||||
if len(matches) < 2 {
|
||||
continue
|
||||
}
|
||||
nctx, convErr := strconv.Atoi(matches[1])
|
||||
if convErr == nil && nctx > 0 {
|
||||
return nctx
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
func uniquePositiveInts(values ...int) []int {
|
||||
result := make([]int, 0, len(values))
|
||||
seen := make(map[int]struct{}, len(values))
|
||||
for _, value := range values {
|
||||
if value <= 0 {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[value]; exists {
|
||||
continue
|
||||
}
|
||||
seen[value] = struct{}{}
|
||||
result = append(result, value)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func maxInt(a, b int) int {
|
||||
if a > b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (p *PatrolService) assembleSeedWithinBudget(sections []seedSection, budgetTokens int) string {
|
||||
if len(sections) == 0 {
|
||||
return ""
|
||||
|
|
|
|||
|
|
@ -3,12 +3,15 @@ package ai
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/baseline"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/chat"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/cost"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/tools"
|
||||
|
|
@ -262,6 +265,89 @@ func TestRunAIAnalysis_EarlyErrors(t *testing.T) {
|
|||
})
|
||||
}
|
||||
|
||||
func TestPatrolSeedRetryBudgets_UsesProviderContextWindow(t *testing.T) {
|
||||
err := fmt.Errorf("API error (400): {\"error\":{\"message\":\"request exceeds the available context size (8192 tokens)\",\"type\":\"exceed_context_size_error\",\"n_ctx\":8192}}")
|
||||
|
||||
got := patrolSeedRetryBudgets(err)
|
||||
want := []int{4096, 2048, 1024}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("patrolSeedRetryBudgets() = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunAIAnalysis_RetriesWithProviderDerivedSeedBudget(t *testing.T) {
|
||||
persistence := config.NewConfigPersistence(t.TempDir())
|
||||
svc := NewService(persistence, nil)
|
||||
svc.cfg = &config.AIConfig{Enabled: true, PatrolModel: "mock:model"}
|
||||
svc.provider = &mockProvider{}
|
||||
|
||||
executor := tools.NewPulseToolExecutor(tools.ExecutorConfig{})
|
||||
promptTokens := make([]int, 0, 2)
|
||||
mockCS := &mockChatService{
|
||||
executor: executor,
|
||||
executePatrolStreamFunc: func(ctx context.Context, req PatrolExecuteRequest, callback ChatStreamCallback) (*PatrolStreamResponse, error) {
|
||||
promptTokens = append(promptTokens, chat.EstimateTokens(req.Prompt))
|
||||
if len(promptTokens) == 1 {
|
||||
return nil, fmt.Errorf("API error (400): {\"error\":{\"message\":\"request exceeds the available context size (4096 tokens)\",\"type\":\"exceed_context_size_error\",\"n_ctx\":4096}}")
|
||||
}
|
||||
return &PatrolStreamResponse{
|
||||
Content: "analysis complete",
|
||||
InputTokens: 8,
|
||||
OutputTokens: 4,
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
svc.SetChatService(mockCS)
|
||||
|
||||
ps := NewPatrolService(svc, nil)
|
||||
ps.SetConfig(PatrolConfig{
|
||||
Enabled: true,
|
||||
AnalyzeNodes: true,
|
||||
AnalyzeGuests: true,
|
||||
})
|
||||
|
||||
state := models.StateSnapshot{
|
||||
Nodes: []models.Node{
|
||||
{ID: "node-1", Name: "pve-1", Status: "online", CPU: 0.20, Memory: models.Memory{Usage: 30.0}},
|
||||
},
|
||||
}
|
||||
for i := 0; i < 600; i++ {
|
||||
state.VMs = append(state.VMs, models.VM{
|
||||
ID: fmt.Sprintf("vm-%d", i),
|
||||
VMID: 100 + i,
|
||||
Name: fmt.Sprintf("flagged-guest-%03d-%s", i, strings.Repeat("x", 96)),
|
||||
Node: "pve-1",
|
||||
Status: "running",
|
||||
CPU: 0.95,
|
||||
Memory: models.Memory{Usage: 91.0},
|
||||
Disk: models.Disk{Usage: 15.0},
|
||||
})
|
||||
}
|
||||
|
||||
triage := ps.runDeterministicTriageState(context.Background(), patrolRuntimeStateForTest(ps, state), nil, nil)
|
||||
seed, _ := ps.buildTriageSeedContextState(triage, patrolRuntimeStateForTest(ps, state), nil, nil)
|
||||
if got := chat.EstimateTokens(seed); got <= 2048 {
|
||||
t.Fatalf("expected initial triage seed to exceed retry budget, got %d tokens", got)
|
||||
}
|
||||
|
||||
result, err := ps.runAIAnalysisState(context.Background(), patrolRuntimeStateForTest(ps, state), &PatrolScope{NoStream: true})
|
||||
if err != nil {
|
||||
t.Fatalf("runAIAnalysisState returned error: %v", err)
|
||||
}
|
||||
if result == nil || result.Response != "analysis complete" {
|
||||
t.Fatalf("unexpected analysis result: %#v", result)
|
||||
}
|
||||
if len(promptTokens) != 2 {
|
||||
t.Fatalf("expected 2 patrol attempts, got %d", len(promptTokens))
|
||||
}
|
||||
if promptTokens[1] >= promptTokens[0] {
|
||||
t.Fatalf("expected retry prompt to shrink, got %d then %d tokens", promptTokens[0], promptTokens[1])
|
||||
}
|
||||
if promptTokens[1] > 2048 {
|
||||
t.Fatalf("expected retry prompt to honor provider-derived budget, got %d tokens", promptTokens[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSeedResourceInventory_QuietSummary(t *testing.T) {
|
||||
ps := NewPatrolService(nil, nil)
|
||||
cfg := DefaultPatrolConfig()
|
||||
|
|
|
|||
|
|
@ -453,12 +453,23 @@ func FormatTriageBriefing(triage *TriageResult) string {
|
|||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString("# Deterministic Triage Results\n")
|
||||
sb.WriteString(formatTriageOverviewSection(triage))
|
||||
sb.WriteString(formatTriageFlagsSection(triage))
|
||||
sb.WriteString(formatTriageHealthySummarySection(triage))
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func formatTriageOverviewSection(triage *TriageResult) string {
|
||||
if triage == nil {
|
||||
return "# Deterministic Triage Results\nNo triage data available.\n"
|
||||
}
|
||||
|
||||
scanned := triage.Summary.TotalNodes + triage.Summary.TotalGuests + triage.Summary.TotalStorage +
|
||||
triage.Summary.TotalDocker + triage.Summary.TotalTrueNAS + triage.Summary.TotalPBS + triage.Summary.TotalPMG
|
||||
sb.WriteString(fmt.Sprintf(
|
||||
"Scanned %d resources: %d nodes, %d guests, %d storage resources (%d pools, %d physical disks), %d docker hosts, %d TrueNAS systems, %d PBS, %d PMG.\n\n",
|
||||
|
||||
return fmt.Sprintf(
|
||||
"# Deterministic Triage Results\nScanned %d resources: %d nodes, %d guests, %d storage resources (%d pools, %d physical disks), %d docker hosts, %d TrueNAS systems, %d PBS, %d PMG.\n\n",
|
||||
scanned,
|
||||
triage.Summary.TotalNodes,
|
||||
triage.Summary.TotalGuests,
|
||||
|
|
@ -469,36 +480,73 @@ func FormatTriageBriefing(triage *TriageResult) string {
|
|||
triage.Summary.TotalTrueNAS,
|
||||
triage.Summary.TotalPBS,
|
||||
triage.Summary.TotalPMG,
|
||||
))
|
||||
)
|
||||
}
|
||||
|
||||
if len(triage.Flags) > 0 {
|
||||
flags := append([]TriageFlag(nil), triage.Flags...)
|
||||
sort.Slice(flags, func(i, j int) bool {
|
||||
ri := triageSeverityRank(flags[i].Severity)
|
||||
rj := triageSeverityRank(flags[j].Severity)
|
||||
if ri != rj {
|
||||
return ri > rj
|
||||
}
|
||||
if flags[i].ResourceType != flags[j].ResourceType {
|
||||
return flags[i].ResourceType < flags[j].ResourceType
|
||||
}
|
||||
return flags[i].ResourceName < flags[j].ResourceName
|
||||
})
|
||||
func formatTriageFlagsSection(triage *TriageResult) string {
|
||||
flags := sortedTriageFlags(triage)
|
||||
if len(flags) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags)))
|
||||
sb.WriteString("| Resource | Type | Flag | Severity | Detail |\n")
|
||||
sb.WriteString("|----------|------|------|----------|--------|\n")
|
||||
for _, flag := range flags {
|
||||
resource := triageResourceName(flag.ResourceName, flag.ResourceID)
|
||||
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n",
|
||||
triageTableEscape(resource),
|
||||
triageTableEscape(flag.ResourceType),
|
||||
triageTableEscape(triageFlagLabel(flag)),
|
||||
triageTableEscape(flag.Severity),
|
||||
triageTableEscape(flag.Reason),
|
||||
))
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags)))
|
||||
sb.WriteString("| Resource | Type | Flag | Severity | Detail |\n")
|
||||
sb.WriteString("|----------|------|------|----------|--------|\n")
|
||||
for _, flag := range flags {
|
||||
resource := triageResourceName(flag.ResourceName, flag.ResourceID)
|
||||
sb.WriteString(fmt.Sprintf("| %s | %s | %s | %s | %s |\n",
|
||||
triageTableEscape(resource),
|
||||
triageTableEscape(flag.ResourceType),
|
||||
triageTableEscape(triageFlagLabel(flag)),
|
||||
triageTableEscape(flag.Severity),
|
||||
triageTableEscape(flag.Reason),
|
||||
))
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func formatTriageFlagsSummary(triage *TriageResult) string {
|
||||
flags := sortedTriageFlags(triage)
|
||||
if len(flags) == 0 {
|
||||
return ""
|
||||
}
|
||||
|
||||
critical := 0
|
||||
warning := 0
|
||||
watch := 0
|
||||
entries := make([]string, 0, triageMinInt(len(flags), 8))
|
||||
for i, flag := range flags {
|
||||
switch strings.ToLower(flag.Severity) {
|
||||
case "critical":
|
||||
critical++
|
||||
case "warning":
|
||||
warning++
|
||||
case "watch":
|
||||
watch++
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
if i < 8 {
|
||||
resource := triageResourceName(flag.ResourceName, flag.ResourceID)
|
||||
entries = append(entries, fmt.Sprintf("%s (%s, %s)", resource, triageFlagLabel(flag), flag.Severity))
|
||||
}
|
||||
}
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("## Flagged Resources (%d)\n", len(flags)))
|
||||
sb.WriteString(fmt.Sprintf("Severity mix: critical=%d, warning=%d, watch=%d.\n", critical, warning, watch))
|
||||
sb.WriteString("Prioritize: ")
|
||||
sb.WriteString(strings.Join(entries, "; "))
|
||||
if len(flags) > len(entries) {
|
||||
sb.WriteString(fmt.Sprintf("; +%d more flagged resources", len(flags)-len(entries)))
|
||||
}
|
||||
sb.WriteString(".\n\n")
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func formatTriageHealthySummarySection(triage *TriageResult) string {
|
||||
if triage == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
healthyGuests := triage.Summary.TotalGuests - triageUniqueFlaggedByTypes(triage.Flags, "vm", "system-container")
|
||||
|
|
@ -532,6 +580,8 @@ func FormatTriageBriefing(triage *TriageResult) string {
|
|||
}
|
||||
|
||||
totalHealthy := healthyNodes + healthyGuests + healthyStorage + healthyDocker + healthyTrueNAS + healthyPBS + healthyPMG
|
||||
|
||||
var sb strings.Builder
|
||||
sb.WriteString(fmt.Sprintf("## Healthy Resources (%d)\n", totalHealthy))
|
||||
sb.WriteString(fmt.Sprintf("Nodes: %d healthy\n", healthyNodes))
|
||||
sb.WriteString(fmt.Sprintf("Guests: %d running, %d stopped\n", triage.Summary.RunningGuests, triage.Summary.StoppedGuests))
|
||||
|
|
@ -543,10 +593,36 @@ func FormatTriageBriefing(triage *TriageResult) string {
|
|||
sb.WriteString(fmt.Sprintf("TrueNAS: %d systems\n", healthyTrueNAS))
|
||||
sb.WriteString(fmt.Sprintf("PBS: %d instances\n", healthyPBS))
|
||||
sb.WriteString(fmt.Sprintf("PMG: %d instances\n", healthyPMG))
|
||||
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func triageMinInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func sortedTriageFlags(triage *TriageResult) []TriageFlag {
|
||||
if triage == nil || len(triage.Flags) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
flags := append([]TriageFlag(nil), triage.Flags...)
|
||||
sort.Slice(flags, func(i, j int) bool {
|
||||
ri := triageSeverityRank(flags[i].Severity)
|
||||
rj := triageSeverityRank(flags[j].Severity)
|
||||
if ri != rj {
|
||||
return ri > rj
|
||||
}
|
||||
if flags[i].ResourceType != flags[j].ResourceType {
|
||||
return flags[i].ResourceType < flags[j].ResourceType
|
||||
}
|
||||
return flags[i].ResourceName < flags[j].ResourceName
|
||||
})
|
||||
return flags
|
||||
}
|
||||
|
||||
func triageBuildSummaryState(snap patrolRuntimeState, flaggedIDs map[string]bool) TriageSummary {
|
||||
counts := patrolRuntimeCountResources(snap)
|
||||
storagePools := patrolStoragePoolRows(snap, nil)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue