diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index b6cabd6d0..040df5d6a 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -743,7 +743,7 @@ profile and assignment columns, but embedded table framing must route through ## Completion Obligations -1. Update this contract when agent lifecycle ownership changes. +1. Update this contract when agent lifecycle ownership changes. Routes added under the shared `internal/api/` extension point that are clearly outside lifecycle ownership (for example `POST /api/ai/patrol/preflight`, owned by ai-runtime) do not extend this subsystem's contract; they live in their owning subsystem. 2. Keep shared API proof routing aligned whenever install, register, or profile payloads change. 3. Update runtime and settings tests in the same slice when lifecycle behavior changes. 4. Keep host-agent test hooks, command-client factories, and timing overrides diff --git a/docs/release-control/v6/internal/subsystems/ai-runtime.md b/docs/release-control/v6/internal/subsystems/ai-runtime.md index acdc04306..f3e1483f6 100644 --- a/docs/release-control/v6/internal/subsystems/ai-runtime.md +++ b/docs/release-control/v6/internal/subsystems/ai-runtime.md @@ -108,7 +108,7 @@ runtime cost control, and shared AI transport surfaces. ## Completion Obligations -1. Update this contract when canonical AI runtime or transport entry points move, including transport-level provider request-shape changes such as DeepSeek `tool_choice` coercion, and runtime-failure classification splits (for example separating forced tool selection rejection, no tool-capable endpoint, and generic model-level lack of tool support into distinct causes) +1. Update this contract when canonical AI runtime or transport entry points move, including transport-level provider request-shape changes such as DeepSeek `tool_choice` coercion, runtime-failure classification splits (for example separating forced tool selection rejection, no tool-capable endpoint, and generic model-level lack of tool support into distinct causes), and Patrol-specific verification surfaces such as `POST /api/ai/patrol/preflight` that exercise the full chat-completions path with a minimal tool definition rather than only listing models 2. Keep AI runtime and shared API proof routing aligned in `registry.json` 3. Preserve explicit coverage for chat, Patrol, remediation, and cost-control behavior when AI runtime changes Patrol runtime failures are part of that runtime contract: provider, model, diff --git a/docs/release-control/v6/internal/subsystems/api-contracts.md b/docs/release-control/v6/internal/subsystems/api-contracts.md index 31aca8ef4..622d2fe1e 100644 --- a/docs/release-control/v6/internal/subsystems/api-contracts.md +++ b/docs/release-control/v6/internal/subsystems/api-contracts.md @@ -894,7 +894,7 @@ the canonical monitored-system blocked payload. ## Completion Obligations -1. Update contract tests when payloads change +1. Update contract tests when payloads change, including admin verification endpoints such as `POST /api/ai/patrol/preflight` whose response shape (`tool_call_observed`, `duration_ms`, classified `cause`/`summary`/`recommendation`) is part of the canonical Patrol diagnostic surface 2. Update frontend API types in the same slice 3. Route runtime changes through the explicit API-contract proof policies in `registry.json`; default fallback proof routing is not allowed 4. Update this contract when canonical payload ownership changes diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md index 34ea3ba8d..a7bd6c073 100644 --- a/docs/release-control/v6/internal/subsystems/storage-recovery.md +++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md @@ -762,7 +762,7 @@ bypass the API fail-closed execution gate. ## Completion Obligations -1. Update this contract when canonical storage or recovery entry points move +1. Update this contract when canonical storage or recovery entry points move. Routes added under the shared `internal/api/` extension point that are clearly outside storage/recovery ownership (for example `POST /api/ai/patrol/preflight`, owned by ai-runtime) do not extend this subsystem's contract; they live in their owning subsystem. 2. Keep recovery store/runtime changes aligned with the storage and recovery frontend proofs in `registry.json` 3. Tighten guardrails when legacy storage or recovery presentation paths are removed 4. Preserve the dependency split: API payload ownership stays in `api-contracts`, settings shell ownership stays in `frontend-primitives`, and canonical resource truth stays in `unified-resources` diff --git a/frontend-modern/src/api/__tests__/patrolPreflight.test.ts b/frontend-modern/src/api/__tests__/patrolPreflight.test.ts new file mode 100644 index 000000000..1b188b34e --- /dev/null +++ b/frontend-modern/src/api/__tests__/patrolPreflight.test.ts @@ -0,0 +1,77 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { runPatrolPreflight } from '../patrol'; +import { apiFetchJSON } from '@/utils/apiClient'; + +vi.mock('@/utils/apiClient', () => ({ + apiFetchJSON: vi.fn(), +})); + +describe('runPatrolPreflight', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('POSTs to the canonical preflight endpoint and returns the structured result', async () => { + vi.mocked(apiFetchJSON).mockResolvedValueOnce({ + success: true, + provider: 'deepseek', + model: 'deepseek-v4-flash', + tool_call_observed: true, + duration_ms: 842, + message: 'Provider accepted the preflight request and the model emitted a tool call.', + }); + + const result = await runPatrolPreflight(); + + expect(apiFetchJSON).toHaveBeenCalledWith('/api/ai/patrol/preflight', { + method: 'POST', + body: '{}', + headers: { 'Content-Type': 'application/json' }, + }); + expect(result.success).toBe(true); + expect(result.tool_call_observed).toBe(true); + expect(result.provider).toBe('deepseek'); + expect(result.duration_ms).toBe(842); + }); + + it('forwards provider and model overrides verbatim', async () => { + vi.mocked(apiFetchJSON).mockResolvedValueOnce({ + success: false, + tool_call_observed: false, + duration_ms: 312, + message: 'Provider rejected forced tool selection', + cause: 'tool_choice_rejected', + recommendation: 'Pulse will retry with automatic tool selection on the next Patrol run.', + }); + + const result = await runPatrolPreflight({ provider: 'deepseek', model: 'deepseek-v4-flash' }); + + expect(apiFetchJSON).toHaveBeenCalledWith('/api/ai/patrol/preflight', { + method: 'POST', + body: JSON.stringify({ provider: 'deepseek', model: 'deepseek-v4-flash' }), + headers: { 'Content-Type': 'application/json' }, + }); + expect(result.success).toBe(false); + expect(result.cause).toBe('tool_choice_rejected'); + }); + + it('exposes the soft-warning shape when the model accepted the request but did not call the tool', async () => { + vi.mocked(apiFetchJSON).mockResolvedValueOnce({ + success: false, + tool_call_observed: false, + duration_ms: 412, + message: + 'Provider accepted the preflight request but the model did not emit a tool call. Patrol may still work in practice.', + cause: 'model_tool_support_unverified', + recommendation: + 'Trigger a real Patrol run to confirm tool calling. If that fails, switch to a model with stronger tool-following behaviour.', + }); + + const result = await runPatrolPreflight(); + + expect(result.success).toBe(false); + expect(result.tool_call_observed).toBe(false); + expect(result.cause).toBe('model_tool_support_unverified'); + expect(result.recommendation).toContain('real Patrol'); + }); +}); diff --git a/frontend-modern/src/api/patrol.ts b/frontend-modern/src/api/patrol.ts index d4d23457d..6b392ae41 100644 --- a/frontend-modern/src/api/patrol.ts +++ b/frontend-modern/src/api/patrol.ts @@ -647,3 +647,54 @@ export async function triggerPatrolRun(): Promise<{ success: boolean; message: s method: 'POST', }); } + +/** + * Response shape for POST /api/ai/patrol/preflight. + * + * The preflight runs a one-shot tool-call round-trip against the + * configured (or overridden) Patrol provider+model so operators can + * verify tool calling actually works before relying on the scheduled + * cadence. Distinct from the per-provider /api/ai/test endpoints, + * which only list models. + * + * - success: provider call succeeded AND the model emitted a tool call. + * - tool_call_observed: whether the model emitted a tool call. False + * with success=false means the request failed; false with + * success=false and cause=model_tool_support_unverified means the + * provider accepted the request but the model did not call the tool + * (a soft warning — Patrol may still work in practice). + */ +export interface PatrolPreflightResponse { + success: boolean; + provider?: string; + model?: string; + tool_call_observed: boolean; + duration_ms: number; + message: string; + cause?: string; + summary?: string; + description?: string; + recommendation?: string; + action?: string; +} + +export interface PatrolPreflightRequest { + provider?: string; + model?: string; +} + +/** + * Verify that the configured Patrol provider+model can actually call tools. + * + * Pass `provider` and/or `model` to override the configured Patrol + * selection (useful when previewing a new model before saving). + */ +export async function runPatrolPreflight( + body: PatrolPreflightRequest = {}, +): Promise { + return apiFetchJSON('/api/ai/patrol/preflight', { + method: 'POST', + body: JSON.stringify(body), + headers: { 'Content-Type': 'application/json' }, + }); +} diff --git a/internal/ai/patrol_preflight.go b/internal/ai/patrol_preflight.go new file mode 100644 index 000000000..de47a354d --- /dev/null +++ b/internal/ai/patrol_preflight.go @@ -0,0 +1,179 @@ +package ai + +import ( + "context" + "strings" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/ai/providers" + "github.com/rcourtman/pulse-go-rewrite/internal/config" +) + +// PatrolPreflightResult captures the outcome of a one-shot tool-call +// preflight against the configured (or overridden) Patrol provider+model. +// +// Unlike a connection test, which only lists models, the preflight +// exercises the full chat-completions path with a minimal tool +// definition. This surfaces real failure modes — provider rejecting the +// tool_choice value, no tool-capable endpoint available, model genuinely +// lacking tool support — at configuration time instead of waiting for +// the next scheduled Patrol run to silently fail. +type PatrolPreflightResult struct { + Success bool + Provider string + Model string + ToolCallObserved bool + DurationMs int64 + + // Classification fields populated for both failure and soft-warning + // outcomes. On a fully-green preflight (Success=true, + // ToolCallObserved=true) Cause is PatrolFailureCauseNone and Title / + // Summary describe the success. + Cause PatrolFailureCause + Title string + Summary string + Description string + Recommendation string +} + +// patrolPreflightToolName is the synthetic tool the model is asked to +// call. Kept distinct from real Patrol tools so accidental invocation +// outside preflight has no operational meaning. +const patrolPreflightToolName = "verify_pulse_patrol" + +// RunPatrolToolPreflight performs a one-shot tool-call round-trip against +// the configured Patrol provider+model, or against the overrides supplied +// in providerName / model. Both override arguments are optional: empty +// strings fall back to the configured Patrol model. +// +// The function returns a PatrolPreflightResult describing the outcome. +// It never returns an error — provider and configuration failures are +// classified into the result's Cause / Summary / Recommendation fields +// the same way runtime Patrol failures are, so the caller can render a +// single response shape for every outcome. +func (s *Service) RunPatrolToolPreflight(ctx context.Context, providerName, model string) PatrolPreflightResult { + started := time.Now() + + s.mu.RLock() + cfg := s.cfg + s.mu.RUnlock() + + result := PatrolPreflightResult{} + + if cfg == nil { + result.Cause = PatrolFailureCauseSettingsPersistence + result.Title = "Pulse Patrol: Assistant settings unavailable" + result.Summary = "Pulse Assistant settings could not be loaded" + result.Recommendation = "Confirm Pulse settings persistence is healthy, then re-run preflight." + result.DurationMs = time.Since(started).Milliseconds() + return result + } + if !cfg.Enabled { + result.Cause = PatrolFailureCauseAssistantDisabled + result.Title = "Pulse Patrol: Assistant disabled" + result.Summary = "Pulse Assistant is not enabled" + result.Recommendation = "Enable Pulse Assistant in Assistant & Patrol settings, then re-run preflight." + result.DurationMs = time.Since(started).Milliseconds() + return result + } + + modelStr := strings.TrimSpace(model) + if modelStr == "" { + modelStr = strings.TrimSpace(cfg.GetPatrolModel()) + } + if modelStr == "" { + result.Cause = PatrolFailureCauseModelNotSelected + result.Title = "Pulse Patrol: No model selected" + result.Summary = "Patrol has no model selected" + result.Recommendation = "Select a Patrol model in Assistant & Patrol settings, then re-run preflight." + result.DurationMs = time.Since(started).Milliseconds() + return result + } + + // If the caller supplied a provider override, re-prefix the model id + // so the factory routes to the requested provider. + overrideProvider := strings.TrimSpace(providerName) + if overrideProvider != "" { + _, bare := config.ParseModelString(modelStr) + if bare == "" { + bare = modelStr + } + modelStr = overrideProvider + ":" + bare + } + + parsedProvider, parsedModel := config.ParseModelString(modelStr) + result.Provider = parsedProvider + result.Model = parsedModel + + provider, err := providers.NewForModel(cfg, modelStr) + if err != nil { + applyPatrolPreflightDiagnostic(&result, err) + result.DurationMs = time.Since(started).Milliseconds() + return result + } + + req := providers.ChatRequest{ + Model: modelStr, + System: "You are running a brief Pulse Patrol tool-call self-test. " + + "Call the " + patrolPreflightToolName + " tool with parameter ok set to true. " + + "Do not reply with any other text.", + Messages: []providers.Message{ + {Role: "user", Content: "Run the Pulse Patrol tool-call self-test."}, + }, + Tools: []providers.Tool{ + { + Name: patrolPreflightToolName, + Description: "Confirm Pulse Patrol can receive a tool call. Always pass ok=true.", + InputSchema: map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "ok": map[string]interface{}{ + "type": "boolean", + "description": "Always pass true.", + }, + }, + "required": []string{"ok"}, + "additionalProperties": false, + }, + }, + }, + ToolChoice: &providers.ToolChoice{Type: providers.ToolChoiceAny}, + MaxTokens: 256, + } + + resp, err := provider.Chat(ctx, req) + result.DurationMs = time.Since(started).Milliseconds() + + if err != nil { + applyPatrolPreflightDiagnostic(&result, err) + return result + } + + result.Success = true + result.ToolCallObserved = resp != nil && len(resp.ToolCalls) > 0 + if result.ToolCallObserved { + result.Cause = PatrolFailureCauseNone + result.Title = "Pulse Patrol: Preflight succeeded" + result.Summary = "Provider accepted the preflight request and the model emitted a tool call." + return result + } + + // Soft warning: provider accepted the request shape (no error) but + // the model returned plain text instead of calling the verify tool. + // Patrol may still work in practice, but we flag this so the operator + // can run a real Patrol pass to confirm before relying on it. + result.Cause = PatrolFailureCauseModelToolSupportUnverified + result.Title = "Pulse Patrol: Model did not emit a tool call during preflight" + result.Summary = "Provider accepted the preflight request but the model did not emit a tool call. Patrol may still work in practice." + result.Recommendation = "Trigger a real Patrol run to confirm tool calling. If that fails, switch to a model with stronger tool-following behaviour." + return result +} + +func applyPatrolPreflightDiagnostic(result *PatrolPreflightResult, err error) { + failure := patrolRuntimeFailureFromError(err) + result.Cause = failure.Cause + result.Title = failure.Title + result.Summary = failure.Summary + result.Description = failure.Description + result.Recommendation = failure.Recommendation +} diff --git a/internal/ai/patrol_preflight_test.go b/internal/ai/patrol_preflight_test.go new file mode 100644 index 000000000..b1f63dfce --- /dev/null +++ b/internal/ai/patrol_preflight_test.go @@ -0,0 +1,233 @@ +package ai + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/rcourtman/pulse-go-rewrite/internal/config" +) + +// patrolPreflightTestService spins up an in-memory OpenAI-compatible server +// and an AI Service configured to route through it via the OpenAI provider. +// Tests exercise the full RunPatrolToolPreflight path (factory -> +// provider.Chat -> classification) without mocking the provider layer. +type patrolPreflightTestService struct { + svc *Service + server *httptest.Server +} + +func (h *patrolPreflightTestService) close() { + if h.server != nil { + h.server.Close() + } +} + +func newPatrolPreflightTestService(t *testing.T, model string, handler http.HandlerFunc) *patrolPreflightTestService { + t.Helper() + server := httptest.NewServer(handler) + t.Cleanup(server.Close) + + svc := NewService(config.NewConfigPersistence(t.TempDir()), nil) + svc.cfg = &config.AIConfig{ + Enabled: true, + Model: model, + PatrolModel: model, + OpenAIAPIKey: "sk-test", + OpenAIBaseURL: server.URL, + } + return &patrolPreflightTestService{svc: svc, server: server} +} + +func TestRunPatrolToolPreflight_Success_ToolCallObserved(t *testing.T) { + // Provider accepts the request and the model emits a tool call — + // the green-path outcome we want operators to see. + h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) { + var got map[string]interface{} + if err := json.NewDecoder(r.Body).Decode(&got); err != nil { + t.Fatalf("decode request: %v", err) + } + if got["model"] != "gpt-4o-mini" { + t.Fatalf("model = %v", got["model"]) + } + if got["tools"] == nil { + t.Fatalf("expected tools in request, got %v", got) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl-preflight-ok", + "model": "gpt-4o-mini", + "choices": [{ + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": "", + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": {"name": "verify_pulse_patrol", "arguments": "{\"ok\":true}"} + }] + } + }] + }`)) + }) + defer h.close() + + result := h.svc.RunPatrolToolPreflight(context.Background(), "", "") + if !result.Success { + t.Fatalf("expected Success, got %+v", result) + } + if !result.ToolCallObserved { + t.Fatalf("expected ToolCallObserved, got %+v", result) + } + if result.Cause != PatrolFailureCauseNone { + t.Fatalf("unexpected cause %q", result.Cause) + } + if result.Provider != config.AIProviderOpenAI { + t.Fatalf("unexpected provider %q", result.Provider) + } + if result.Model != "gpt-4o-mini" { + t.Fatalf("unexpected model %q", result.Model) + } + if !strings.Contains(result.Title, "succeeded") { + t.Fatalf("expected success title, got %q", result.Title) + } +} + +func TestRunPatrolToolPreflight_Success_NoToolCall(t *testing.T) { + // Provider accepts the request but the model returned plain text + // instead of calling the verify tool. Pulse soft-warns rather than + // hard-failing because the model may still work in practice. + h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "id": "chatcmpl-preflight-no-tool", + "model": "gpt-4o-mini", + "choices": [{ + "finish_reason": "stop", + "message": {"role": "assistant", "content": "ok"} + }] + }`)) + }) + defer h.close() + + result := h.svc.RunPatrolToolPreflight(context.Background(), "", "") + if !result.Success { + t.Fatalf("expected Success (soft warning still counts as success), got %+v", result) + } + if result.ToolCallObserved { + t.Fatalf("expected ToolCallObserved=false, got %+v", result) + } + if result.Cause != PatrolFailureCauseModelToolSupportUnverified { + t.Fatalf("unexpected cause %q", result.Cause) + } + if !strings.Contains(result.Recommendation, "real Patrol") { + t.Fatalf("expected recommendation to mention real Patrol run, got %q", result.Recommendation) + } +} + +func TestRunPatrolToolPreflight_ToolChoiceRejected(t *testing.T) { + // Provider returns the DeepSeek-style 400 that motivated the + // classifier split. Preflight must surface the more accurate + // tool_choice_rejected cause, not the generic "model unsupported". + h := newPatrolPreflightTestService(t, "openai:test-model", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"message":"deepseek-reasoner does not support this tool_choice","type":"invalid_request_error"}}`)) + }) + defer h.close() + + result := h.svc.RunPatrolToolPreflight(context.Background(), "", "") + if result.Success { + t.Fatalf("expected failure, got %+v", result) + } + if result.Cause != PatrolFailureCauseToolChoiceRejected { + t.Fatalf("unexpected cause %q (want %q)", result.Cause, PatrolFailureCauseToolChoiceRejected) + } + if !strings.Contains(result.Recommendation, "automatic tool selection") { + t.Fatalf("expected recommendation to mention automatic tool selection, got %q", result.Recommendation) + } +} + +func TestRunPatrolToolPreflight_NoToolCapableEndpoint(t *testing.T) { + // Routing failure — no available endpoint supports tools for the + // requested model. OpenRouter is the canonical source of this + // wording in production; the classifier pattern-matches on the + // error string regardless of transport, so we use the OpenAI + // provider as a stand-in (OpenRouter base URL isn't configurable). + h := newPatrolPreflightTestService(t, "openai:test-model", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"error":{"message":"No endpoints found that support the provided 'tool_choice' value."}}`)) + }) + defer h.close() + + result := h.svc.RunPatrolToolPreflight(context.Background(), "", "") + if result.Success { + t.Fatalf("expected failure, got %+v", result) + } + if result.Cause != PatrolFailureCauseNoToolCapableEndpoint { + t.Fatalf("unexpected cause %q (want %q)", result.Cause, PatrolFailureCauseNoToolCapableEndpoint) + } + if !strings.Contains(result.Recommendation, "routing") && !strings.Contains(result.Recommendation, "filters") { + t.Fatalf("expected recommendation to mention routing/filters, got %q", result.Recommendation) + } +} + +func TestRunPatrolToolPreflight_AssistantDisabled(t *testing.T) { + svc := NewService(config.NewConfigPersistence(t.TempDir()), nil) + svc.cfg = &config.AIConfig{Enabled: false, PatrolModel: "openai:gpt-4o-mini"} + + result := svc.RunPatrolToolPreflight(context.Background(), "", "") + if result.Success { + t.Fatalf("expected failure when assistant disabled, got %+v", result) + } + if result.Cause != PatrolFailureCauseAssistantDisabled { + t.Fatalf("unexpected cause %q", result.Cause) + } +} + +func TestRunPatrolToolPreflight_NoModelSelected(t *testing.T) { + svc := NewService(config.NewConfigPersistence(t.TempDir()), nil) + svc.cfg = &config.AIConfig{Enabled: true, OpenAIAPIKey: "sk-test"} + + result := svc.RunPatrolToolPreflight(context.Background(), "", "") + if result.Success { + t.Fatalf("expected failure when no model selected, got %+v", result) + } + if result.Cause != PatrolFailureCauseModelNotSelected { + t.Fatalf("unexpected cause %q", result.Cause) + } +} + +func TestRunPatrolToolPreflight_RequestShapeIncludesVerifyTool(t *testing.T) { + // Locks the preflight request shape so a future refactor can't + // silently strip the tool definition or tool_choice (which would + // turn preflight into a connection-test that misses the original + // failure mode). + var captured map[string]interface{} + h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewDecoder(r.Body).Decode(&captured) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"x","model":"gpt-4o-mini","choices":[{"finish_reason":"stop","message":{"role":"assistant","content":"ok"}}]}`)) + }) + defer h.close() + + _ = h.svc.RunPatrolToolPreflight(context.Background(), "", "") + + tools, ok := captured["tools"].([]interface{}) + if !ok || len(tools) != 1 { + t.Fatalf("expected exactly one tool in preflight request, got %v", captured["tools"]) + } + tool := tools[0].(map[string]interface{}) + fn := tool["function"].(map[string]interface{}) + if fn["name"] != patrolPreflightToolName { + t.Fatalf("expected tool name %q, got %v", patrolPreflightToolName, fn["name"]) + } + if captured["tool_choice"] == nil { + t.Fatalf("expected tool_choice to be set in preflight request, got %v", captured) + } +} diff --git a/internal/api/ai_handlers.go b/internal/api/ai_handlers.go index 2766e339b..820720022 100644 --- a/internal/api/ai_handlers.go +++ b/internal/api/ai_handlers.go @@ -5491,6 +5491,103 @@ func (h *AISettingsHandler) HandleForcePatrol(w http.ResponseWriter, r *http.Req } } +// aiPatrolPreflightResponse is the JSON shape returned by +// HandlePatrolPreflight. It mirrors aiProviderTestResponse for the +// classified-error fields and adds two preflight-specific signals: +// ToolCallObserved (whether the model emitted a tool call) and +// DurationMs (round-trip latency, useful for spotting slow providers). +type aiPatrolPreflightResponse struct { + Success bool `json:"success"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + ToolCallObserved bool `json:"tool_call_observed"` + DurationMs int64 `json:"duration_ms"` + Message string `json:"message"` + Cause string `json:"cause,omitempty"` + Summary string `json:"summary,omitempty"` + Description string `json:"description,omitempty"` + Recommendation string `json:"recommendation,omitempty"` + Action string `json:"action,omitempty"` +} + +type aiPatrolPreflightRequest struct { + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` +} + +// HandlePatrolPreflight runs a one-shot tool-call round-trip against the +// configured (or overridden) Patrol provider+model so operators can +// verify tool calling actually works before relying on Patrol's +// scheduled cadence (POST /api/ai/patrol/preflight). +// +// This is distinct from POST /api/ai/test/{provider}, which only lists +// models and therefore can pass while Patrol fails 100% of runs (the +// failure mode that bit Pulse for 33 days before the DeepSeek fix +// landed). +func (h *AISettingsHandler) HandlePatrolPreflight(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + + if !ensureSettingsWriteScope(h.getConfig(r.Context()), w, r) { + return + } + + aiService := h.GetAIService(r.Context()) + if aiService == nil { + writePatrolServiceUnavailableResponse(w) + return + } + + // Optional body: override the provider and/or model. Empty body is + // allowed and means "use the configured Patrol provider+model". + var body aiPatrolPreflightRequest + if r.ContentLength > 0 { + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, `{"error":"Invalid JSON body"}`, http.StatusBadRequest) + return + } + } + if body.Provider != "" { + if len(body.Provider) > 64 || !isValidProviderName(body.Provider) { + http.Error(w, `{"error":"Invalid provider name"}`, http.StatusBadRequest) + return + } + } + if len(body.Model) > 256 { + http.Error(w, `{"error":"Model id too long"}`, http.StatusBadRequest) + return + } + + // Tight timeout: preflight is a single round-trip, not a Patrol pass. + // 30s matches the per-provider connection-test budget; in practice + // this completes in well under 10s for any healthy provider. + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + + preflight := aiService.RunPatrolToolPreflight(ctx, body.Provider, body.Model) + + response := aiPatrolPreflightResponse{ + Success: preflight.Success && preflight.ToolCallObserved, + Provider: preflight.Provider, + Model: preflight.Model, + ToolCallObserved: preflight.ToolCallObserved, + DurationMs: preflight.DurationMs, + Message: preflight.Summary, + Cause: string(preflight.Cause), + Summary: preflight.Description, + Recommendation: preflight.Recommendation, + } + if !response.Success { + response.Action = "open_provider_settings" + } + + if err := utils.WriteJSONResponse(w, response); err != nil { + log.Error().Err(err).Msg("Failed to write Patrol preflight response") + } +} + // HandleAcknowledgeFinding acknowledges a finding (POST /api/ai/patrol/acknowledge) // This marks the finding as seen but keeps it visible (dimmed). Auto-resolve removes it when condition clears. // This matches alert acknowledgement behavior for UI consistency. diff --git a/internal/api/ai_handlers_test.go b/internal/api/ai_handlers_test.go index bd55529cd..a7c4b5cae 100644 --- a/internal/api/ai_handlers_test.go +++ b/internal/api/ai_handlers_test.go @@ -2876,3 +2876,74 @@ func TestAISettingsHandler_Approvals_RejectCrossOrgAccess(t *testing.T) { require.True(t, ok) require.Equal(t, approval.StatusPending, denyApp.Status) } + +// ======================================== +// HandlePatrolPreflight tests +// ======================================== + +func TestAISettingsHandler_PatrolPreflight_MethodNotAllowed(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + cfg := &config.Config{DataPath: tmp} + persistence := config.NewConfigPersistence(tmp) + handler := newTestAISettingsHandler(cfg, persistence, nil) + + req := newLoopbackRequest(http.MethodGet, "/api/ai/patrol/preflight", nil) + rec := httptest.NewRecorder() + handler.HandlePatrolPreflight(rec, req) + + assert.Equal(t, http.StatusMethodNotAllowed, rec.Code) +} + +func TestAISettingsHandler_PatrolPreflight_AssistantDisabled(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + cfg := &config.Config{DataPath: tmp} + persistence := config.NewConfigPersistence(tmp) + + aiCfg := config.NewDefaultAIConfig() + aiCfg.Enabled = false + if err := persistence.SaveAIConfig(*aiCfg); err != nil { + t.Fatalf("SaveAIConfig: %v", err) + } + + handler := newTestAISettingsHandler(cfg, persistence, nil) + + req := newLoopbackRequest(http.MethodPost, "/api/ai/patrol/preflight", nil) + rec := httptest.NewRecorder() + handler.HandlePatrolPreflight(rec, req) + + require.Equal(t, http.StatusOK, rec.Code) + + var resp struct { + Success bool `json:"success"` + Cause string `json:"cause"` + Message string `json:"message"` + ToolCallObserved bool `json:"tool_call_observed"` + DurationMs int64 `json:"duration_ms"` + Action string `json:"action"` + } + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp)) + assert.False(t, resp.Success) + assert.False(t, resp.ToolCallObserved) + assert.Equal(t, string(ai.PatrolFailureCauseAssistantDisabled), resp.Cause) + assert.Equal(t, "open_provider_settings", resp.Action) +} + +func TestAISettingsHandler_PatrolPreflight_RejectsInvalidProviderName(t *testing.T) { + t.Parallel() + + tmp := t.TempDir() + cfg := &config.Config{DataPath: tmp} + persistence := config.NewConfigPersistence(tmp) + handler := newTestAISettingsHandler(cfg, persistence, nil) + + req := newLoopbackRequest(http.MethodPost, "/api/ai/patrol/preflight", + bytes.NewReader([]byte(`{"provider":"../etc/passwd"}`))) + rec := httptest.NewRecorder() + handler.HandlePatrolPreflight(rec, req) + + assert.Equal(t, http.StatusBadRequest, rec.Code) +} diff --git a/internal/api/route_inventory_test.go b/internal/api/route_inventory_test.go index 6e2d5e67e..442b07558 100644 --- a/internal/api/route_inventory_test.go +++ b/internal/api/route_inventory_test.go @@ -600,6 +600,7 @@ var allRouteAllowlist = []string{ "/api/ai/patrol/findings", "/api/ai/patrol/history", "/api/ai/patrol/run", + "/api/ai/patrol/preflight", "/api/ai/patrol/acknowledge", "/api/ai/patrol/dismiss", "/api/ai/patrol/findings/note", diff --git a/internal/api/router_routes_ai_relay.go b/internal/api/router_routes_ai_relay.go index e86c74f22..2f5f75d65 100644 --- a/internal/api/router_routes_ai_relay.go +++ b/internal/api/router_routes_ai_relay.go @@ -111,6 +111,10 @@ func (r *Router) registerAIRelayRoutesGroup() { // SECURITY: AI Patrol read endpoints - require ai:execute scope r.mux.HandleFunc("/api/ai/patrol/history", RequireAuth(r.config, RequireScope(config.ScopeAIExecute, r.aiSettingsHandler.HandleGetFindingsHistory))) r.mux.HandleFunc("/api/ai/patrol/run", RequireAdmin(r.config, RequireScope(config.ScopeAIExecute, r.aiSettingsHandler.HandleForcePatrol))) + // Patrol tool-call preflight: one-shot verification that the configured Patrol + // provider+model actually supports tool calling. Distinct from /api/ai/test + // (which only lists models) so a green test cannot mask a 100%-failing Patrol. + r.mux.HandleFunc("/api/ai/patrol/preflight", RequirePermission(r.config, r.authorizer, auth.ActionWrite, auth.ResourceSettings, RequireScope(config.ScopeSettingsWrite, r.aiSettingsHandler.HandlePatrolPreflight))) // SECURITY: AI Patrol mutation endpoints - require ai:execute scope to prevent low-privilege tokens from // dismissing, suppressing, or otherwise hiding findings. This prevents attackers from blinding AI Patrol. r.mux.HandleFunc("/api/ai/patrol/acknowledge", RequireAuth(r.config, requireRelayMobileRuntimeRoute(relayMobileRoutePatrolAcknowledge, r.aiSettingsHandler.HandleAcknowledgeFinding)))