mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-05-20 01:01:20 +00:00
Add POST /api/ai/patrol/preflight tool-call verification
The existing per-provider /api/ai/test endpoints only call ListModels — they pass for every provider that returns a catalog, even when Patrol fails 100% of runs because tools aren't actually wired up. That gap is what let the DeepSeek tool_choice rejection silently fail Patrol for 33 days before the recent fix landed. POST /api/ai/patrol/preflight runs a one-shot tool-call round-trip with the configured (or overridden) Patrol provider+model and a minimal verify_pulse_patrol tool. Failures route through ClassifyPatrolRuntimeFailure so the new tool_choice_rejected and no_tool_capable_endpoint causes surface here too. A successful provider call where the model returned plain text (no tool call) is reported as a soft warning (model_tool_support_unverified): Patrol may still work but the operator should run a real pass to confirm. The endpoint bypasses the chat service so cost recording isn't charged for verification, and uses ScopeSettingsWrite to align with the existing /api/ai/test gating. Backend + typed frontend client (runPatrolPreflight); UI button on Assistant & Patrol settings follows. Contracts updated: - ai-runtime: completion obligation extended to cover the new verification surface - api-contracts: payload shape (tool_call_observed, duration_ms) noted in obligations - agent-lifecycle, storage-recovery: dependent-extension acknowledgment that ai-runtime owns the new route despite it living under internal/api/
This commit is contained in:
parent
566520f43e
commit
e26a57a157
12 changed files with 717 additions and 4 deletions
|
|
@ -743,7 +743,7 @@ profile and assignment columns, but embedded table framing must route through
|
|||
|
||||
## Completion Obligations
|
||||
|
||||
1. Update this contract when agent lifecycle ownership changes.
|
||||
1. Update this contract when agent lifecycle ownership changes. Routes added under the shared `internal/api/` extension point that are clearly outside lifecycle ownership (for example `POST /api/ai/patrol/preflight`, owned by ai-runtime) do not extend this subsystem's contract; they live in their owning subsystem.
|
||||
2. Keep shared API proof routing aligned whenever install, register, or profile payloads change.
|
||||
3. Update runtime and settings tests in the same slice when lifecycle behavior changes.
|
||||
4. Keep host-agent test hooks, command-client factories, and timing overrides
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ runtime cost control, and shared AI transport surfaces.
|
|||
|
||||
## Completion Obligations
|
||||
|
||||
1. Update this contract when canonical AI runtime or transport entry points move, including transport-level provider request-shape changes such as DeepSeek `tool_choice` coercion, and runtime-failure classification splits (for example separating forced tool selection rejection, no tool-capable endpoint, and generic model-level lack of tool support into distinct causes)
|
||||
1. Update this contract when canonical AI runtime or transport entry points move, including transport-level provider request-shape changes such as DeepSeek `tool_choice` coercion, runtime-failure classification splits (for example separating forced tool selection rejection, no tool-capable endpoint, and generic model-level lack of tool support into distinct causes), and Patrol-specific verification surfaces such as `POST /api/ai/patrol/preflight` that exercise the full chat-completions path with a minimal tool definition rather than only listing models
|
||||
2. Keep AI runtime and shared API proof routing aligned in `registry.json`
|
||||
3. Preserve explicit coverage for chat, Patrol, remediation, and cost-control behavior when AI runtime changes
|
||||
Patrol runtime failures are part of that runtime contract: provider, model,
|
||||
|
|
|
|||
|
|
@ -894,7 +894,7 @@ the canonical monitored-system blocked payload.
|
|||
|
||||
## Completion Obligations
|
||||
|
||||
1. Update contract tests when payloads change
|
||||
1. Update contract tests when payloads change, including admin verification endpoints such as `POST /api/ai/patrol/preflight` whose response shape (`tool_call_observed`, `duration_ms`, classified `cause`/`summary`/`recommendation`) is part of the canonical Patrol diagnostic surface
|
||||
2. Update frontend API types in the same slice
|
||||
3. Route runtime changes through the explicit API-contract proof policies in `registry.json`; default fallback proof routing is not allowed
|
||||
4. Update this contract when canonical payload ownership changes
|
||||
|
|
|
|||
|
|
@ -762,7 +762,7 @@ bypass the API fail-closed execution gate.
|
|||
|
||||
## Completion Obligations
|
||||
|
||||
1. Update this contract when canonical storage or recovery entry points move
|
||||
1. Update this contract when canonical storage or recovery entry points move. Routes added under the shared `internal/api/` extension point that are clearly outside storage/recovery ownership (for example `POST /api/ai/patrol/preflight`, owned by ai-runtime) do not extend this subsystem's contract; they live in their owning subsystem.
|
||||
2. Keep recovery store/runtime changes aligned with the storage and recovery frontend proofs in `registry.json`
|
||||
3. Tighten guardrails when legacy storage or recovery presentation paths are removed
|
||||
4. Preserve the dependency split: API payload ownership stays in `api-contracts`, settings shell ownership stays in `frontend-primitives`, and canonical resource truth stays in `unified-resources`
|
||||
|
|
|
|||
77
frontend-modern/src/api/__tests__/patrolPreflight.test.ts
Normal file
77
frontend-modern/src/api/__tests__/patrolPreflight.test.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runPatrolPreflight } from '../patrol';
|
||||
import { apiFetchJSON } from '@/utils/apiClient';
|
||||
|
||||
vi.mock('@/utils/apiClient', () => ({
|
||||
apiFetchJSON: vi.fn(),
|
||||
}));
|
||||
|
||||
describe('runPatrolPreflight', () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it('POSTs to the canonical preflight endpoint and returns the structured result', async () => {
|
||||
vi.mocked(apiFetchJSON).mockResolvedValueOnce({
|
||||
success: true,
|
||||
provider: 'deepseek',
|
||||
model: 'deepseek-v4-flash',
|
||||
tool_call_observed: true,
|
||||
duration_ms: 842,
|
||||
message: 'Provider accepted the preflight request and the model emitted a tool call.',
|
||||
});
|
||||
|
||||
const result = await runPatrolPreflight();
|
||||
|
||||
expect(apiFetchJSON).toHaveBeenCalledWith('/api/ai/patrol/preflight', {
|
||||
method: 'POST',
|
||||
body: '{}',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.tool_call_observed).toBe(true);
|
||||
expect(result.provider).toBe('deepseek');
|
||||
expect(result.duration_ms).toBe(842);
|
||||
});
|
||||
|
||||
it('forwards provider and model overrides verbatim', async () => {
|
||||
vi.mocked(apiFetchJSON).mockResolvedValueOnce({
|
||||
success: false,
|
||||
tool_call_observed: false,
|
||||
duration_ms: 312,
|
||||
message: 'Provider rejected forced tool selection',
|
||||
cause: 'tool_choice_rejected',
|
||||
recommendation: 'Pulse will retry with automatic tool selection on the next Patrol run.',
|
||||
});
|
||||
|
||||
const result = await runPatrolPreflight({ provider: 'deepseek', model: 'deepseek-v4-flash' });
|
||||
|
||||
expect(apiFetchJSON).toHaveBeenCalledWith('/api/ai/patrol/preflight', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({ provider: 'deepseek', model: 'deepseek-v4-flash' }),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.cause).toBe('tool_choice_rejected');
|
||||
});
|
||||
|
||||
it('exposes the soft-warning shape when the model accepted the request but did not call the tool', async () => {
|
||||
vi.mocked(apiFetchJSON).mockResolvedValueOnce({
|
||||
success: false,
|
||||
tool_call_observed: false,
|
||||
duration_ms: 412,
|
||||
message:
|
||||
'Provider accepted the preflight request but the model did not emit a tool call. Patrol may still work in practice.',
|
||||
cause: 'model_tool_support_unverified',
|
||||
recommendation:
|
||||
'Trigger a real Patrol run to confirm tool calling. If that fails, switch to a model with stronger tool-following behaviour.',
|
||||
});
|
||||
|
||||
const result = await runPatrolPreflight();
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.tool_call_observed).toBe(false);
|
||||
expect(result.cause).toBe('model_tool_support_unverified');
|
||||
expect(result.recommendation).toContain('real Patrol');
|
||||
});
|
||||
});
|
||||
|
|
@ -647,3 +647,54 @@ export async function triggerPatrolRun(): Promise<{ success: boolean; message: s
|
|||
method: 'POST',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Response shape for POST /api/ai/patrol/preflight.
|
||||
*
|
||||
* The preflight runs a one-shot tool-call round-trip against the
|
||||
* configured (or overridden) Patrol provider+model so operators can
|
||||
* verify tool calling actually works before relying on the scheduled
|
||||
* cadence. Distinct from the per-provider /api/ai/test endpoints,
|
||||
* which only list models.
|
||||
*
|
||||
* - success: provider call succeeded AND the model emitted a tool call.
|
||||
* - tool_call_observed: whether the model emitted a tool call. False
|
||||
* with success=false means the request failed; false with
|
||||
* success=false and cause=model_tool_support_unverified means the
|
||||
* provider accepted the request but the model did not call the tool
|
||||
* (a soft warning — Patrol may still work in practice).
|
||||
*/
|
||||
export interface PatrolPreflightResponse {
|
||||
success: boolean;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
tool_call_observed: boolean;
|
||||
duration_ms: number;
|
||||
message: string;
|
||||
cause?: string;
|
||||
summary?: string;
|
||||
description?: string;
|
||||
recommendation?: string;
|
||||
action?: string;
|
||||
}
|
||||
|
||||
export interface PatrolPreflightRequest {
|
||||
provider?: string;
|
||||
model?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify that the configured Patrol provider+model can actually call tools.
|
||||
*
|
||||
* Pass `provider` and/or `model` to override the configured Patrol
|
||||
* selection (useful when previewing a new model before saving).
|
||||
*/
|
||||
export async function runPatrolPreflight(
|
||||
body: PatrolPreflightRequest = {},
|
||||
): Promise<PatrolPreflightResponse> {
|
||||
return apiFetchJSON<PatrolPreflightResponse>('/api/ai/patrol/preflight', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify(body),
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
});
|
||||
}
|
||||
|
|
|
|||
179
internal/ai/patrol_preflight.go
Normal file
179
internal/ai/patrol_preflight.go
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/ai/providers"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
)
|
||||
|
||||
// PatrolPreflightResult captures the outcome of a one-shot tool-call
|
||||
// preflight against the configured (or overridden) Patrol provider+model.
|
||||
//
|
||||
// Unlike a connection test, which only lists models, the preflight
|
||||
// exercises the full chat-completions path with a minimal tool
|
||||
// definition. This surfaces real failure modes — provider rejecting the
|
||||
// tool_choice value, no tool-capable endpoint available, model genuinely
|
||||
// lacking tool support — at configuration time instead of waiting for
|
||||
// the next scheduled Patrol run to silently fail.
|
||||
type PatrolPreflightResult struct {
|
||||
Success bool
|
||||
Provider string
|
||||
Model string
|
||||
ToolCallObserved bool
|
||||
DurationMs int64
|
||||
|
||||
// Classification fields populated for both failure and soft-warning
|
||||
// outcomes. On a fully-green preflight (Success=true,
|
||||
// ToolCallObserved=true) Cause is PatrolFailureCauseNone and Title /
|
||||
// Summary describe the success.
|
||||
Cause PatrolFailureCause
|
||||
Title string
|
||||
Summary string
|
||||
Description string
|
||||
Recommendation string
|
||||
}
|
||||
|
||||
// patrolPreflightToolName is the synthetic tool the model is asked to
|
||||
// call. Kept distinct from real Patrol tools so accidental invocation
|
||||
// outside preflight has no operational meaning.
|
||||
const patrolPreflightToolName = "verify_pulse_patrol"
|
||||
|
||||
// RunPatrolToolPreflight performs a one-shot tool-call round-trip against
|
||||
// the configured Patrol provider+model, or against the overrides supplied
|
||||
// in providerName / model. Both override arguments are optional: empty
|
||||
// strings fall back to the configured Patrol model.
|
||||
//
|
||||
// The function returns a PatrolPreflightResult describing the outcome.
|
||||
// It never returns an error — provider and configuration failures are
|
||||
// classified into the result's Cause / Summary / Recommendation fields
|
||||
// the same way runtime Patrol failures are, so the caller can render a
|
||||
// single response shape for every outcome.
|
||||
func (s *Service) RunPatrolToolPreflight(ctx context.Context, providerName, model string) PatrolPreflightResult {
|
||||
started := time.Now()
|
||||
|
||||
s.mu.RLock()
|
||||
cfg := s.cfg
|
||||
s.mu.RUnlock()
|
||||
|
||||
result := PatrolPreflightResult{}
|
||||
|
||||
if cfg == nil {
|
||||
result.Cause = PatrolFailureCauseSettingsPersistence
|
||||
result.Title = "Pulse Patrol: Assistant settings unavailable"
|
||||
result.Summary = "Pulse Assistant settings could not be loaded"
|
||||
result.Recommendation = "Confirm Pulse settings persistence is healthy, then re-run preflight."
|
||||
result.DurationMs = time.Since(started).Milliseconds()
|
||||
return result
|
||||
}
|
||||
if !cfg.Enabled {
|
||||
result.Cause = PatrolFailureCauseAssistantDisabled
|
||||
result.Title = "Pulse Patrol: Assistant disabled"
|
||||
result.Summary = "Pulse Assistant is not enabled"
|
||||
result.Recommendation = "Enable Pulse Assistant in Assistant & Patrol settings, then re-run preflight."
|
||||
result.DurationMs = time.Since(started).Milliseconds()
|
||||
return result
|
||||
}
|
||||
|
||||
modelStr := strings.TrimSpace(model)
|
||||
if modelStr == "" {
|
||||
modelStr = strings.TrimSpace(cfg.GetPatrolModel())
|
||||
}
|
||||
if modelStr == "" {
|
||||
result.Cause = PatrolFailureCauseModelNotSelected
|
||||
result.Title = "Pulse Patrol: No model selected"
|
||||
result.Summary = "Patrol has no model selected"
|
||||
result.Recommendation = "Select a Patrol model in Assistant & Patrol settings, then re-run preflight."
|
||||
result.DurationMs = time.Since(started).Milliseconds()
|
||||
return result
|
||||
}
|
||||
|
||||
// If the caller supplied a provider override, re-prefix the model id
|
||||
// so the factory routes to the requested provider.
|
||||
overrideProvider := strings.TrimSpace(providerName)
|
||||
if overrideProvider != "" {
|
||||
_, bare := config.ParseModelString(modelStr)
|
||||
if bare == "" {
|
||||
bare = modelStr
|
||||
}
|
||||
modelStr = overrideProvider + ":" + bare
|
||||
}
|
||||
|
||||
parsedProvider, parsedModel := config.ParseModelString(modelStr)
|
||||
result.Provider = parsedProvider
|
||||
result.Model = parsedModel
|
||||
|
||||
provider, err := providers.NewForModel(cfg, modelStr)
|
||||
if err != nil {
|
||||
applyPatrolPreflightDiagnostic(&result, err)
|
||||
result.DurationMs = time.Since(started).Milliseconds()
|
||||
return result
|
||||
}
|
||||
|
||||
req := providers.ChatRequest{
|
||||
Model: modelStr,
|
||||
System: "You are running a brief Pulse Patrol tool-call self-test. " +
|
||||
"Call the " + patrolPreflightToolName + " tool with parameter ok set to true. " +
|
||||
"Do not reply with any other text.",
|
||||
Messages: []providers.Message{
|
||||
{Role: "user", Content: "Run the Pulse Patrol tool-call self-test."},
|
||||
},
|
||||
Tools: []providers.Tool{
|
||||
{
|
||||
Name: patrolPreflightToolName,
|
||||
Description: "Confirm Pulse Patrol can receive a tool call. Always pass ok=true.",
|
||||
InputSchema: map[string]interface{}{
|
||||
"type": "object",
|
||||
"properties": map[string]interface{}{
|
||||
"ok": map[string]interface{}{
|
||||
"type": "boolean",
|
||||
"description": "Always pass true.",
|
||||
},
|
||||
},
|
||||
"required": []string{"ok"},
|
||||
"additionalProperties": false,
|
||||
},
|
||||
},
|
||||
},
|
||||
ToolChoice: &providers.ToolChoice{Type: providers.ToolChoiceAny},
|
||||
MaxTokens: 256,
|
||||
}
|
||||
|
||||
resp, err := provider.Chat(ctx, req)
|
||||
result.DurationMs = time.Since(started).Milliseconds()
|
||||
|
||||
if err != nil {
|
||||
applyPatrolPreflightDiagnostic(&result, err)
|
||||
return result
|
||||
}
|
||||
|
||||
result.Success = true
|
||||
result.ToolCallObserved = resp != nil && len(resp.ToolCalls) > 0
|
||||
if result.ToolCallObserved {
|
||||
result.Cause = PatrolFailureCauseNone
|
||||
result.Title = "Pulse Patrol: Preflight succeeded"
|
||||
result.Summary = "Provider accepted the preflight request and the model emitted a tool call."
|
||||
return result
|
||||
}
|
||||
|
||||
// Soft warning: provider accepted the request shape (no error) but
|
||||
// the model returned plain text instead of calling the verify tool.
|
||||
// Patrol may still work in practice, but we flag this so the operator
|
||||
// can run a real Patrol pass to confirm before relying on it.
|
||||
result.Cause = PatrolFailureCauseModelToolSupportUnverified
|
||||
result.Title = "Pulse Patrol: Model did not emit a tool call during preflight"
|
||||
result.Summary = "Provider accepted the preflight request but the model did not emit a tool call. Patrol may still work in practice."
|
||||
result.Recommendation = "Trigger a real Patrol run to confirm tool calling. If that fails, switch to a model with stronger tool-following behaviour."
|
||||
return result
|
||||
}
|
||||
|
||||
func applyPatrolPreflightDiagnostic(result *PatrolPreflightResult, err error) {
|
||||
failure := patrolRuntimeFailureFromError(err)
|
||||
result.Cause = failure.Cause
|
||||
result.Title = failure.Title
|
||||
result.Summary = failure.Summary
|
||||
result.Description = failure.Description
|
||||
result.Recommendation = failure.Recommendation
|
||||
}
|
||||
233
internal/ai/patrol_preflight_test.go
Normal file
233
internal/ai/patrol_preflight_test.go
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
package ai
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
)
|
||||
|
||||
// patrolPreflightTestService spins up an in-memory OpenAI-compatible server
|
||||
// and an AI Service configured to route through it via the OpenAI provider.
|
||||
// Tests exercise the full RunPatrolToolPreflight path (factory ->
|
||||
// provider.Chat -> classification) without mocking the provider layer.
|
||||
type patrolPreflightTestService struct {
|
||||
svc *Service
|
||||
server *httptest.Server
|
||||
}
|
||||
|
||||
func (h *patrolPreflightTestService) close() {
|
||||
if h.server != nil {
|
||||
h.server.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func newPatrolPreflightTestService(t *testing.T, model string, handler http.HandlerFunc) *patrolPreflightTestService {
|
||||
t.Helper()
|
||||
server := httptest.NewServer(handler)
|
||||
t.Cleanup(server.Close)
|
||||
|
||||
svc := NewService(config.NewConfigPersistence(t.TempDir()), nil)
|
||||
svc.cfg = &config.AIConfig{
|
||||
Enabled: true,
|
||||
Model: model,
|
||||
PatrolModel: model,
|
||||
OpenAIAPIKey: "sk-test",
|
||||
OpenAIBaseURL: server.URL,
|
||||
}
|
||||
return &patrolPreflightTestService{svc: svc, server: server}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_Success_ToolCallObserved(t *testing.T) {
|
||||
// Provider accepts the request and the model emits a tool call —
|
||||
// the green-path outcome we want operators to see.
|
||||
h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) {
|
||||
var got map[string]interface{}
|
||||
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
|
||||
t.Fatalf("decode request: %v", err)
|
||||
}
|
||||
if got["model"] != "gpt-4o-mini" {
|
||||
t.Fatalf("model = %v", got["model"])
|
||||
}
|
||||
if got["tools"] == nil {
|
||||
t.Fatalf("expected tools in request, got %v", got)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id": "chatcmpl-preflight-ok",
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "verify_pulse_patrol", "arguments": "{\"ok\":true}"}
|
||||
}]
|
||||
}
|
||||
}]
|
||||
}`))
|
||||
})
|
||||
defer h.close()
|
||||
|
||||
result := h.svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if !result.Success {
|
||||
t.Fatalf("expected Success, got %+v", result)
|
||||
}
|
||||
if !result.ToolCallObserved {
|
||||
t.Fatalf("expected ToolCallObserved, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseNone {
|
||||
t.Fatalf("unexpected cause %q", result.Cause)
|
||||
}
|
||||
if result.Provider != config.AIProviderOpenAI {
|
||||
t.Fatalf("unexpected provider %q", result.Provider)
|
||||
}
|
||||
if result.Model != "gpt-4o-mini" {
|
||||
t.Fatalf("unexpected model %q", result.Model)
|
||||
}
|
||||
if !strings.Contains(result.Title, "succeeded") {
|
||||
t.Fatalf("expected success title, got %q", result.Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_Success_NoToolCall(t *testing.T) {
|
||||
// Provider accepts the request but the model returned plain text
|
||||
// instead of calling the verify tool. Pulse soft-warns rather than
|
||||
// hard-failing because the model may still work in practice.
|
||||
h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"id": "chatcmpl-preflight-no-tool",
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "ok"}
|
||||
}]
|
||||
}`))
|
||||
})
|
||||
defer h.close()
|
||||
|
||||
result := h.svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if !result.Success {
|
||||
t.Fatalf("expected Success (soft warning still counts as success), got %+v", result)
|
||||
}
|
||||
if result.ToolCallObserved {
|
||||
t.Fatalf("expected ToolCallObserved=false, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseModelToolSupportUnverified {
|
||||
t.Fatalf("unexpected cause %q", result.Cause)
|
||||
}
|
||||
if !strings.Contains(result.Recommendation, "real Patrol") {
|
||||
t.Fatalf("expected recommendation to mention real Patrol run, got %q", result.Recommendation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_ToolChoiceRejected(t *testing.T) {
|
||||
// Provider returns the DeepSeek-style 400 that motivated the
|
||||
// classifier split. Preflight must surface the more accurate
|
||||
// tool_choice_rejected cause, not the generic "model unsupported".
|
||||
h := newPatrolPreflightTestService(t, "openai:test-model", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"deepseek-reasoner does not support this tool_choice","type":"invalid_request_error"}}`))
|
||||
})
|
||||
defer h.close()
|
||||
|
||||
result := h.svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if result.Success {
|
||||
t.Fatalf("expected failure, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseToolChoiceRejected {
|
||||
t.Fatalf("unexpected cause %q (want %q)", result.Cause, PatrolFailureCauseToolChoiceRejected)
|
||||
}
|
||||
if !strings.Contains(result.Recommendation, "automatic tool selection") {
|
||||
t.Fatalf("expected recommendation to mention automatic tool selection, got %q", result.Recommendation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_NoToolCapableEndpoint(t *testing.T) {
|
||||
// Routing failure — no available endpoint supports tools for the
|
||||
// requested model. OpenRouter is the canonical source of this
|
||||
// wording in production; the classifier pattern-matches on the
|
||||
// error string regardless of transport, so we use the OpenAI
|
||||
// provider as a stand-in (OpenRouter base URL isn't configurable).
|
||||
h := newPatrolPreflightTestService(t, "openai:test-model", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"No endpoints found that support the provided 'tool_choice' value."}}`))
|
||||
})
|
||||
defer h.close()
|
||||
|
||||
result := h.svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if result.Success {
|
||||
t.Fatalf("expected failure, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseNoToolCapableEndpoint {
|
||||
t.Fatalf("unexpected cause %q (want %q)", result.Cause, PatrolFailureCauseNoToolCapableEndpoint)
|
||||
}
|
||||
if !strings.Contains(result.Recommendation, "routing") && !strings.Contains(result.Recommendation, "filters") {
|
||||
t.Fatalf("expected recommendation to mention routing/filters, got %q", result.Recommendation)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_AssistantDisabled(t *testing.T) {
|
||||
svc := NewService(config.NewConfigPersistence(t.TempDir()), nil)
|
||||
svc.cfg = &config.AIConfig{Enabled: false, PatrolModel: "openai:gpt-4o-mini"}
|
||||
|
||||
result := svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if result.Success {
|
||||
t.Fatalf("expected failure when assistant disabled, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseAssistantDisabled {
|
||||
t.Fatalf("unexpected cause %q", result.Cause)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_NoModelSelected(t *testing.T) {
|
||||
svc := NewService(config.NewConfigPersistence(t.TempDir()), nil)
|
||||
svc.cfg = &config.AIConfig{Enabled: true, OpenAIAPIKey: "sk-test"}
|
||||
|
||||
result := svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
if result.Success {
|
||||
t.Fatalf("expected failure when no model selected, got %+v", result)
|
||||
}
|
||||
if result.Cause != PatrolFailureCauseModelNotSelected {
|
||||
t.Fatalf("unexpected cause %q", result.Cause)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunPatrolToolPreflight_RequestShapeIncludesVerifyTool(t *testing.T) {
|
||||
// Locks the preflight request shape so a future refactor can't
|
||||
// silently strip the tool definition or tool_choice (which would
|
||||
// turn preflight into a connection-test that misses the original
|
||||
// failure mode).
|
||||
var captured map[string]interface{}
|
||||
h := newPatrolPreflightTestService(t, "openai:gpt-4o-mini", func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewDecoder(r.Body).Decode(&captured)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"id":"x","model":"gpt-4o-mini","choices":[{"finish_reason":"stop","message":{"role":"assistant","content":"ok"}}]}`))
|
||||
})
|
||||
defer h.close()
|
||||
|
||||
_ = h.svc.RunPatrolToolPreflight(context.Background(), "", "")
|
||||
|
||||
tools, ok := captured["tools"].([]interface{})
|
||||
if !ok || len(tools) != 1 {
|
||||
t.Fatalf("expected exactly one tool in preflight request, got %v", captured["tools"])
|
||||
}
|
||||
tool := tools[0].(map[string]interface{})
|
||||
fn := tool["function"].(map[string]interface{})
|
||||
if fn["name"] != patrolPreflightToolName {
|
||||
t.Fatalf("expected tool name %q, got %v", patrolPreflightToolName, fn["name"])
|
||||
}
|
||||
if captured["tool_choice"] == nil {
|
||||
t.Fatalf("expected tool_choice to be set in preflight request, got %v", captured)
|
||||
}
|
||||
}
|
||||
|
|
@ -5491,6 +5491,103 @@ func (h *AISettingsHandler) HandleForcePatrol(w http.ResponseWriter, r *http.Req
|
|||
}
|
||||
}
|
||||
|
||||
// aiPatrolPreflightResponse is the JSON shape returned by
|
||||
// HandlePatrolPreflight. It mirrors aiProviderTestResponse for the
|
||||
// classified-error fields and adds two preflight-specific signals:
|
||||
// ToolCallObserved (whether the model emitted a tool call) and
|
||||
// DurationMs (round-trip latency, useful for spotting slow providers).
|
||||
type aiPatrolPreflightResponse struct {
|
||||
Success bool `json:"success"`
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
ToolCallObserved bool `json:"tool_call_observed"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Message string `json:"message"`
|
||||
Cause string `json:"cause,omitempty"`
|
||||
Summary string `json:"summary,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Recommendation string `json:"recommendation,omitempty"`
|
||||
Action string `json:"action,omitempty"`
|
||||
}
|
||||
|
||||
type aiPatrolPreflightRequest struct {
|
||||
Provider string `json:"provider,omitempty"`
|
||||
Model string `json:"model,omitempty"`
|
||||
}
|
||||
|
||||
// HandlePatrolPreflight runs a one-shot tool-call round-trip against the
|
||||
// configured (or overridden) Patrol provider+model so operators can
|
||||
// verify tool calling actually works before relying on Patrol's
|
||||
// scheduled cadence (POST /api/ai/patrol/preflight).
|
||||
//
|
||||
// This is distinct from POST /api/ai/test/{provider}, which only lists
|
||||
// models and therefore can pass while Patrol fails 100% of runs (the
|
||||
// failure mode that bit Pulse for 33 days before the DeepSeek fix
|
||||
// landed).
|
||||
func (h *AISettingsHandler) HandlePatrolPreflight(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
if !ensureSettingsWriteScope(h.getConfig(r.Context()), w, r) {
|
||||
return
|
||||
}
|
||||
|
||||
aiService := h.GetAIService(r.Context())
|
||||
if aiService == nil {
|
||||
writePatrolServiceUnavailableResponse(w)
|
||||
return
|
||||
}
|
||||
|
||||
// Optional body: override the provider and/or model. Empty body is
|
||||
// allowed and means "use the configured Patrol provider+model".
|
||||
var body aiPatrolPreflightRequest
|
||||
if r.ContentLength > 0 {
|
||||
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||
http.Error(w, `{"error":"Invalid JSON body"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
if body.Provider != "" {
|
||||
if len(body.Provider) > 64 || !isValidProviderName(body.Provider) {
|
||||
http.Error(w, `{"error":"Invalid provider name"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
if len(body.Model) > 256 {
|
||||
http.Error(w, `{"error":"Model id too long"}`, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Tight timeout: preflight is a single round-trip, not a Patrol pass.
|
||||
// 30s matches the per-provider connection-test budget; in practice
|
||||
// this completes in well under 10s for any healthy provider.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
preflight := aiService.RunPatrolToolPreflight(ctx, body.Provider, body.Model)
|
||||
|
||||
response := aiPatrolPreflightResponse{
|
||||
Success: preflight.Success && preflight.ToolCallObserved,
|
||||
Provider: preflight.Provider,
|
||||
Model: preflight.Model,
|
||||
ToolCallObserved: preflight.ToolCallObserved,
|
||||
DurationMs: preflight.DurationMs,
|
||||
Message: preflight.Summary,
|
||||
Cause: string(preflight.Cause),
|
||||
Summary: preflight.Description,
|
||||
Recommendation: preflight.Recommendation,
|
||||
}
|
||||
if !response.Success {
|
||||
response.Action = "open_provider_settings"
|
||||
}
|
||||
|
||||
if err := utils.WriteJSONResponse(w, response); err != nil {
|
||||
log.Error().Err(err).Msg("Failed to write Patrol preflight response")
|
||||
}
|
||||
}
|
||||
|
||||
// HandleAcknowledgeFinding acknowledges a finding (POST /api/ai/patrol/acknowledge)
|
||||
// This marks the finding as seen but keeps it visible (dimmed). Auto-resolve removes it when condition clears.
|
||||
// This matches alert acknowledgement behavior for UI consistency.
|
||||
|
|
|
|||
|
|
@ -2876,3 +2876,74 @@ func TestAISettingsHandler_Approvals_RejectCrossOrgAccess(t *testing.T) {
|
|||
require.True(t, ok)
|
||||
require.Equal(t, approval.StatusPending, denyApp.Status)
|
||||
}
|
||||
|
||||
// ========================================
|
||||
// HandlePatrolPreflight tests
|
||||
// ========================================
|
||||
|
||||
func TestAISettingsHandler_PatrolPreflight_MethodNotAllowed(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmp := t.TempDir()
|
||||
cfg := &config.Config{DataPath: tmp}
|
||||
persistence := config.NewConfigPersistence(tmp)
|
||||
handler := newTestAISettingsHandler(cfg, persistence, nil)
|
||||
|
||||
req := newLoopbackRequest(http.MethodGet, "/api/ai/patrol/preflight", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandlePatrolPreflight(rec, req)
|
||||
|
||||
assert.Equal(t, http.StatusMethodNotAllowed, rec.Code)
|
||||
}
|
||||
|
||||
func TestAISettingsHandler_PatrolPreflight_AssistantDisabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmp := t.TempDir()
|
||||
cfg := &config.Config{DataPath: tmp}
|
||||
persistence := config.NewConfigPersistence(tmp)
|
||||
|
||||
aiCfg := config.NewDefaultAIConfig()
|
||||
aiCfg.Enabled = false
|
||||
if err := persistence.SaveAIConfig(*aiCfg); err != nil {
|
||||
t.Fatalf("SaveAIConfig: %v", err)
|
||||
}
|
||||
|
||||
handler := newTestAISettingsHandler(cfg, persistence, nil)
|
||||
|
||||
req := newLoopbackRequest(http.MethodPost, "/api/ai/patrol/preflight", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandlePatrolPreflight(rec, req)
|
||||
|
||||
require.Equal(t, http.StatusOK, rec.Code)
|
||||
|
||||
var resp struct {
|
||||
Success bool `json:"success"`
|
||||
Cause string `json:"cause"`
|
||||
Message string `json:"message"`
|
||||
ToolCallObserved bool `json:"tool_call_observed"`
|
||||
DurationMs int64 `json:"duration_ms"`
|
||||
Action string `json:"action"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
||||
assert.False(t, resp.Success)
|
||||
assert.False(t, resp.ToolCallObserved)
|
||||
assert.Equal(t, string(ai.PatrolFailureCauseAssistantDisabled), resp.Cause)
|
||||
assert.Equal(t, "open_provider_settings", resp.Action)
|
||||
}
|
||||
|
||||
func TestAISettingsHandler_PatrolPreflight_RejectsInvalidProviderName(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tmp := t.TempDir()
|
||||
cfg := &config.Config{DataPath: tmp}
|
||||
persistence := config.NewConfigPersistence(tmp)
|
||||
handler := newTestAISettingsHandler(cfg, persistence, nil)
|
||||
|
||||
req := newLoopbackRequest(http.MethodPost, "/api/ai/patrol/preflight",
|
||||
bytes.NewReader([]byte(`{"provider":"../etc/passwd"}`)))
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandlePatrolPreflight(rec, req)
|
||||
|
||||
assert.Equal(t, http.StatusBadRequest, rec.Code)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -600,6 +600,7 @@ var allRouteAllowlist = []string{
|
|||
"/api/ai/patrol/findings",
|
||||
"/api/ai/patrol/history",
|
||||
"/api/ai/patrol/run",
|
||||
"/api/ai/patrol/preflight",
|
||||
"/api/ai/patrol/acknowledge",
|
||||
"/api/ai/patrol/dismiss",
|
||||
"/api/ai/patrol/findings/note",
|
||||
|
|
|
|||
|
|
@ -111,6 +111,10 @@ func (r *Router) registerAIRelayRoutesGroup() {
|
|||
// SECURITY: AI Patrol read endpoints - require ai:execute scope
|
||||
r.mux.HandleFunc("/api/ai/patrol/history", RequireAuth(r.config, RequireScope(config.ScopeAIExecute, r.aiSettingsHandler.HandleGetFindingsHistory)))
|
||||
r.mux.HandleFunc("/api/ai/patrol/run", RequireAdmin(r.config, RequireScope(config.ScopeAIExecute, r.aiSettingsHandler.HandleForcePatrol)))
|
||||
// Patrol tool-call preflight: one-shot verification that the configured Patrol
|
||||
// provider+model actually supports tool calling. Distinct from /api/ai/test
|
||||
// (which only lists models) so a green test cannot mask a 100%-failing Patrol.
|
||||
r.mux.HandleFunc("/api/ai/patrol/preflight", RequirePermission(r.config, r.authorizer, auth.ActionWrite, auth.ResourceSettings, RequireScope(config.ScopeSettingsWrite, r.aiSettingsHandler.HandlePatrolPreflight)))
|
||||
// SECURITY: AI Patrol mutation endpoints - require ai:execute scope to prevent low-privilege tokens from
|
||||
// dismissing, suppressing, or otherwise hiding findings. This prevents attackers from blinding AI Patrol.
|
||||
r.mux.HandleFunc("/api/ai/patrol/acknowledge", RequireAuth(r.config, requireRelayMobileRuntimeRoute(relayMobileRoutePatrolAcknowledge, r.aiSettingsHandler.HandleAcknowledgeFinding)))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue