fix: surface provider content filter and preserve context tokens (#963)

* fix: surface provider content filter and preserve context tokens

* fix: complete filtered turn handling across surfaces

- context: accumulate token estimate for zero-usage steps to preserve
  the tokenCount / tokenCountCoveredMessageCount invariant
- turn/goal: pause the goal when a turn is blocked by safety policy
- subagent: surface a filtered child turn as a distinct error
- acp: map filtered to the native ACP refusal stop reason
- tui: show a filtered-specific message in the btw panel
- cli: drop the redundant content_filter suffix from the error message
- tests: cover filtered across cli, web, acp, and goal flows
This commit is contained in:
7Sageer 2026-06-22 17:32:05 +08:00 committed by GitHub
parent 3b9938b4c3
commit 4292ae9f9b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 231 additions and 14 deletions

View file

@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---
Surface provider safety-policy blocks instead of silently treating them as completed turns, and prevent the context token count from dropping to zero after a filtered response.

View file

@ -795,5 +795,8 @@ function hasTurnId(event: Event): event is Event & { readonly turnId: number } {
function formatTurnEndedFailure(event: Extract<Event, { type: 'turn.ended' }>): string {
if (event.error !== undefined) return `${event.error.code}: ${event.error.message}`;
if (event.reason === 'filtered') {
return 'Provider safety policy blocked the response.';
}
return `Prompt turn ended with reason: ${event.reason}`;
}

View file

@ -202,5 +202,8 @@ function formatBtwTurnEnd(event: TurnEndedEvent): string {
if (event.reason === 'cancelled') {
return 'Interrupted by user';
}
if (event.reason === 'filtered') {
return 'Provider safety policy blocked the response.';
}
return `BTW turn ended with reason: ${event.reason}`;
}

View file

@ -325,6 +325,9 @@ export class SessionEventHandler {
if (event.reason === 'cancelled') {
this.markActiveAgentSwarmsCancelled();
}
if (event.reason === 'filtered') {
this.host.showStatus('Turn stopped: provider safety policy blocked the response.', 'error');
}
const todos = this.host.state.todoPanel.getTodos();
if (todos.length > 0 && todos.every((t) => t.status === 'done')) {
this.host.streamingUI.setTodoList([]);
@ -356,6 +359,15 @@ export class SessionEventHandler {
private handleStepCompleted(event: TurnStepCompletedEvent): void {
this.host.streamingUI.flushNow();
this.maybeShowDebugTiming(event);
if (event.providerFinishReason === 'filtered') {
this.host.showNotice(
'Provider safety policy blocked the response.',
`The model output was filtered (${event.rawFinishReason ?? 'content_filter'}).`,
);
return;
}
if (event.finishReason !== 'max_tokens') return;
const truncatedCount = this.host.streamingUI.markStepTruncated(

View file

@ -807,6 +807,31 @@ describe('runPrompt', () => {
expect(mocks.harnessClose).toHaveBeenCalled();
});
it('rejects with a friendly message when the provider filters the response', async () => {
mocks.session.prompt.mockImplementationOnce(async () => {
for (const handler of mocks.eventHandlers) {
handler(mocks.mainEvent({ type: 'turn.started', turnId: 2, origin: { kind: 'user' } }));
handler(
mocks.mainEvent({
type: 'turn.ended',
turnId: 2,
reason: 'filtered',
}),
);
}
});
await expect(
runPrompt(opts(), '1.2.3-test', {
stdout: { write: vi.fn(() => true) },
stderr: { write: vi.fn(() => true) },
}),
).rejects.toThrow('Provider safety policy blocked the response.');
expect(mocks.shutdownTelemetry).toHaveBeenCalled();
expect(mocks.harnessClose).toHaveBeenCalled();
});
it('approval fallback approves if an unexpected approval request reaches SDK', async () => {
await runPrompt(opts(), '1.2.3-test', {
stdout: { write: vi.fn(() => true) },

View file

@ -916,7 +916,7 @@ export function createAgentProjector(): AgentProjector {
sessionId,
messageId: msgId,
content: msg.content.map((c) => ({ ...c })),
status: reason === 'failed' ? 'error' : 'completed',
status: reason === 'failed' || reason === 'filtered' ? 'error' : 'completed',
durationMs,
});
}
@ -926,7 +926,8 @@ export function createAgentProjector(): AgentProjector {
const usageSnapshot = buildUsageSnapshot(s);
out.push({ type: 'sessionUsageUpdated', sessionId, usage: usageSnapshot });
const newStatus = reason === 'cancelled' ? 'aborted' : reason === 'failed' ? 'aborted' : 'idle';
const newStatus =
reason === 'cancelled' || reason === 'failed' || reason === 'filtered' ? 'aborted' : 'idle';
out.push({
type: 'sessionStatusChanged',
sessionId,

View file

@ -56,6 +56,11 @@ export function assistantDeltaToSessionUpdate(
* belong on the JSON-RPC error channel). Returning `end_turn` keeps the
* client unblocked; the caller is expected to log the `error` payload
* separately so the failure is observable in the agent logs.
* `filtered` `refusal`: the provider's safety policy blocked the
* response. ACP's `refusal` stop reason is the native signal for a
* model/provider decline, so the client can render the block instead of
* mistaking it for a clean `end_turn`. The caller additionally logs the
* block so it stays observable in the agent logs.
*/
export function turnEndReasonToStopReason(reason: TurnEndReason): AcpStopReason {
switch (reason) {
@ -65,6 +70,8 @@ export function turnEndReasonToStopReason(reason: TurnEndReason): AcpStopReason
return 'cancelled';
case 'failed':
return 'end_turn';
case 'filtered':
return 'refusal';
}
}

View file

@ -1151,6 +1151,13 @@ export class AcpSession {
return;
}
} else {
if (event.reason === 'filtered') {
// The provider's safety policy blocked the response. It is
// mapped to ACP `refusal` (see turnEndReasonToStopReason); log
// it here too so the block stays observable in the agent logs,
// mirroring the `failed` branch above.
log.warn('acp: turn ended with filtered reason', { sessionId });
}
argsByToolCall.clear();
startedToolCalls.clear();
// Drop the turnId so a late-arriving approval (e.g. an SDK

View file

@ -23,6 +23,7 @@ import {
type Session,
} from '@moonshot-ai/kimi-code-sdk';
import { turnEndReasonToStopReason } from '../src/events-map';
import { AcpServer } from '../src/server';
import { AUTHED_STATUS } from './_helpers/harness-stubs';
@ -258,4 +259,29 @@ describe('AcpServer error mapping', () => {
const response = await client.prompt({ sessionId, prompt: [textBlock('hi')] });
expect(response.stopReason).toBe('cancelled');
});
it('maps the filtered turn-end reason to ACP stopReason refusal', () => {
// ACP has a native `refusal` stop reason that matches a provider safety
// block; mapping filtered to anything else (e.g. end_turn) would let the
// client mistake the block for a clean turn.
expect(turnEndReasonToStopReason('filtered')).toBe('refusal');
});
it('resolves with refusal when turn.ended reason is filtered', async () => {
const sessionId = 'sess-filtered';
const { session, unsubscribeCount } = makeScriptedSession(sessionId, {
script: [
{ type: 'turn.ended', sessionId, agentId: 'main', turnId: 1, reason: 'filtered' } as Event,
],
});
const { agentStream, clientStream } = makeInMemoryStreamPair();
new AgentSideConnection((c) => new AcpServer(makeHarnessWithSession(session), c), agentStream);
const client = new ClientSideConnection(() => new StubClient(), clientStream);
await client.newSession({ cwd: '/tmp/x', mcpServers: [] });
const response = await client.prompt({ sessionId, prompt: [textBlock('hi')] });
expect(response.stopReason).toBe('refusal');
expect(unsubscribeCount()).toBe(1);
});
});

View file

@ -255,13 +255,26 @@ export class ContextMemory {
this.openSteps.delete(event.uuid);
if (event.usage !== undefined) {
const openStepIndex = openStep === undefined ? -1 : this._history.indexOf(openStep);
this._tokenCount =
const coveredCount =
openStepIndex === -1 ? this._history.length : openStepIndex + 1;
const totalUsage =
event.usage.inputCacheRead +
event.usage.inputCacheCreation +
event.usage.inputOther +
event.usage.output;
this.tokenCountCoveredMessageCount =
openStepIndex === -1 ? this._history.length : openStepIndex + 1;
if (totalUsage > 0) {
this._tokenCount = totalUsage;
} else {
// The provider reported zero usage (e.g. content filter). Do not
// overwrite the accumulated context token count with 0; add an
// estimate for the newly covered messages so the invariant between
// _tokenCount and tokenCountCoveredMessageCount stays intact.
const previousCoveredCount = this.tokenCountCoveredMessageCount;
this._tokenCount += estimateTokensForMessages(
this._history.slice(previousCoveredCount, coveredCount),
);
}
this.tokenCountCoveredMessageCount = coveredCount;
}
this.flushDeferredMessagesIfToolExchangeClosed();
return;

View file

@ -33,7 +33,7 @@ import {
type LoopTurnInterruptedEvent,
type LoopTurnStopReason,
} from '../../loop/index';
import type { AgentEvent, TurnEndedEvent } from '../../rpc';
import type { AgentEvent, TurnEndedEvent, TurnEndReason } from '../../rpc';
import type { TelemetryPropertyValue } from '../../telemetry';
import { abortable, isUserCancellation, userCancellationReason } from '../../utils/abort';
import { USER_PROMPT_ORIGIN, type PromptOrigin } from '../context';
@ -76,6 +76,7 @@ const GOAL_PROVIDER_AUTH_PAUSE_PREFIX = 'Paused after provider authentication er
const GOAL_PROVIDER_API_PAUSE_PREFIX = 'Paused after provider API error';
const GOAL_MODEL_CONFIG_PAUSE_PREFIX = 'Paused after model configuration error';
const GOAL_RUNTIME_PAUSE_PREFIX = 'Paused after runtime error';
const GOAL_PROVIDER_FILTERED_PAUSE_REASON = 'Paused after provider safety policy block';
/**
* The prompt the goal driver appends to start each continuation turn the
@ -325,7 +326,8 @@ export class TurnFlow {
if (
goalBecameActive &&
end.event.reason !== 'cancelled' &&
end.event.reason !== 'failed'
end.event.reason !== 'failed' &&
end.event.reason !== 'filtered'
) {
return await this.driveGoal(
this.allocateTurnId(),
@ -384,6 +386,10 @@ export class TurnFlow {
await this.agent.goal.pauseActiveGoal({ reason: goalFailurePauseReason(end.event.error) });
return end;
}
if (end.event.reason === 'filtered') {
await this.agent.goal.pauseActiveGoal({ reason: GOAL_PROVIDER_FILTERED_PAUSE_REASON });
return end;
}
if (end.blockedByUserPromptHook === true) {
await this.agent.goal.markBlocked({ reason: 'Blocked by UserPromptSubmit hook' });
return end;
@ -469,10 +475,12 @@ export class TurnFlow {
} else {
const stopReason = await this.runStepLoop(turnId, signal);
completedStopReason = stopReason;
const reason: TurnEndReason =
stopReason === 'aborted' ? 'cancelled' : stopReason === 'filtered' ? 'filtered' : 'completed';
ended = {
type: 'turn.ended',
turnId,
reason: stopReason === 'aborted' ? 'cancelled' : 'completed',
reason,
durationMs: Date.now() - startedAt,
};
}

View file

@ -97,7 +97,7 @@ interface PromptState {
body: PromptSubmission;
createdAt: string;
turnId: number | null;
/** Set on `turn.ended` for the top-level turn (reason='completed'|'failed'). */
/** Set on `turn.ended` for the top-level turn (reason='completed'|'failed'|'filtered'). */
completed: boolean;
/** Set on `turn.ended` with reason='cancelled' or after a successful abort RPC. */
aborted: boolean;
@ -205,7 +205,7 @@ function isTurnStarted(e: Event): e is Event & { type: 'turn.started'; turnId: n
function isTurnEnded(e: Event): e is Event & {
type: 'turn.ended';
turnId: number;
reason: 'completed' | 'cancelled' | 'failed';
reason: 'completed' | 'cancelled' | 'failed' | 'filtered';
} {
return (e as { type?: string }).type === 'turn.ended';
}
@ -853,7 +853,7 @@ export class PromptService
sessionId: sid,
promptId: state.promptId,
finishedAt: new Date().toISOString(),
reason: reason === 'failed' ? 'failed' : 'completed',
reason: reason === 'failed' || reason === 'filtered' ? 'failed' : 'completed',
};
this._active.delete(key);
// Fire typed listeners BEFORE publishing the synth event.

View file

@ -202,7 +202,7 @@ export class SessionService extends Disposable implements ISessionService {
case 'turn.ended': {
this._activeTurns.delete(sessionId);
const reason = (event as { reason?: string }).reason;
if (reason === 'cancelled' || reason === 'failed') {
if (reason === 'cancelled' || reason === 'failed' || reason === 'filtered') {
this._abortedTurns.add(sessionId);
} else {
this._abortedTurns.delete(sessionId);

View file

@ -468,6 +468,9 @@ async function runChildTurnToCompletion(child: Agent, signal: AbortSignal): Prom
const completion = await child.turn.waitForCurrentTurn(signal);
const turnEnded = completion.event;
if (turnEnded.reason !== 'completed') {
if (turnEnded.reason === 'filtered') {
throw new Error('Subagent turn blocked by provider safety policy');
}
if (turnEnded.error?.code === ErrorCodes.PROVIDER_RATE_LIMIT) {
throw providerRateLimitErrorFromPayload(turnEnded.error);
}

View file

@ -588,6 +588,41 @@ describe('Agent context', () => {
);
});
it('does not zero tokenCount when a filtered step reports zero usage', () => {
const ctx = testAgent();
ctx.configure();
ctx.appendAssistantTextWithUsage(1, 'previous answer', 1_000);
expect(ctx.agent.context.tokenCount).toBe(1_000);
const stepUuid = 'context-filtered-step';
ctx.agent.context.appendUserMessage([{ type: 'text', text: 'next prompt' }]);
ctx.dispatch({
type: 'context.append_loop_event',
event: { type: 'step.begin', uuid: stepUuid, turnId: '0', step: 2 },
});
ctx.dispatch({
type: 'context.append_loop_event',
event: {
type: 'step.end',
uuid: stepUuid,
turnId: '0',
step: 2,
usage: {
inputOther: 0,
output: 0,
inputCacheRead: 0,
inputCacheCreation: 0,
},
finishReason: 'filtered',
},
});
expect(ctx.agent.context.tokenCount).toBeGreaterThan(1_000);
expect(ctx.agent.context.tokenCountWithPending).toBeGreaterThanOrEqual(
ctx.agent.context.tokenCount,
);
});
it('undo only counts real user prompts, skipping background notifications', () => {
const ctx = testAgent();
ctx.configure();

View file

@ -425,6 +425,50 @@ describe('Agent turn flow', () => {
);
});
it('ends the turn with reason filtered when the provider filters a non-empty response', async () => {
const generate: GenerateFn = async () => ({
id: null,
message: {
role: 'assistant',
content: [{ type: 'text', text: 'some filtered text' }],
toolCalls: [],
},
usage: {
inputOther: 10,
output: 5,
inputCacheRead: 0,
inputCacheCreation: 0,
},
finishReason: 'filtered',
rawFinishReason: 'content_filter',
});
const ctx = testAgent({
generate,
...singleAttemptAgentOptions(),
});
ctx.configure();
await ctx.rpc.prompt({ input: [{ type: 'text', text: 'Trigger filtered response' }] });
const events = await ctx.untilTurnEnd();
expect(events).toContainEqual(
expect.objectContaining({
type: '[rpc]',
event: 'turn.ended',
args: expect.objectContaining({
reason: 'filtered',
}),
}),
);
expect(events).not.toContainEqual(
expect.objectContaining({
type: '[rpc]',
event: 'turn.ended',
args: expect.objectContaining({ reason: 'completed' }),
}),
);
});
it('emits a friendly model.not_configured error when no model is configured', async () => {
const ctx = testAgent();

View file

@ -490,6 +490,31 @@ describe('goal session end-to-end', () => {
expect(goal?.terminalReason).toBe('Paused after runtime error: unexpected failure');
});
it('pauses the goal on provider safety policy blocks', async () => {
const sessionDir = await makeTempDir();
const events: Array<Record<string, unknown>> = [];
const generate: NonNullable<AgentOptions['generate']> = async () => ({
id: null,
message: { role: 'assistant', content: [{ type: 'text', text: 'filtered' }], toolCalls: [] },
usage: { inputOther: 0, output: 0, inputCacheRead: 0, inputCacheCreation: 0 },
finishReason: 'filtered',
rawFinishReason: 'content_filter',
});
const { session, agent } = await setupSession(sessionDir, events, ['GetGoal'], generate);
const api = new SessionAPIImpl(session);
await api.createGoal({ agentId: 'main', objective: 'work' });
agent.turn.prompt([{ type: 'text', text: 'work' }]);
await agent.turn.waitForCurrentTurn();
const goal = (await api.getGoal({ agentId: 'main' })).goal;
expect(goal?.status).toBe('paused');
expect(goal?.terminalReason).toBe('Paused after provider safety policy block');
expect(events).toContainEqual(
expect.objectContaining({ type: 'turn.ended', reason: 'filtered' }),
);
});
it('blocks the goal when the initial prompt hook blocks the objective', async () => {
const sessionDir = await makeTempDir();
const events: Array<Record<string, unknown>> = [];

View file

@ -292,7 +292,7 @@ export interface McpOAuthAuthorizationUrlUpdateData {
readonly authorizationUrl: string;
}
export type TurnEndReason = 'completed' | 'cancelled' | 'failed';
export type TurnEndReason = 'completed' | 'cancelled' | 'failed' | 'filtered';
export interface AgentStatusUpdatedEvent {
readonly type: 'agent.status.updated';
@ -919,7 +919,7 @@ export const mcpOAuthAuthorizationUrlUpdateDataSchema = z.object({
authorizationUrl: z.string(),
}) satisfies z.ZodType<McpOAuthAuthorizationUrlUpdateData>;
export const turnEndReasonSchema = z.enum(['completed', 'cancelled', 'failed']) satisfies z.ZodType<TurnEndReason>;
export const turnEndReasonSchema = z.enum(['completed', 'cancelled', 'failed', 'filtered']) satisfies z.ZodType<TurnEndReason>;
export const agentStatusUpdatedEventSchema = z.object({
type: z.literal('agent.status.updated'),