feat(core): share compression caches with OpenAI providers (#8418)

* feat(core): share compression cache with OpenAI providers

* chore: refresh generated settings schema

* fix(core): scope compression cache marker to OpenAI

* test(core): cover OpenAI cache guards

* fix(core): tighten OpenAI cache-sharing contracts

* test(core): pin OpenAI prompt cache guards

* fix(core): centralize prompt cache sharing gate

* test(core): assert absent cache sharing marker

* fix(core): partition OpenAI cache keys for subagents

* fix(core): cover default OpenAI cache endpoint

* fix(core): preserve prompt cache identity for forks

* fix: avoid OpenAI cache fields on DashScope default
This commit is contained in:
Dragon 2026-08-07 15:04:57 +08:00 committed by GitHub
parent 517b64a9e8
commit 6897ef7440
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 903 additions and 40 deletions

View file

@ -13,7 +13,8 @@ Compression first attempts a specialized single-turn request when all of the
following are true:
- the compression model is the current main model;
- the active provider is Anthropic or DashScope and cache control is enabled;
- the active provider is Anthropic or OpenAI-compatible and cache control is
enabled;
- the chat has a provider-reported prompt token count to anchor the estimate;
- the effective prompt token count plus the bounded compression output reserve
fits the model's context window.
@ -39,6 +40,21 @@ their cache identity differs from the main session. Media-bearing histories
use the shared path first so the unchanged provider-facing prefix can reuse the
main session's cache.
OpenAI-compatible endpoints use the same prefix-preserving request shape even
when their cache controls are unknown, allowing server-side automatic prefix
caches such as vLLM to match it. Qwen Code does not send provider-specific cache
fields to these endpoints. For the official OpenAI API, requests share a stable
session cache key; each concurrently running non-fork subagent appends its
stable agent identity so unrelated prefixes do not compete under the parent's
key, while a fork retains the session key because its inherited prefix matches
the parent. This follows OpenAI's recommendation to keep total traffic across
all prefixes for one key near 15 requests per minute and partition
higher-volume traffic with a
[stable mapping](https://developers.openai.com/api/docs/guides/prompt-caching#improve-cache-hit-rates-with-a-prompt-cache-key).
GPT-5.6 and later compression requests additionally mark the last reusable
user/tool boundaries and select explicit-only cache mode, so the new
compression directive does not move the effective cache breakpoint.
## Verification
Unit tests assert exact system, tools, full-history, and trailing-directive
@ -47,3 +63,19 @@ window preflight; media slimming after fallback; tool-call and
malformed-response fallback; and cancellation behavior. Provider testing
should compare the serialized request prefix and cached-token usage for the
main turn and compression request.
The reported live-provider cache figures are one-off validation evidence, not
a checked-in benchmark. To reproduce them, route one interactive CLI process
through a transparent OpenAI-compatible proxy, send a nonce-bearing main turn,
then run `/compress` and a follow-up that recalls the nonce. Record only message
roles, lengths and hashes, tool hashes, cache-field presence, and provider usage
(never prompts or authorization). Verify that the main and compression requests
share the same system, initial user, and tool hashes, and compute each hit rate
as `cached_tokens / prompt_tokens`. The validation used DashScope `qwen3.7-max`:
the main turn reported 18,944 / 19,627 cached tokens (96.52%), and compression
reported 18,944 / 20,409 (92.82%). A deterministic exact-prefix mock comparison
matched the longest byte-identical serialized prefix: clean `main` took the cold
summarizer path with zero cached tokens, while this design matched 28,032 /
29,034 prompt tokens (96.55%). Credentials, captured prompts, and the private
proxy harness are intentionally not committed; the unit tests are the durable
request-shape verification.

View file

@ -1649,7 +1649,7 @@ const SETTINGS_SCHEMA = {
category: 'Generation Configuration',
requiresRestart: false,
default: true,
description: 'Enable cache control for DashScope providers.',
description: 'Enable provider prompt-cache controls.',
parentKey: 'generationConfig',
showInDialog: false,
},

View file

@ -684,6 +684,54 @@ describe('BaseLlmClient', () => {
expect(result.hadToolCall).toBe(true);
});
it('forwards the prompt-cache-sharing marker to the provider request', async () => {
mockConfig.getContentGeneratorConfig.mockReturnValue({
model: 'test-model',
authType: AuthType.USE_OPENAI,
});
mockGenerateContentStream.mockImplementation(async () =>
mockTextStream(['summary']),
);
await client.generateText({
contents: [{ role: 'user', parts: [{ text: 'summarize' }] }],
model: 'test-model',
abortSignal: abortController.signal,
stream: true,
promptCacheSharing: true,
});
expect(mockGenerateContentStream).toHaveBeenCalledWith(
expect.objectContaining({ promptCacheSharing: true }),
'',
);
});
it.each([false, undefined])(
'does not forward the prompt-cache-sharing marker when disabled (%s)',
async (promptCacheSharing) => {
mockConfig.getContentGeneratorConfig.mockReturnValue({
model: 'test-model',
authType: AuthType.USE_OPENAI,
});
mockGenerateContentStream.mockImplementation(async () =>
mockTextStream(['summary']),
);
await client.generateText({
contents: [{ role: 'user', parts: [{ text: 'summarize' }] }],
model: 'test-model',
abortSignal: abortController.signal,
stream: true,
promptCacheSharing,
});
expect(mockGenerateContentStream.mock.calls[0]?.[0]).not.toHaveProperty(
'promptCacheSharing',
);
},
);
it('drops thought parts and tolerates a stream that omits usage', async () => {
async function* streamWithThought(): AsyncGenerator<GenerateContentResponse> {
yield createMockTextResponse('answer');

View file

@ -19,6 +19,7 @@ import type { Config } from '../config/config.js';
import type {
ContentGenerator,
ContentGeneratorConfig,
PromptCacheSharingParameters,
} from './contentGenerator.js';
import { AuthType, createContentGenerator } from './contentGenerator.js';
import type { ResolvedModelConfig } from '../models/types.js';
@ -108,6 +109,12 @@ export interface GenerateTextOptions {
* deltas are collected into the same `{ text, usage }` result.
*/
stream?: boolean;
/**
* Let the OpenAI adapter mark the unchanged history prefix for cache reuse.
* This is only for requests ending in a non-reusable trailing directive;
* the adapter deliberately excludes the final message from cache marking.
*/
promptCacheSharing?: boolean;
/**
* When true, throw instead of silently falling back to the main generator if
* a distinct generator for `model` can't be created (model not registered, or
@ -396,10 +403,11 @@ export class BaseLlmClient {
).slimmedHistory;
try {
const request = {
const request: PromptCacheSharingParameters = {
model: requestModel,
config: requestConfig,
contents: requestContents,
...(options.promptCacheSharing && { promptCacheSharing: true }),
};
// Both branches resolve to the same `{ text, usage }` shape so a single

View file

@ -61,6 +61,14 @@ export enum AuthType {
USE_ANTHROPIC = 'anthropic',
}
export type PromptCacheSharingParameters = GenerateContentParameters & {
/**
* Marks reusable history before a non-reusable trailing directive. The
* final message is deliberately excluded from cache breakpoints.
*/
promptCacheSharing?: boolean;
};
/**
* Supported input modalities for a model.
* Omitted or false fields mean the model does not support that input type.
@ -98,7 +106,7 @@ export type ContentGeneratorConfig = {
retryInitialDelayMs?: number; // Initial delay for stream rate-limit retries
retryMaxDelayMs?: number; // Maximum delay for stream rate-limit retries
retryErrorCodes?: number[]; // Additional error codes that trigger rate-limit retry
enableCacheControl?: boolean; // Enable cache control for DashScope providers
enableCacheControl?: boolean; // Enable provider prompt-cache controls
// Force `scope: 'global'` on Anthropic cache_control entries even when the
// base URL is not an Anthropic-native origin (e.g. proxy providers like
// Routify, OpenRouter). Requires the proxy to forward `cache_control` fields

View file

@ -43,6 +43,8 @@ import {
setGenAiUsageProvenance,
} from '../../telemetry/gen-ai-usage.js';
import { setToolCallPreparations } from '../tool-call-preparation.js';
import { runWithAgentContext } from '../../agents/runtime/agent-context.js';
import { runInForkContext } from '../../tools/agent/fork-subagent.js';
// Mock dependencies
const mockReportOpenAiRequest = vi.hoisted(() => vi.fn());
@ -4352,6 +4354,335 @@ describe('ContentGenerationPipeline', () => {
);
});
it('adds an official OpenAI session cache key to regular requests', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.5';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
const messages = [
{ role: 'user', content: 'Hello' },
] as OpenAI.Chat.ChatCompletionMessageParam[];
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
messages,
);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.5',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
},
'prompt-id',
);
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
expect.objectContaining({
prompt_cache_key: 'qwen-code:session-123',
messages,
}),
expect.anything(),
);
});
it('partitions official OpenAI cache keys for concurrent subagents', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
{ role: 'user', content: 'Hello from a subagent' },
] as OpenAI.Chat.ChatCompletionMessageParam[]);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await runWithAgentContext('Explore-a1b2c3d4', () =>
pipeline.execute(
{
model: 'gpt-5.6',
contents: [
{ role: 'user', parts: [{ text: 'Hello from a subagent' }] },
],
},
'prompt-id',
),
);
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
expect.objectContaining({
prompt_cache_key: 'qwen-code:session-123:Explore-a1b2c3d4',
}),
expect.anything(),
);
});
it('preserves the session cache key for forked agents', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
{ role: 'user', content: 'Hello from a fork' },
] as OpenAI.Chat.ChatCompletionMessageParam[]);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await runInForkContext(() =>
runWithAgentContext('fork-a1b2c3d4', () =>
pipeline.execute(
{
model: 'gpt-5.6',
contents: [
{ role: 'user', parts: [{ text: 'Hello from a fork' }] },
],
},
'prompt-id',
),
),
);
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
expect.objectContaining({
prompt_cache_key: 'qwen-code:session-123',
}),
expect.anything(),
);
});
it('does not add explicit cache fields to regular GPT-5.6 requests', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
const messages = [
{ role: 'system', content: 'You are helpful.' },
{ role: 'user', content: 'First question' },
{ role: 'assistant', content: 'First answer' },
{ role: 'user', content: 'Follow-up question' },
] as OpenAI.Chat.ChatCompletionMessageParam[];
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
messages,
);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.6',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
},
'prompt-id',
);
const sent = (mockClient.chat.completions.create as Mock).mock
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: unknown;
};
expect(sent.prompt_cache_key).toBe('qwen-code:session-123');
expect(sent.prompt_cache_options).toBeUndefined();
expect(sent.messages).toEqual(messages);
});
it('does not add official OpenAI cache fields when cache control is disabled', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockContentGeneratorConfig.enableCacheControl = false;
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
{ role: 'user', content: 'Hello' },
] as OpenAI.Chat.ChatCompletionMessageParam[]);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.6',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
promptCacheSharing: true,
},
'prompt-id',
);
const sent = (mockClient.chat.completions.create as Mock).mock
.calls[0]?.[0] as Record<string, unknown>;
expect(sent['prompt_cache_key']).toBeUndefined();
expect(sent['prompt_cache_options']).toBeUndefined();
});
it('does not add official OpenAI cache fields to third-party compatible endpoints', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.deepseek.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
const messages = [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
] as OpenAI.Chat.ChatCompletionMessageParam[];
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
messages,
);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.6',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
promptCacheSharing: true,
},
'prompt-id',
);
const sent = (mockClient.chat.completions.create as Mock).mock
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: unknown;
};
expect(sent.prompt_cache_key).toBeUndefined();
expect(sent.prompt_cache_options).toBeUndefined();
expect(sent.messages).toEqual(messages);
});
it('marks the stable official OpenAI prefix for GPT-5.6 compression', async () => {
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
const messages = [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
] as OpenAI.Chat.ChatCompletionMessageParam[];
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
messages,
);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.6',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
promptCacheSharing: true,
},
'prompt-id',
);
const sent = (mockClient.chat.completions.create as Mock).mock
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: { mode?: string };
};
expect(sent.prompt_cache_key).toBe('qwen-code:session-123');
expect(sent.prompt_cache_options).toEqual({ mode: 'explicit' });
expect(sent.messages[1]?.content).toEqual([
{
type: 'text',
text: 'main request',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
expect(sent.messages.at(-1)?.content).toBe('compression directive');
});
it('does not add official OpenAI cache fields when baseUrl is unset', async () => {
mockContentGeneratorConfig.baseUrl = undefined;
mockContentGeneratorConfig.model = 'gpt-5.6';
mockCliConfig = {
getSessionId: vi.fn().mockReturnValue('session-123'),
} as unknown as Config;
mockConfig.cliConfig = mockCliConfig;
pipeline = new ContentGenerationPipeline(mockConfig);
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
{ role: 'system', content: 'system' },
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
] as OpenAI.Chat.ChatCompletionMessageParam[]);
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
new GenerateContentResponse(),
);
(mockClient.chat.completions.create as Mock).mockResolvedValue({
id: 'test',
choices: [{ message: { content: 'response' } }],
});
await pipeline.execute(
{
model: 'gpt-5.6',
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
promptCacheSharing: true,
},
'prompt-id',
);
const sent = (mockClient.chat.completions.create as Mock).mock
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: { mode?: string };
};
expect(sent.prompt_cache_key).toBeUndefined();
expect(sent.prompt_cache_options).toBeUndefined();
expect(sent.messages[1]?.content).toBe('main request');
});
it('should pass arbitrary samplingParams keys through verbatim when the window has room (e.g. max_completion_tokens for GPT-5)', async () => {
// Arrange: user sets a GPT-5 / o-series shape in samplingParams.
// None of these are typed fields; all must appear on the wire because

View file

@ -9,9 +9,16 @@ import {
type GenerateContentParameters,
GenerateContentResponse,
} from '@google/genai';
import type { ContentGeneratorConfig } from '../contentGenerator.js';
import type {
ContentGeneratorConfig,
PromptCacheSharingParameters,
} from '../contentGenerator.js';
import { OpenAIContentConverter } from './converter.js';
import { DashScopeOpenAICompatibleProvider } from './provider/dashscope.js';
import {
applyOfficialOpenAIPromptCaching,
isOfficialOpenAIEndpoint,
} from './prefix-caching.js';
import { isDeepSeekHostname } from './provider/deepseek.js';
import { openaiRequestCaptureContext } from './requestCaptureContext.js';
import { StreamingToolCallParser } from './streamingToolCallParser.js';
@ -45,6 +52,8 @@ import {
reportOpenAiResponse,
type GenAiAttemptHandle,
} from '../../telemetry/gen-ai-request.js';
import { getCurrentAgentId } from '../../agents/runtime/agent-context.js';
import { isInForkExecution } from '../../tools/agent/fork-subagent.js';
const debugLogger = createDebugLogger('OPENAI_PIPELINE');
@ -446,7 +455,7 @@ export class ContentGenerationPipeline {
}
async execute(
request: GenerateContentParameters,
request: PromptCacheSharingParameters,
userPromptId: string,
): Promise<GenerateContentResponse> {
return this.executeWithErrorHandling(
@ -486,7 +495,7 @@ export class ContentGenerationPipeline {
}
async executeStream(
request: GenerateContentParameters,
request: PromptCacheSharingParameters,
userPromptId: string,
): Promise<AsyncGenerator<GenerateContentResponse>> {
return this.executeWithErrorHandling(
@ -938,7 +947,7 @@ export class ContentGenerationPipeline {
}
private async buildRequest(
request: GenerateContentParameters,
request: PromptCacheSharingParameters,
userPromptId: string,
context: RequestContext,
isStreaming: boolean,
@ -992,10 +1001,21 @@ export class ContentGenerationPipeline {
}
// Let provider enhance the request (e.g., add metadata, cache control)
const providerRequest = this.config.provider.buildRequest(
let providerRequest = this.config.provider.buildRequest(
baseRequest,
userPromptId,
);
if (
this.contentGeneratorConfig.enableCacheControl !== false &&
isOfficialOpenAIEndpoint(this.contentGeneratorConfig)
) {
providerRequest = applyOfficialOpenAIPromptCaching(
providerRequest,
this.config.cliConfig.getSessionId?.(),
request.promptCacheSharing === true,
isInForkExecution() ? undefined : (getCurrentAgentId() ?? undefined),
);
}
// Reasoning is disabled when either:
// - the per-request opt-out is set (forked queries for suggestions),
@ -1292,7 +1312,7 @@ export class ContentGenerationPipeline {
* Common error handling wrapper for execute methods
*/
private async executeWithErrorHandling<T>(
request: GenerateContentParameters,
request: PromptCacheSharingParameters,
userPromptId: string,
isStreaming: boolean,
executor: (

View file

@ -0,0 +1,306 @@
/**
* @license
* Copyright 2025 Qwen
* SPDX-License-Identifier: Apache-2.0
*/
import { describe, expect, it } from 'vitest';
import type OpenAI from 'openai';
import { AuthType, type ContentGeneratorConfig } from '../contentGenerator.js';
import {
applyOfficialOpenAIPromptCaching,
isOfficialOpenAIEndpoint,
supportsExplicitOpenAIPromptCaching,
supportsOpenAIPrefixCaching,
} from './prefix-caching.js';
function config(authType: AuthType, baseUrl?: string): ContentGeneratorConfig {
return { model: 'test-model', authType, baseUrl };
}
describe('supportsOpenAIPrefixCaching', () => {
it.each([
'https://api.openai.com/v1',
'https://api.deepseek.com/v1',
'https://proxy.example/v1',
])('accepts OpenAI-compatible endpoint %s', (baseUrl) => {
expect(
supportsOpenAIPrefixCaching(config(AuthType.USE_OPENAI, baseUrl)),
).toBe(true);
});
it('keeps non-OpenAI providers excluded', () => {
expect(supportsOpenAIPrefixCaching(config(AuthType.USE_GEMINI))).toBe(
false,
);
});
it('keeps Qwen OAuth on its existing DashScope path', () => {
expect(
supportsOpenAIPrefixCaching(
config(
AuthType.QWEN_OAUTH,
'https://dashscope.aliyuncs.com/compatible-mode/v1',
),
),
).toBe(true);
expect(
supportsOpenAIPrefixCaching(
config(AuthType.QWEN_OAUTH, 'https://proxy.example/v1'),
),
).toBe(true);
});
});
describe('official OpenAI prompt caching', () => {
it('recognizes only the official OpenAI API origin', () => {
expect(isOfficialOpenAIEndpoint(config(AuthType.USE_OPENAI))).toBe(false);
expect(
isOfficialOpenAIEndpoint(
config(AuthType.USE_OPENAI, 'https://api.openai.com/v1'),
),
).toBe(true);
expect(
isOfficialOpenAIEndpoint(
config(AuthType.USE_OPENAI, 'https://api.openai.com.evil.test/v1'),
),
).toBe(false);
expect(
isOfficialOpenAIEndpoint(
config(AuthType.QWEN_OAUTH, 'https://api.openai.com/v1'),
),
).toBe(false);
});
it.each([
['gpt-5', false],
['gpt-5.5', false],
['gpt-5.6', true],
['gpt-5.6.1', true],
['gpt-5.6-2026-08-01', true],
['gpt-6', true],
['o4-mini', false],
])('classifies explicit caching support for %s', (model, expected) => {
expect(supportsExplicitOpenAIPromptCaching(model)).toBe(expected);
});
it('adds a stable key and marks reusable boundaries for GPT-5.6 compression', () => {
const request = {
model: 'gpt-5.6',
messages: [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'calling a tool' },
{ role: 'tool', tool_call_id: 'call-1', content: 'tool result' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
true,
) as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: { mode?: string };
};
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
expect(result.prompt_cache_options).toEqual({ mode: 'explicit' });
expect(result.messages[1]?.content).toEqual([
{
type: 'text',
text: 'main request',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
expect(result.messages[3]?.content).toEqual([
{
type: 'text',
text: 'tool result',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
expect(result.messages.at(-1)?.content).toBe('compression directive');
});
it('uses automatic caching without unsupported fields on older models', () => {
const request = {
model: 'gpt-5.5',
messages: [
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
true,
) as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: unknown;
};
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
expect(result.prompt_cache_options).toBeUndefined();
expect(result.messages).toEqual(request.messages);
});
it('does not enable explicit mode without a reusable boundary', () => {
const request = {
model: 'gpt-5.6',
messages: [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'compression directive' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
true,
) as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: unknown;
};
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
expect(result.prompt_cache_options).toBeUndefined();
expect(result.messages).toEqual(request.messages);
});
it('partitions a session cache key by subagent identity', () => {
const request = {
model: 'gpt-5.6',
messages: [{ role: 'user', content: 'subagent request' }],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
false,
'Explore-a1b2c3d4',
);
expect(result.prompt_cache_key).toBe(
'qwen-code:session-123:Explore-a1b2c3d4',
);
});
it('does not rewrite regular GPT-5.6 requests', () => {
const request = {
model: 'gpt-5.6',
messages: [
{ role: 'user', content: 'main request' },
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'next request' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
false,
) as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: unknown;
};
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
expect(result.prompt_cache_options).toBeUndefined();
expect(result.messages).toEqual(request.messages);
});
it('preserves an existing prompt cache key', () => {
const request = {
model: 'gpt-5.6',
messages: [{ role: 'user', content: 'main request' }],
prompt_cache_key: 'custom-cache-key',
} as OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_key: string;
};
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
false,
);
expect(result.prompt_cache_key).toBe('custom-cache-key');
});
it('marks only the two most recent reusable boundaries', () => {
const request = {
model: 'gpt-5.6',
messages: [
{ role: 'system', content: 'system' },
{ role: 'user', content: 'old request' },
{ role: 'assistant', content: 'old tool call' },
{ role: 'tool', tool_call_id: 'call-1', content: 'old tool result' },
{ role: 'assistant', content: 'middle response' },
{ role: 'user', content: 'recent request' },
{ role: 'assistant', content: 'recent tool call' },
{
role: 'tool',
tool_call_id: 'call-2',
content: 'recent tool result',
},
{ role: 'user', content: 'compression directive' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
true,
);
expect(result.messages[1]?.content).toBe('old request');
expect(result.messages[3]?.content).toBe('old tool result');
expect(result.messages[5]?.content).toEqual([
{
type: 'text',
text: 'recent request',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
expect(result.messages[7]?.content).toEqual([
{
type: 'text',
text: 'recent tool result',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
});
it('marks the last part of array content at a reusable boundary', () => {
const request = {
model: 'gpt-5.6',
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'first part' },
{ type: 'text', text: 'last part' },
],
},
{ role: 'assistant', content: 'main response' },
{ role: 'user', content: 'compression directive' },
],
} as OpenAI.Chat.ChatCompletionCreateParams;
const result = applyOfficialOpenAIPromptCaching(
request,
'session-123',
true,
);
expect(result.messages[0]?.content).toEqual([
{ type: 'text', text: 'first part' },
{
type: 'text',
text: 'last part',
prompt_cache_breakpoint: { mode: 'explicit' },
},
]);
});
});

View file

@ -4,13 +4,112 @@
* SPDX-License-Identifier: Apache-2.0
*/
import type { ContentGeneratorConfig } from '../contentGenerator.js';
import { DashScopeOpenAICompatibleProvider } from './provider/dashscope.js';
import type OpenAI from 'openai';
import { AuthType, type ContentGeneratorConfig } from '../contentGenerator.js';
type OpenAIRequestWithExplicitCaching =
OpenAI.Chat.ChatCompletionCreateParams & {
prompt_cache_options?: {
mode?: 'implicit' | 'explicit';
ttl?: string;
};
};
type OpenAIContentPartWithBreakpoint = OpenAI.Chat.ChatCompletionContentPart & {
prompt_cache_breakpoint?: { mode: 'explicit' };
};
const CACHE_KEY_PREFIX = 'qwen-code:';
const EXPLICIT_BREAKPOINT_COUNT = 2;
export function supportsOpenAIPrefixCaching(
contentGeneratorConfig: ContentGeneratorConfig,
): boolean {
return DashScopeOpenAICompatibleProvider.isDashScopeProvider(
contentGeneratorConfig,
return (
contentGeneratorConfig.authType === AuthType.USE_OPENAI ||
contentGeneratorConfig.authType === AuthType.QWEN_OAUTH
);
}
export function isOfficialOpenAIEndpoint(
contentGeneratorConfig: ContentGeneratorConfig,
): boolean {
if (contentGeneratorConfig.authType !== AuthType.USE_OPENAI) return false;
const { baseUrl } = contentGeneratorConfig;
// An unset base URL is routed to the DashScope provider by provider
// selection, so only an explicit official OpenAI endpoint qualifies.
if (!baseUrl) return false;
try {
return new URL(baseUrl).hostname.toLowerCase() === 'api.openai.com';
} catch {
return false;
}
}
export function supportsExplicitOpenAIPromptCaching(model: string): boolean {
const match = /^gpt-(\d+)(?:\.(\d+))?(?:[-.]|$)/i.exec(model);
if (!match) return false;
const major = Number(match[1]);
const minor = Number(match[2] ?? 0);
return major > 5 || (major === 5 && minor >= 6);
}
function withCacheBreakpoint(
message: OpenAI.Chat.ChatCompletionMessageParam,
): OpenAI.Chat.ChatCompletionMessageParam | undefined {
if (message.role !== 'user' && message.role !== 'tool') return undefined;
const marker = { prompt_cache_breakpoint: { mode: 'explicit' as const } };
if (typeof message.content === 'string') {
return {
...message,
content: [{ type: 'text', text: message.content, ...marker }],
};
}
if (!Array.isArray(message.content) || message.content.length === 0) {
return undefined;
}
const content = [...message.content] as OpenAIContentPartWithBreakpoint[];
const lastIndex = content.length - 1;
content[lastIndex] = { ...content[lastIndex], ...marker };
return { ...message, content } as OpenAI.Chat.ChatCompletionMessageParam;
}
export function applyOfficialOpenAIPromptCaching(
request: OpenAI.Chat.ChatCompletionCreateParams,
sessionId: string | undefined,
cacheSharing: boolean,
cacheKeyPartition?: string,
): OpenAI.Chat.ChatCompletionCreateParams {
const result = { ...request } as OpenAIRequestWithExplicitCaching;
if (sessionId && !result.prompt_cache_key) {
const partition = cacheKeyPartition ? `:${cacheKeyPartition}` : '';
result.prompt_cache_key = `${CACHE_KEY_PREFIX}${sessionId}${partition}`;
}
if (!cacheSharing || !supportsExplicitOpenAIPromptCaching(request.model)) {
return result;
}
const messages = [...request.messages];
let marked = 0;
// Skip the trailing compression directive. Two earlier user/tool boundaries
// cover both the last main request and an unsent pending tool result.
for (
let index = messages.length - 2;
index >= 0 && marked < EXPLICIT_BREAKPOINT_COUNT;
index -= 1
) {
const message = messages[index];
const updated = message ? withCacheBreakpoint(message) : undefined;
if (!updated) continue;
messages[index] = updated;
marked += 1;
}
if (marked === 0) return result;
result.messages = messages;
result.prompt_cache_options = {
...result.prompt_cache_options,
mode: 'explicit',
};
return result;
}

View file

@ -19,7 +19,10 @@ import { tokenLimit } from '../core/tokenLimits.js';
import type { GeminiChat } from '../core/geminiChat.js';
import type { Config } from '../config/config.js';
import { ApprovalMode } from '../config/config.js';
import type { BaseLlmClient } from '../core/baseLlmClient.js';
import type {
BaseLlmClient,
GenerateTextOptions,
} from '../core/baseLlmClient.js';
import { AuthType } from '../core/contentGenerator.js';
import { PreCompactTrigger, PostCompactTrigger } from '../hooks/types.js';
import * as sideQueryModule from '../utils/sideQuery.js';
@ -2277,19 +2280,9 @@ describe('ChatCompressionService.compress cache sharing', () => {
});
expect(generateText).toHaveBeenCalledTimes(1);
const request = generateText.mock.calls[0]![0] as {
contents: Content[];
systemInstruction?: string;
config?: {
tools?: unknown;
thinkingConfig?: {
includeThoughts?: boolean;
thinkingBudget?: number;
};
maxOutputTokens?: number;
};
};
const request = generateText.mock.calls[0]![0] as GenerateTextOptions;
expect(request.systemInstruction).toBe(mainSystemInstruction);
expect(request.promptCacheSharing).toBe(true);
expect(request.config?.tools).toBe(tools);
expect(request.config?.thinkingConfig?.includeThoughts).toBe(true);
expect(request.config?.thinkingConfig?.thinkingBudget).toBe(
@ -2380,6 +2373,34 @@ describe('ChatCompressionService.compress cache sharing', () => {
},
);
it.each([
'https://api.openai.com/v1',
'https://api.deepseek.com/v1',
'https://proxy.example/v1',
])(
'uses cache sharing for OpenAI-compatible endpoint %s',
async (baseUrl) => {
const { chat, config, generateText } = makeFixture({
authType: AuthType.USE_OPENAI,
baseUrl,
});
const coldSpy = vi.spyOn(sideQueryModule, 'runSideQuery');
await new ChatCompressionService().compress(chat, {
promptId: 'p',
force: true,
config,
consecutiveFailures: 0,
originalTokenCount: 180_000,
});
expect(generateText).toHaveBeenCalledTimes(1);
const request = generateText.mock.calls[0]![0] as GenerateTextOptions;
expect(request.promptCacheSharing).toBe(true);
expect(coldSpy).not.toHaveBeenCalled();
},
);
it('appends a pending tool result after the cached history and before the directive', async () => {
const history: Content[] = [
{ role: 'user', parts: [{ text: 'read the file' }] },
@ -2811,13 +2832,6 @@ describe('ChatCompressionService.compress cache sharing', () => {
name: 'a provider without explicit cache support',
options: { authType: AuthType.USE_GEMINI },
},
{
name: 'a non-DashScope OpenAI-compatible provider',
options: {
authType: AuthType.USE_OPENAI,
baseUrl: 'https://api.openai.com/v1',
},
},
{
name: 'disabled cache control',
options: { enableCacheControl: false },

View file

@ -322,11 +322,7 @@ function supportsCompressionCacheSharing(config: Config): boolean {
const provider = config.getContentGeneratorConfig();
if (provider.enableCacheControl === false) return false;
if (provider.authType === AuthType.USE_ANTHROPIC) return true;
return (
(provider.authType === AuthType.QWEN_OAUTH ||
provider.authType === AuthType.USE_OPENAI) &&
supportsOpenAIPrefixCaching(provider)
);
return supportsOpenAIPrefixCaching(provider);
}
function hasStateSnapshot(summary: string): boolean {
@ -725,6 +721,7 @@ export class ChatCompressionService {
promptId,
stream: true,
maxAttempts: 1,
promptCacheSharing: true,
});
if (!sharedResult.hadToolCall && hasStateSnapshot(sharedResult.text)) {
summaryResult = sharedResult;

View file

@ -727,7 +727,7 @@
"minimum": 1
},
"enableCacheControl": {
"description": "Enable cache control for DashScope providers.",
"description": "Enable provider prompt-cache controls.",
"type": "boolean",
"default": true
},