mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-09-11 19:46:21 +00:00
feat(core): share compression caches with OpenAI providers (#8418)
* feat(core): share compression cache with OpenAI providers * chore: refresh generated settings schema * fix(core): scope compression cache marker to OpenAI * test(core): cover OpenAI cache guards * fix(core): tighten OpenAI cache-sharing contracts * test(core): pin OpenAI prompt cache guards * fix(core): centralize prompt cache sharing gate * test(core): assert absent cache sharing marker * fix(core): partition OpenAI cache keys for subagents * fix(core): cover default OpenAI cache endpoint * fix(core): preserve prompt cache identity for forks * fix: avoid OpenAI cache fields on DashScope default
This commit is contained in:
parent
517b64a9e8
commit
6897ef7440
12 changed files with 903 additions and 40 deletions
|
|
@ -13,7 +13,8 @@ Compression first attempts a specialized single-turn request when all of the
|
|||
following are true:
|
||||
|
||||
- the compression model is the current main model;
|
||||
- the active provider is Anthropic or DashScope and cache control is enabled;
|
||||
- the active provider is Anthropic or OpenAI-compatible and cache control is
|
||||
enabled;
|
||||
- the chat has a provider-reported prompt token count to anchor the estimate;
|
||||
- the effective prompt token count plus the bounded compression output reserve
|
||||
fits the model's context window.
|
||||
|
|
@ -39,6 +40,21 @@ their cache identity differs from the main session. Media-bearing histories
|
|||
use the shared path first so the unchanged provider-facing prefix can reuse the
|
||||
main session's cache.
|
||||
|
||||
OpenAI-compatible endpoints use the same prefix-preserving request shape even
|
||||
when their cache controls are unknown, allowing server-side automatic prefix
|
||||
caches such as vLLM to match it. Qwen Code does not send provider-specific cache
|
||||
fields to these endpoints. For the official OpenAI API, requests share a stable
|
||||
session cache key; each concurrently running non-fork subagent appends its
|
||||
stable agent identity so unrelated prefixes do not compete under the parent's
|
||||
key, while a fork retains the session key because its inherited prefix matches
|
||||
the parent. This follows OpenAI's recommendation to keep total traffic across
|
||||
all prefixes for one key near 15 requests per minute and partition
|
||||
higher-volume traffic with a
|
||||
[stable mapping](https://developers.openai.com/api/docs/guides/prompt-caching#improve-cache-hit-rates-with-a-prompt-cache-key).
|
||||
GPT-5.6 and later compression requests additionally mark the last reusable
|
||||
user/tool boundaries and select explicit-only cache mode, so the new
|
||||
compression directive does not move the effective cache breakpoint.
|
||||
|
||||
## Verification
|
||||
|
||||
Unit tests assert exact system, tools, full-history, and trailing-directive
|
||||
|
|
@ -47,3 +63,19 @@ window preflight; media slimming after fallback; tool-call and
|
|||
malformed-response fallback; and cancellation behavior. Provider testing
|
||||
should compare the serialized request prefix and cached-token usage for the
|
||||
main turn and compression request.
|
||||
|
||||
The reported live-provider cache figures are one-off validation evidence, not
|
||||
a checked-in benchmark. To reproduce them, route one interactive CLI process
|
||||
through a transparent OpenAI-compatible proxy, send a nonce-bearing main turn,
|
||||
then run `/compress` and a follow-up that recalls the nonce. Record only message
|
||||
roles, lengths and hashes, tool hashes, cache-field presence, and provider usage
|
||||
(never prompts or authorization). Verify that the main and compression requests
|
||||
share the same system, initial user, and tool hashes, and compute each hit rate
|
||||
as `cached_tokens / prompt_tokens`. The validation used DashScope `qwen3.7-max`:
|
||||
the main turn reported 18,944 / 19,627 cached tokens (96.52%), and compression
|
||||
reported 18,944 / 20,409 (92.82%). A deterministic exact-prefix mock comparison
|
||||
matched the longest byte-identical serialized prefix: clean `main` took the cold
|
||||
summarizer path with zero cached tokens, while this design matched 28,032 /
|
||||
29,034 prompt tokens (96.55%). Credentials, captured prompts, and the private
|
||||
proxy harness are intentionally not committed; the unit tests are the durable
|
||||
request-shape verification.
|
||||
|
|
|
|||
|
|
@ -1649,7 +1649,7 @@ const SETTINGS_SCHEMA = {
|
|||
category: 'Generation Configuration',
|
||||
requiresRestart: false,
|
||||
default: true,
|
||||
description: 'Enable cache control for DashScope providers.',
|
||||
description: 'Enable provider prompt-cache controls.',
|
||||
parentKey: 'generationConfig',
|
||||
showInDialog: false,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -684,6 +684,54 @@ describe('BaseLlmClient', () => {
|
|||
expect(result.hadToolCall).toBe(true);
|
||||
});
|
||||
|
||||
it('forwards the prompt-cache-sharing marker to the provider request', async () => {
|
||||
mockConfig.getContentGeneratorConfig.mockReturnValue({
|
||||
model: 'test-model',
|
||||
authType: AuthType.USE_OPENAI,
|
||||
});
|
||||
mockGenerateContentStream.mockImplementation(async () =>
|
||||
mockTextStream(['summary']),
|
||||
);
|
||||
|
||||
await client.generateText({
|
||||
contents: [{ role: 'user', parts: [{ text: 'summarize' }] }],
|
||||
model: 'test-model',
|
||||
abortSignal: abortController.signal,
|
||||
stream: true,
|
||||
promptCacheSharing: true,
|
||||
});
|
||||
|
||||
expect(mockGenerateContentStream).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ promptCacheSharing: true }),
|
||||
'',
|
||||
);
|
||||
});
|
||||
|
||||
it.each([false, undefined])(
|
||||
'does not forward the prompt-cache-sharing marker when disabled (%s)',
|
||||
async (promptCacheSharing) => {
|
||||
mockConfig.getContentGeneratorConfig.mockReturnValue({
|
||||
model: 'test-model',
|
||||
authType: AuthType.USE_OPENAI,
|
||||
});
|
||||
mockGenerateContentStream.mockImplementation(async () =>
|
||||
mockTextStream(['summary']),
|
||||
);
|
||||
|
||||
await client.generateText({
|
||||
contents: [{ role: 'user', parts: [{ text: 'summarize' }] }],
|
||||
model: 'test-model',
|
||||
abortSignal: abortController.signal,
|
||||
stream: true,
|
||||
promptCacheSharing,
|
||||
});
|
||||
|
||||
expect(mockGenerateContentStream.mock.calls[0]?.[0]).not.toHaveProperty(
|
||||
'promptCacheSharing',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('drops thought parts and tolerates a stream that omits usage', async () => {
|
||||
async function* streamWithThought(): AsyncGenerator<GenerateContentResponse> {
|
||||
yield createMockTextResponse('answer');
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import type { Config } from '../config/config.js';
|
|||
import type {
|
||||
ContentGenerator,
|
||||
ContentGeneratorConfig,
|
||||
PromptCacheSharingParameters,
|
||||
} from './contentGenerator.js';
|
||||
import { AuthType, createContentGenerator } from './contentGenerator.js';
|
||||
import type { ResolvedModelConfig } from '../models/types.js';
|
||||
|
|
@ -108,6 +109,12 @@ export interface GenerateTextOptions {
|
|||
* deltas are collected into the same `{ text, usage }` result.
|
||||
*/
|
||||
stream?: boolean;
|
||||
/**
|
||||
* Let the OpenAI adapter mark the unchanged history prefix for cache reuse.
|
||||
* This is only for requests ending in a non-reusable trailing directive;
|
||||
* the adapter deliberately excludes the final message from cache marking.
|
||||
*/
|
||||
promptCacheSharing?: boolean;
|
||||
/**
|
||||
* When true, throw instead of silently falling back to the main generator if
|
||||
* a distinct generator for `model` can't be created (model not registered, or
|
||||
|
|
@ -396,10 +403,11 @@ export class BaseLlmClient {
|
|||
).slimmedHistory;
|
||||
|
||||
try {
|
||||
const request = {
|
||||
const request: PromptCacheSharingParameters = {
|
||||
model: requestModel,
|
||||
config: requestConfig,
|
||||
contents: requestContents,
|
||||
...(options.promptCacheSharing && { promptCacheSharing: true }),
|
||||
};
|
||||
|
||||
// Both branches resolve to the same `{ text, usage }` shape so a single
|
||||
|
|
|
|||
|
|
@ -61,6 +61,14 @@ export enum AuthType {
|
|||
USE_ANTHROPIC = 'anthropic',
|
||||
}
|
||||
|
||||
export type PromptCacheSharingParameters = GenerateContentParameters & {
|
||||
/**
|
||||
* Marks reusable history before a non-reusable trailing directive. The
|
||||
* final message is deliberately excluded from cache breakpoints.
|
||||
*/
|
||||
promptCacheSharing?: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* Supported input modalities for a model.
|
||||
* Omitted or false fields mean the model does not support that input type.
|
||||
|
|
@ -98,7 +106,7 @@ export type ContentGeneratorConfig = {
|
|||
retryInitialDelayMs?: number; // Initial delay for stream rate-limit retries
|
||||
retryMaxDelayMs?: number; // Maximum delay for stream rate-limit retries
|
||||
retryErrorCodes?: number[]; // Additional error codes that trigger rate-limit retry
|
||||
enableCacheControl?: boolean; // Enable cache control for DashScope providers
|
||||
enableCacheControl?: boolean; // Enable provider prompt-cache controls
|
||||
// Force `scope: 'global'` on Anthropic cache_control entries even when the
|
||||
// base URL is not an Anthropic-native origin (e.g. proxy providers like
|
||||
// Routify, OpenRouter). Requires the proxy to forward `cache_control` fields
|
||||
|
|
|
|||
|
|
@ -43,6 +43,8 @@ import {
|
|||
setGenAiUsageProvenance,
|
||||
} from '../../telemetry/gen-ai-usage.js';
|
||||
import { setToolCallPreparations } from '../tool-call-preparation.js';
|
||||
import { runWithAgentContext } from '../../agents/runtime/agent-context.js';
|
||||
import { runInForkContext } from '../../tools/agent/fork-subagent.js';
|
||||
|
||||
// Mock dependencies
|
||||
const mockReportOpenAiRequest = vi.hoisted(() => vi.fn());
|
||||
|
|
@ -4352,6 +4354,335 @@ describe('ContentGenerationPipeline', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('adds an official OpenAI session cache key to regular requests', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.5';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
const messages = [
|
||||
{ role: 'user', content: 'Hello' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[];
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
|
||||
messages,
|
||||
);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.5',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
prompt_cache_key: 'qwen-code:session-123',
|
||||
messages,
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('partitions official OpenAI cache keys for concurrent subagents', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
|
||||
{ role: 'user', content: 'Hello from a subagent' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[]);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await runWithAgentContext('Explore-a1b2c3d4', () =>
|
||||
pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [
|
||||
{ role: 'user', parts: [{ text: 'Hello from a subagent' }] },
|
||||
],
|
||||
},
|
||||
'prompt-id',
|
||||
),
|
||||
);
|
||||
|
||||
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
prompt_cache_key: 'qwen-code:session-123:Explore-a1b2c3d4',
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves the session cache key for forked agents', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
|
||||
{ role: 'user', content: 'Hello from a fork' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[]);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await runInForkContext(() =>
|
||||
runWithAgentContext('fork-a1b2c3d4', () =>
|
||||
pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [
|
||||
{ role: 'user', parts: [{ text: 'Hello from a fork' }] },
|
||||
],
|
||||
},
|
||||
'prompt-id',
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
expect(mockClient.chat.completions.create).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
prompt_cache_key: 'qwen-code:session-123',
|
||||
}),
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not add explicit cache fields to regular GPT-5.6 requests', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
const messages = [
|
||||
{ role: 'system', content: 'You are helpful.' },
|
||||
{ role: 'user', content: 'First question' },
|
||||
{ role: 'assistant', content: 'First answer' },
|
||||
{ role: 'user', content: 'Follow-up question' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[];
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
|
||||
messages,
|
||||
);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
const sent = (mockClient.chat.completions.create as Mock).mock
|
||||
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: unknown;
|
||||
};
|
||||
expect(sent.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(sent.prompt_cache_options).toBeUndefined();
|
||||
expect(sent.messages).toEqual(messages);
|
||||
});
|
||||
|
||||
it('does not add official OpenAI cache fields when cache control is disabled', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockContentGeneratorConfig.enableCacheControl = false;
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
|
||||
{ role: 'user', content: 'Hello' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[]);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
promptCacheSharing: true,
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
const sent = (mockClient.chat.completions.create as Mock).mock
|
||||
.calls[0]?.[0] as Record<string, unknown>;
|
||||
expect(sent['prompt_cache_key']).toBeUndefined();
|
||||
expect(sent['prompt_cache_options']).toBeUndefined();
|
||||
});
|
||||
|
||||
it('does not add official OpenAI cache fields to third-party compatible endpoints', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.deepseek.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
const messages = [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[];
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
|
||||
messages,
|
||||
);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
promptCacheSharing: true,
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
const sent = (mockClient.chat.completions.create as Mock).mock
|
||||
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: unknown;
|
||||
};
|
||||
expect(sent.prompt_cache_key).toBeUndefined();
|
||||
expect(sent.prompt_cache_options).toBeUndefined();
|
||||
expect(sent.messages).toEqual(messages);
|
||||
});
|
||||
|
||||
it('marks the stable official OpenAI prefix for GPT-5.6 compression', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = 'https://api.openai.com/v1';
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
const messages = [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[];
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue(
|
||||
messages,
|
||||
);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
promptCacheSharing: true,
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
const sent = (mockClient.chat.completions.create as Mock).mock
|
||||
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: { mode?: string };
|
||||
};
|
||||
expect(sent.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(sent.prompt_cache_options).toEqual({ mode: 'explicit' });
|
||||
expect(sent.messages[1]?.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'main request',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
expect(sent.messages.at(-1)?.content).toBe('compression directive');
|
||||
});
|
||||
|
||||
it('does not add official OpenAI cache fields when baseUrl is unset', async () => {
|
||||
mockContentGeneratorConfig.baseUrl = undefined;
|
||||
mockContentGeneratorConfig.model = 'gpt-5.6';
|
||||
mockCliConfig = {
|
||||
getSessionId: vi.fn().mockReturnValue('session-123'),
|
||||
} as unknown as Config;
|
||||
mockConfig.cliConfig = mockCliConfig;
|
||||
pipeline = new ContentGenerationPipeline(mockConfig);
|
||||
(mockConverter.convertGeminiRequestToOpenAI as Mock).mockReturnValue([
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
] as OpenAI.Chat.ChatCompletionMessageParam[]);
|
||||
(mockConverter.convertOpenAIResponseToGemini as Mock).mockReturnValue(
|
||||
new GenerateContentResponse(),
|
||||
);
|
||||
(mockClient.chat.completions.create as Mock).mockResolvedValue({
|
||||
id: 'test',
|
||||
choices: [{ message: { content: 'response' } }],
|
||||
});
|
||||
|
||||
await pipeline.execute(
|
||||
{
|
||||
model: 'gpt-5.6',
|
||||
contents: [{ role: 'user', parts: [{ text: 'Hello' }] }],
|
||||
promptCacheSharing: true,
|
||||
},
|
||||
'prompt-id',
|
||||
);
|
||||
|
||||
const sent = (mockClient.chat.completions.create as Mock).mock
|
||||
.calls[0]?.[0] as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: { mode?: string };
|
||||
};
|
||||
expect(sent.prompt_cache_key).toBeUndefined();
|
||||
expect(sent.prompt_cache_options).toBeUndefined();
|
||||
expect(sent.messages[1]?.content).toBe('main request');
|
||||
});
|
||||
|
||||
it('should pass arbitrary samplingParams keys through verbatim when the window has room (e.g. max_completion_tokens for GPT-5)', async () => {
|
||||
// Arrange: user sets a GPT-5 / o-series shape in samplingParams.
|
||||
// None of these are typed fields; all must appear on the wire because
|
||||
|
|
|
|||
|
|
@ -9,9 +9,16 @@ import {
|
|||
type GenerateContentParameters,
|
||||
GenerateContentResponse,
|
||||
} from '@google/genai';
|
||||
import type { ContentGeneratorConfig } from '../contentGenerator.js';
|
||||
import type {
|
||||
ContentGeneratorConfig,
|
||||
PromptCacheSharingParameters,
|
||||
} from '../contentGenerator.js';
|
||||
import { OpenAIContentConverter } from './converter.js';
|
||||
import { DashScopeOpenAICompatibleProvider } from './provider/dashscope.js';
|
||||
import {
|
||||
applyOfficialOpenAIPromptCaching,
|
||||
isOfficialOpenAIEndpoint,
|
||||
} from './prefix-caching.js';
|
||||
import { isDeepSeekHostname } from './provider/deepseek.js';
|
||||
import { openaiRequestCaptureContext } from './requestCaptureContext.js';
|
||||
import { StreamingToolCallParser } from './streamingToolCallParser.js';
|
||||
|
|
@ -45,6 +52,8 @@ import {
|
|||
reportOpenAiResponse,
|
||||
type GenAiAttemptHandle,
|
||||
} from '../../telemetry/gen-ai-request.js';
|
||||
import { getCurrentAgentId } from '../../agents/runtime/agent-context.js';
|
||||
import { isInForkExecution } from '../../tools/agent/fork-subagent.js';
|
||||
|
||||
const debugLogger = createDebugLogger('OPENAI_PIPELINE');
|
||||
|
||||
|
|
@ -446,7 +455,7 @@ export class ContentGenerationPipeline {
|
|||
}
|
||||
|
||||
async execute(
|
||||
request: GenerateContentParameters,
|
||||
request: PromptCacheSharingParameters,
|
||||
userPromptId: string,
|
||||
): Promise<GenerateContentResponse> {
|
||||
return this.executeWithErrorHandling(
|
||||
|
|
@ -486,7 +495,7 @@ export class ContentGenerationPipeline {
|
|||
}
|
||||
|
||||
async executeStream(
|
||||
request: GenerateContentParameters,
|
||||
request: PromptCacheSharingParameters,
|
||||
userPromptId: string,
|
||||
): Promise<AsyncGenerator<GenerateContentResponse>> {
|
||||
return this.executeWithErrorHandling(
|
||||
|
|
@ -938,7 +947,7 @@ export class ContentGenerationPipeline {
|
|||
}
|
||||
|
||||
private async buildRequest(
|
||||
request: GenerateContentParameters,
|
||||
request: PromptCacheSharingParameters,
|
||||
userPromptId: string,
|
||||
context: RequestContext,
|
||||
isStreaming: boolean,
|
||||
|
|
@ -992,10 +1001,21 @@ export class ContentGenerationPipeline {
|
|||
}
|
||||
|
||||
// Let provider enhance the request (e.g., add metadata, cache control)
|
||||
const providerRequest = this.config.provider.buildRequest(
|
||||
let providerRequest = this.config.provider.buildRequest(
|
||||
baseRequest,
|
||||
userPromptId,
|
||||
);
|
||||
if (
|
||||
this.contentGeneratorConfig.enableCacheControl !== false &&
|
||||
isOfficialOpenAIEndpoint(this.contentGeneratorConfig)
|
||||
) {
|
||||
providerRequest = applyOfficialOpenAIPromptCaching(
|
||||
providerRequest,
|
||||
this.config.cliConfig.getSessionId?.(),
|
||||
request.promptCacheSharing === true,
|
||||
isInForkExecution() ? undefined : (getCurrentAgentId() ?? undefined),
|
||||
);
|
||||
}
|
||||
|
||||
// Reasoning is disabled when either:
|
||||
// - the per-request opt-out is set (forked queries for suggestions),
|
||||
|
|
@ -1292,7 +1312,7 @@ export class ContentGenerationPipeline {
|
|||
* Common error handling wrapper for execute methods
|
||||
*/
|
||||
private async executeWithErrorHandling<T>(
|
||||
request: GenerateContentParameters,
|
||||
request: PromptCacheSharingParameters,
|
||||
userPromptId: string,
|
||||
isStreaming: boolean,
|
||||
executor: (
|
||||
|
|
|
|||
|
|
@ -0,0 +1,306 @@
|
|||
/**
|
||||
* @license
|
||||
* Copyright 2025 Qwen
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import type OpenAI from 'openai';
|
||||
import { AuthType, type ContentGeneratorConfig } from '../contentGenerator.js';
|
||||
import {
|
||||
applyOfficialOpenAIPromptCaching,
|
||||
isOfficialOpenAIEndpoint,
|
||||
supportsExplicitOpenAIPromptCaching,
|
||||
supportsOpenAIPrefixCaching,
|
||||
} from './prefix-caching.js';
|
||||
|
||||
function config(authType: AuthType, baseUrl?: string): ContentGeneratorConfig {
|
||||
return { model: 'test-model', authType, baseUrl };
|
||||
}
|
||||
|
||||
describe('supportsOpenAIPrefixCaching', () => {
|
||||
it.each([
|
||||
'https://api.openai.com/v1',
|
||||
'https://api.deepseek.com/v1',
|
||||
'https://proxy.example/v1',
|
||||
])('accepts OpenAI-compatible endpoint %s', (baseUrl) => {
|
||||
expect(
|
||||
supportsOpenAIPrefixCaching(config(AuthType.USE_OPENAI, baseUrl)),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps non-OpenAI providers excluded', () => {
|
||||
expect(supportsOpenAIPrefixCaching(config(AuthType.USE_GEMINI))).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps Qwen OAuth on its existing DashScope path', () => {
|
||||
expect(
|
||||
supportsOpenAIPrefixCaching(
|
||||
config(
|
||||
AuthType.QWEN_OAUTH,
|
||||
'https://dashscope.aliyuncs.com/compatible-mode/v1',
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
supportsOpenAIPrefixCaching(
|
||||
config(AuthType.QWEN_OAUTH, 'https://proxy.example/v1'),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('official OpenAI prompt caching', () => {
|
||||
it('recognizes only the official OpenAI API origin', () => {
|
||||
expect(isOfficialOpenAIEndpoint(config(AuthType.USE_OPENAI))).toBe(false);
|
||||
expect(
|
||||
isOfficialOpenAIEndpoint(
|
||||
config(AuthType.USE_OPENAI, 'https://api.openai.com/v1'),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
isOfficialOpenAIEndpoint(
|
||||
config(AuthType.USE_OPENAI, 'https://api.openai.com.evil.test/v1'),
|
||||
),
|
||||
).toBe(false);
|
||||
expect(
|
||||
isOfficialOpenAIEndpoint(
|
||||
config(AuthType.QWEN_OAUTH, 'https://api.openai.com/v1'),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['gpt-5', false],
|
||||
['gpt-5.5', false],
|
||||
['gpt-5.6', true],
|
||||
['gpt-5.6.1', true],
|
||||
['gpt-5.6-2026-08-01', true],
|
||||
['gpt-6', true],
|
||||
['o4-mini', false],
|
||||
])('classifies explicit caching support for %s', (model, expected) => {
|
||||
expect(supportsExplicitOpenAIPromptCaching(model)).toBe(expected);
|
||||
});
|
||||
|
||||
it('adds a stable key and marks reusable boundaries for GPT-5.6 compression', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'calling a tool' },
|
||||
{ role: 'tool', tool_call_id: 'call-1', content: 'tool result' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
true,
|
||||
) as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: { mode?: string };
|
||||
};
|
||||
|
||||
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(result.prompt_cache_options).toEqual({ mode: 'explicit' });
|
||||
expect(result.messages[1]?.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'main request',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
expect(result.messages[3]?.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'tool result',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
expect(result.messages.at(-1)?.content).toBe('compression directive');
|
||||
});
|
||||
|
||||
it('uses automatic caching without unsupported fields on older models', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.5',
|
||||
messages: [
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
true,
|
||||
) as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: unknown;
|
||||
};
|
||||
|
||||
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(result.prompt_cache_options).toBeUndefined();
|
||||
expect(result.messages).toEqual(request.messages);
|
||||
});
|
||||
|
||||
it('does not enable explicit mode without a reusable boundary', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
true,
|
||||
) as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: unknown;
|
||||
};
|
||||
|
||||
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(result.prompt_cache_options).toBeUndefined();
|
||||
expect(result.messages).toEqual(request.messages);
|
||||
});
|
||||
|
||||
it('partitions a session cache key by subagent identity', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [{ role: 'user', content: 'subagent request' }],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
false,
|
||||
'Explore-a1b2c3d4',
|
||||
);
|
||||
|
||||
expect(result.prompt_cache_key).toBe(
|
||||
'qwen-code:session-123:Explore-a1b2c3d4',
|
||||
);
|
||||
});
|
||||
|
||||
it('does not rewrite regular GPT-5.6 requests', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'user', content: 'main request' },
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'next request' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
false,
|
||||
) as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: unknown;
|
||||
};
|
||||
|
||||
expect(result.prompt_cache_key).toBe('qwen-code:session-123');
|
||||
expect(result.prompt_cache_options).toBeUndefined();
|
||||
expect(result.messages).toEqual(request.messages);
|
||||
});
|
||||
|
||||
it('preserves an existing prompt cache key', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [{ role: 'user', content: 'main request' }],
|
||||
prompt_cache_key: 'custom-cache-key',
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_key: string;
|
||||
};
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
false,
|
||||
);
|
||||
|
||||
expect(result.prompt_cache_key).toBe('custom-cache-key');
|
||||
});
|
||||
|
||||
it('marks only the two most recent reusable boundaries', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{ role: 'system', content: 'system' },
|
||||
{ role: 'user', content: 'old request' },
|
||||
{ role: 'assistant', content: 'old tool call' },
|
||||
{ role: 'tool', tool_call_id: 'call-1', content: 'old tool result' },
|
||||
{ role: 'assistant', content: 'middle response' },
|
||||
{ role: 'user', content: 'recent request' },
|
||||
{ role: 'assistant', content: 'recent tool call' },
|
||||
{
|
||||
role: 'tool',
|
||||
tool_call_id: 'call-2',
|
||||
content: 'recent tool result',
|
||||
},
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
true,
|
||||
);
|
||||
|
||||
expect(result.messages[1]?.content).toBe('old request');
|
||||
expect(result.messages[3]?.content).toBe('old tool result');
|
||||
expect(result.messages[5]?.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'recent request',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
expect(result.messages[7]?.content).toEqual([
|
||||
{
|
||||
type: 'text',
|
||||
text: 'recent tool result',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('marks the last part of array content at a reusable boundary', () => {
|
||||
const request = {
|
||||
model: 'gpt-5.6',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'first part' },
|
||||
{ type: 'text', text: 'last part' },
|
||||
],
|
||||
},
|
||||
{ role: 'assistant', content: 'main response' },
|
||||
{ role: 'user', content: 'compression directive' },
|
||||
],
|
||||
} as OpenAI.Chat.ChatCompletionCreateParams;
|
||||
|
||||
const result = applyOfficialOpenAIPromptCaching(
|
||||
request,
|
||||
'session-123',
|
||||
true,
|
||||
);
|
||||
|
||||
expect(result.messages[0]?.content).toEqual([
|
||||
{ type: 'text', text: 'first part' },
|
||||
{
|
||||
type: 'text',
|
||||
text: 'last part',
|
||||
prompt_cache_breakpoint: { mode: 'explicit' },
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
|
@ -4,13 +4,112 @@
|
|||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import type { ContentGeneratorConfig } from '../contentGenerator.js';
|
||||
import { DashScopeOpenAICompatibleProvider } from './provider/dashscope.js';
|
||||
import type OpenAI from 'openai';
|
||||
import { AuthType, type ContentGeneratorConfig } from '../contentGenerator.js';
|
||||
|
||||
type OpenAIRequestWithExplicitCaching =
|
||||
OpenAI.Chat.ChatCompletionCreateParams & {
|
||||
prompt_cache_options?: {
|
||||
mode?: 'implicit' | 'explicit';
|
||||
ttl?: string;
|
||||
};
|
||||
};
|
||||
|
||||
type OpenAIContentPartWithBreakpoint = OpenAI.Chat.ChatCompletionContentPart & {
|
||||
prompt_cache_breakpoint?: { mode: 'explicit' };
|
||||
};
|
||||
|
||||
const CACHE_KEY_PREFIX = 'qwen-code:';
|
||||
const EXPLICIT_BREAKPOINT_COUNT = 2;
|
||||
|
||||
export function supportsOpenAIPrefixCaching(
|
||||
contentGeneratorConfig: ContentGeneratorConfig,
|
||||
): boolean {
|
||||
return DashScopeOpenAICompatibleProvider.isDashScopeProvider(
|
||||
contentGeneratorConfig,
|
||||
return (
|
||||
contentGeneratorConfig.authType === AuthType.USE_OPENAI ||
|
||||
contentGeneratorConfig.authType === AuthType.QWEN_OAUTH
|
||||
);
|
||||
}
|
||||
|
||||
export function isOfficialOpenAIEndpoint(
|
||||
contentGeneratorConfig: ContentGeneratorConfig,
|
||||
): boolean {
|
||||
if (contentGeneratorConfig.authType !== AuthType.USE_OPENAI) return false;
|
||||
const { baseUrl } = contentGeneratorConfig;
|
||||
// An unset base URL is routed to the DashScope provider by provider
|
||||
// selection, so only an explicit official OpenAI endpoint qualifies.
|
||||
if (!baseUrl) return false;
|
||||
try {
|
||||
return new URL(baseUrl).hostname.toLowerCase() === 'api.openai.com';
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function supportsExplicitOpenAIPromptCaching(model: string): boolean {
|
||||
const match = /^gpt-(\d+)(?:\.(\d+))?(?:[-.]|$)/i.exec(model);
|
||||
if (!match) return false;
|
||||
const major = Number(match[1]);
|
||||
const minor = Number(match[2] ?? 0);
|
||||
return major > 5 || (major === 5 && minor >= 6);
|
||||
}
|
||||
|
||||
function withCacheBreakpoint(
|
||||
message: OpenAI.Chat.ChatCompletionMessageParam,
|
||||
): OpenAI.Chat.ChatCompletionMessageParam | undefined {
|
||||
if (message.role !== 'user' && message.role !== 'tool') return undefined;
|
||||
const marker = { prompt_cache_breakpoint: { mode: 'explicit' as const } };
|
||||
if (typeof message.content === 'string') {
|
||||
return {
|
||||
...message,
|
||||
content: [{ type: 'text', text: message.content, ...marker }],
|
||||
};
|
||||
}
|
||||
if (!Array.isArray(message.content) || message.content.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
const content = [...message.content] as OpenAIContentPartWithBreakpoint[];
|
||||
const lastIndex = content.length - 1;
|
||||
content[lastIndex] = { ...content[lastIndex], ...marker };
|
||||
return { ...message, content } as OpenAI.Chat.ChatCompletionMessageParam;
|
||||
}
|
||||
|
||||
export function applyOfficialOpenAIPromptCaching(
|
||||
request: OpenAI.Chat.ChatCompletionCreateParams,
|
||||
sessionId: string | undefined,
|
||||
cacheSharing: boolean,
|
||||
cacheKeyPartition?: string,
|
||||
): OpenAI.Chat.ChatCompletionCreateParams {
|
||||
const result = { ...request } as OpenAIRequestWithExplicitCaching;
|
||||
if (sessionId && !result.prompt_cache_key) {
|
||||
const partition = cacheKeyPartition ? `:${cacheKeyPartition}` : '';
|
||||
result.prompt_cache_key = `${CACHE_KEY_PREFIX}${sessionId}${partition}`;
|
||||
}
|
||||
if (!cacheSharing || !supportsExplicitOpenAIPromptCaching(request.model)) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const messages = [...request.messages];
|
||||
let marked = 0;
|
||||
// Skip the trailing compression directive. Two earlier user/tool boundaries
|
||||
// cover both the last main request and an unsent pending tool result.
|
||||
for (
|
||||
let index = messages.length - 2;
|
||||
index >= 0 && marked < EXPLICIT_BREAKPOINT_COUNT;
|
||||
index -= 1
|
||||
) {
|
||||
const message = messages[index];
|
||||
const updated = message ? withCacheBreakpoint(message) : undefined;
|
||||
if (!updated) continue;
|
||||
messages[index] = updated;
|
||||
marked += 1;
|
||||
}
|
||||
if (marked === 0) return result;
|
||||
|
||||
result.messages = messages;
|
||||
result.prompt_cache_options = {
|
||||
...result.prompt_cache_options,
|
||||
mode: 'explicit',
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,10 @@ import { tokenLimit } from '../core/tokenLimits.js';
|
|||
import type { GeminiChat } from '../core/geminiChat.js';
|
||||
import type { Config } from '../config/config.js';
|
||||
import { ApprovalMode } from '../config/config.js';
|
||||
import type { BaseLlmClient } from '../core/baseLlmClient.js';
|
||||
import type {
|
||||
BaseLlmClient,
|
||||
GenerateTextOptions,
|
||||
} from '../core/baseLlmClient.js';
|
||||
import { AuthType } from '../core/contentGenerator.js';
|
||||
import { PreCompactTrigger, PostCompactTrigger } from '../hooks/types.js';
|
||||
import * as sideQueryModule from '../utils/sideQuery.js';
|
||||
|
|
@ -2277,19 +2280,9 @@ describe('ChatCompressionService.compress cache sharing', () => {
|
|||
});
|
||||
|
||||
expect(generateText).toHaveBeenCalledTimes(1);
|
||||
const request = generateText.mock.calls[0]![0] as {
|
||||
contents: Content[];
|
||||
systemInstruction?: string;
|
||||
config?: {
|
||||
tools?: unknown;
|
||||
thinkingConfig?: {
|
||||
includeThoughts?: boolean;
|
||||
thinkingBudget?: number;
|
||||
};
|
||||
maxOutputTokens?: number;
|
||||
};
|
||||
};
|
||||
const request = generateText.mock.calls[0]![0] as GenerateTextOptions;
|
||||
expect(request.systemInstruction).toBe(mainSystemInstruction);
|
||||
expect(request.promptCacheSharing).toBe(true);
|
||||
expect(request.config?.tools).toBe(tools);
|
||||
expect(request.config?.thinkingConfig?.includeThoughts).toBe(true);
|
||||
expect(request.config?.thinkingConfig?.thinkingBudget).toBe(
|
||||
|
|
@ -2380,6 +2373,34 @@ describe('ChatCompressionService.compress cache sharing', () => {
|
|||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
'https://api.openai.com/v1',
|
||||
'https://api.deepseek.com/v1',
|
||||
'https://proxy.example/v1',
|
||||
])(
|
||||
'uses cache sharing for OpenAI-compatible endpoint %s',
|
||||
async (baseUrl) => {
|
||||
const { chat, config, generateText } = makeFixture({
|
||||
authType: AuthType.USE_OPENAI,
|
||||
baseUrl,
|
||||
});
|
||||
const coldSpy = vi.spyOn(sideQueryModule, 'runSideQuery');
|
||||
|
||||
await new ChatCompressionService().compress(chat, {
|
||||
promptId: 'p',
|
||||
force: true,
|
||||
config,
|
||||
consecutiveFailures: 0,
|
||||
originalTokenCount: 180_000,
|
||||
});
|
||||
|
||||
expect(generateText).toHaveBeenCalledTimes(1);
|
||||
const request = generateText.mock.calls[0]![0] as GenerateTextOptions;
|
||||
expect(request.promptCacheSharing).toBe(true);
|
||||
expect(coldSpy).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it('appends a pending tool result after the cached history and before the directive', async () => {
|
||||
const history: Content[] = [
|
||||
{ role: 'user', parts: [{ text: 'read the file' }] },
|
||||
|
|
@ -2811,13 +2832,6 @@ describe('ChatCompressionService.compress cache sharing', () => {
|
|||
name: 'a provider without explicit cache support',
|
||||
options: { authType: AuthType.USE_GEMINI },
|
||||
},
|
||||
{
|
||||
name: 'a non-DashScope OpenAI-compatible provider',
|
||||
options: {
|
||||
authType: AuthType.USE_OPENAI,
|
||||
baseUrl: 'https://api.openai.com/v1',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'disabled cache control',
|
||||
options: { enableCacheControl: false },
|
||||
|
|
|
|||
|
|
@ -322,11 +322,7 @@ function supportsCompressionCacheSharing(config: Config): boolean {
|
|||
const provider = config.getContentGeneratorConfig();
|
||||
if (provider.enableCacheControl === false) return false;
|
||||
if (provider.authType === AuthType.USE_ANTHROPIC) return true;
|
||||
return (
|
||||
(provider.authType === AuthType.QWEN_OAUTH ||
|
||||
provider.authType === AuthType.USE_OPENAI) &&
|
||||
supportsOpenAIPrefixCaching(provider)
|
||||
);
|
||||
return supportsOpenAIPrefixCaching(provider);
|
||||
}
|
||||
|
||||
function hasStateSnapshot(summary: string): boolean {
|
||||
|
|
@ -725,6 +721,7 @@ export class ChatCompressionService {
|
|||
promptId,
|
||||
stream: true,
|
||||
maxAttempts: 1,
|
||||
promptCacheSharing: true,
|
||||
});
|
||||
if (!sharedResult.hadToolCall && hasStateSnapshot(sharedResult.text)) {
|
||||
summaryResult = sharedResult;
|
||||
|
|
|
|||
|
|
@ -727,7 +727,7 @@
|
|||
"minimum": 1
|
||||
},
|
||||
"enableCacheControl": {
|
||||
"description": "Enable cache control for DashScope providers.",
|
||||
"description": "Enable provider prompt-cache controls.",
|
||||
"type": "boolean",
|
||||
"default": true
|
||||
},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue