vrr/zed
2
0
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2026-07-10 00:13:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18

Support provider-side compaction in the language model clients (#59145)

Browse source 
Adds provider-side context compaction support to the Anthropic and
OpenAI API clients. Client plumbing only; not wired into the UI.

Release Notes:

- N/A
This commit is contained in:
Agus Zubiaga 2026-06-16 10:06:20 -03:00 • committed by GitHub
parent 4c0717facc
commit 33a54ce423
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 605 additions and 25 deletions
Download patch file Download diff file Expand all files Collapse all files

12
crates/agent/src/thread.rs
View file

@ -2917,7 +2917,8 @@ impl Thread {
| LanguageModelCompletionEvent::ReasoningDetails(_)
| LanguageModelCompletionEvent::ToolUse(_)
| LanguageModelCompletionEvent::ToolUseJsonParseError { .. }
| LanguageModelCompletionEvent::StartMessage { .. } => {}
| LanguageModelCompletionEvent::StartMessage { .. }
| LanguageModelCompletionEvent::Compaction(_) => {}
}
}
@ -3119,7 +3120,7 @@ impl Thread {
Stop(StopReason::Refusal) => return Err(CompletionError::Refusal.into()),
Stop(StopReason::MaxTokens) => return Err(CompletionError::MaxTokens.into()),
Stop(StopReason::ToolUse | StopReason::EndTurn) => {}
Started | Queued { .. } => {}
Started | Queued { .. } | Compaction(_) => {}
}
Ok(None)
@ -3710,6 +3711,7 @@ impl Thread {
thinking_allowed: self.thinking_enabled || !model.supports_disabling_thinking(),
thinking_effort: self.thinking_effort.clone(),
speed: self.speed(),
compact_at_tokens: None,
};
log::debug!("Completion request built successfully");
@ -4329,7 +4331,8 @@ fn user_message_byte_len(message: &LanguageModelRequestMessage) -> usize {
MessageContent::Thinking { .. }
| MessageContent::RedactedThinking(_)
| MessageContent::ToolResult(_)
| MessageContent::ToolUse(_) => 0,
| MessageContent::ToolUse(_)
| MessageContent::Compaction(_) => 0,
})
.sum()
}
@ -4365,7 +4368,8 @@ fn truncate_user_message_to_byte_budget(
MessageContent::Thinking { .. }
| MessageContent::RedactedThinking(_)
| MessageContent::ToolResult(_)
| MessageContent::ToolUse(_) => {}
| MessageContent::ToolUse(_)
| MessageContent::Compaction(_) => {}
}
}

2
crates/agent_ui/src/buffer_codegen.rs
View file

@ -551,6 +551,7 @@ impl CodegenAlternative {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
}
}))
}
@ -631,6 +632,7 @@ impl CodegenAlternative {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
}
}))
}

1
crates/agent_ui/src/terminal_inline_assistant.rs
View file

@ -274,6 +274,7 @@ impl TerminalInlineAssistant {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
}
}))
}

93
crates/anthropic/src/anthropic.rs
View file

@ -1,5 +1,6 @@
use std::io;
use std::str::FromStr;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{Context as _, Result};
@ -23,6 +24,9 @@ const FAST_MODE_BETA_HEADER: &str = "fast-mode-2026-02-01";
pub const FABLE_MODEL_ID_PREFIX: &str = "claude-fable-5";
pub const FABLE_FALLBACK_MODEL_ID: &str = "claude-opus-4-8";
/// <https://platform.claude.com/docs/en/build-with-claude/compaction>
pub const COMPACTION_BETA_HEADER: &str = "compact-2026-01-12";
#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
pub enum AnthropicModelMode {
@ -95,6 +99,7 @@ pub struct Model {
pub supports_adaptive_thinking: bool,
pub supports_images: bool,
pub supports_speed: bool,
pub supports_compaction: bool,
pub supported_effort_levels: Vec<Effort>,
/// A model id to substitute when invoking tools, used for models that
/// don't support tool calling natively.
@ -156,10 +161,25 @@ impl Model {
"claude-opus-4-6" | "claude-opus-4-7" | "claude-opus-4-8"
);
// <https://platform.claude.com/docs/en/build-with-claude/compaction#supported-models>
let supports_compaction = matches!(
entry.id.as_str(),
"claude-fable-5"
| "claude-mythos-5"
| "claude-mythos-preview"
| "claude-opus-4-8"
| "claude-opus-4-7"
| "claude-opus-4-6"
| "claude-sonnet-4-6"
);
let mut extra_beta_headers = Vec::new();
if supports_speed {
extra_beta_headers.push(FAST_MODE_BETA_HEADER.to_string());
}
if supports_compaction {
extra_beta_headers.push(COMPACTION_BETA_HEADER.to_string());
}
Self {
display_name: entry.display_name,
@ -172,6 +192,7 @@ impl Model {
supports_adaptive_thinking,
supports_images,
supports_speed,
supports_compaction,
supported_effort_levels,
tool_override: None,
extra_beta_headers,
@ -619,6 +640,12 @@ pub enum RequestContent {
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
},
#[serde(rename = "compaction")]
Compaction {
content: Option<Arc<str>>,
#[serde(skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
},
}
#[derive(Debug, Serialize, Deserialize)]
@ -650,6 +677,8 @@ pub enum ResponseContent {
name: String,
input: serde_json::Value,
},
#[serde(rename = "compaction")]
Compaction { content: Option<Arc<str>> },
}
#[derive(Debug, Serialize, Deserialize)]
@ -728,6 +757,32 @@ pub enum StringOrContents {
Content(Vec<RequestContent>),
}
/// Server-side context management configuration.
///
/// <https://platform.claude.com/docs/en/build-with-claude/compaction>
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ContextManagement {
pub edits: Vec<ContextManagementEdit>,
}
/// A context management edit strategy.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "type")]
pub enum ContextManagementEdit {
#[serde(rename = "compact_20260112")]
Compact {
#[serde(default, skip_serializing_if = "Option::is_none")]
trigger: Option<CompactionTrigger>,
},
}
/// When to trigger server-side compaction.
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum CompactionTrigger {
InputTokens { value: u64 },
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Request {
pub model: String,
@ -748,6 +803,8 @@ pub struct Request {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_control: Option<CacheControl>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context_management: Option<ContextManagement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_config: Option<OutputConfig>,
@ -793,6 +850,34 @@ pub struct Usage {
pub cache_creation_input_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_tokens: Option<u64>,
/// Only populated when a new compaction is triggered during the request.
/// The top-level token fields exclude compaction iterations, so total
/// billable usage is the sum across all iterations.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub iterations: Option<Vec<UsageIteration>>,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct UsageIteration {
#[serde(rename = "type")]
pub iteration_type: UsageIterationType,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_tokens: Option<u64>,
}
#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum UsageIterationType {
Compaction,
Message,
#[serde(other)]
Unknown,
}
#[derive(Debug, Serialize, Deserialize)]
@ -845,6 +930,8 @@ pub enum ContentDelta {
SignatureDelta { signature: String },
#[serde(rename = "input_json_delta")]
InputJsonDelta { partial_json: String },
#[serde(rename = "compaction_delta")]
CompactionDelta { content: Option<Arc<str>> },
}
#[derive(Debug, Serialize, Deserialize)]
@ -1116,7 +1203,11 @@ mod tests {
));
assert!(model.supports_speed);
assert_eq!(model.beta_headers().as_deref(), Some(FAST_MODE_BETA_HEADER));
let beta_headers = model
.beta_headers()
.expect("model should have beta headers");
assert!(beta_headers.contains(FAST_MODE_BETA_HEADER));
assert!(beta_headers.contains(COMPACTION_BETA_HEADER));
}
#[test]

199
crates/anthropic/src/completion.rs
View file

@ -2,9 +2,10 @@ use anyhow::Result;
use collections::HashMap;
use futures::{Stream, StreamExt};
use language_model_core::{
LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelProviderName,
LanguageModelRequest, LanguageModelToolChoice, LanguageModelToolResultContent,
LanguageModelToolUse, MessageContent, Role, StopReason, TokenUsage,
CompactionContent, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelProviderName, LanguageModelRequest, LanguageModelToolChoice,
LanguageModelToolResultContent, LanguageModelToolUse, MessageContent, Role, StopReason,
TokenUsage,
util::{fix_streamed_json, parse_tool_arguments},
};
use std::pin::Pin;
@ -12,9 +13,10 @@ use std::str::FromStr;
use crate::{
AdaptiveThinkingDisplay, AnthropicError, AnthropicModelMode, CacheControl, CacheControlType,
CacheTtl, ContentDelta, Event, ImageSource, Message, RequestContent, ResponseContent,
StringOrContents, Thinking, Tool, ToolChoice, ToolResultContent, ToolResultPart, Usage,
completion_error_from_anthropic, completion_error_from_anthropic_api,
CacheTtl, CompactionTrigger, ContentDelta, ContextManagement, ContextManagementEdit, Event,
ImageSource, Message, RequestContent, ResponseContent, StringOrContents, Thinking, Tool,
ToolChoice, ToolResultContent, ToolResultPart, Usage, completion_error_from_anthropic,
completion_error_from_anthropic_api,
};
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
@ -47,6 +49,10 @@ fn set_cache_control(content: &mut RequestContent, cache_control: Option<CacheCo
| RequestContent::ToolResult {
cache_control: target,
..
}
| RequestContent::Compaction {
cache_control: target,
..
} => {
*target = cache_control;
true
@ -148,6 +154,17 @@ fn to_anthropic_content(content: MessageContent) -> Option<RequestContent> {
cache_control: None,
})
}
MessageContent::Compaction(CompactionContent::Summary { content }) => {
Some(RequestContent::Compaction {
content,
cache_control: None,
})
}
// Encrypted compaction blocks come from other providers, and a
// Pending block is a streaming-only UI signal; neither is replayed.
MessageContent::Compaction(
CompactionContent::Encrypted { .. } | CompactionContent::Pending,
) => None,
}
}
@ -317,6 +334,11 @@ pub fn into_anthropic(
temperature: request.temperature.or(Some(default_temperature)),
top_k: None,
top_p: None,
context_management: request.compact_at_tokens.map(|value| ContextManagement {
edits: vec![ContextManagementEdit::Compact {
trigger: Some(CompactionTrigger::InputTokens { value }),
}],
}),
}
}
@ -385,6 +407,11 @@ impl AnthropicEventMapper {
);
Vec::new()
}
ResponseContent::Compaction { content } => {
vec![Ok(LanguageModelCompletionEvent::Compaction(
CompactionContent::Summary { content },
))]
}
},
Event::ContentBlockDelta { index, delta } => match delta {
ContentDelta::TextDelta { text } => {
@ -402,6 +429,11 @@ impl AnthropicEventMapper {
signature: Some(signature),
})]
}
ContentDelta::CompactionDelta { content } => {
vec![Ok(LanguageModelCompletionEvent::Compaction(
CompactionContent::Summary { content },
))]
}
ContentDelta::InputJsonDelta { partial_json } => {
if let Some(tool_use) = self.tool_uses_by_index.get_mut(&index) {
tool_use.input_json.push_str(&partial_json);
@ -534,8 +566,10 @@ fn convert_usage(usage: &Usage) -> TokenUsage {
#[cfg(test)]
mod tests {
use super::*;
use crate::AnthropicModelMode;
use language_model_core::{LanguageModelImage, LanguageModelRequestMessage, MessageContent};
use crate::{AnthropicModelMode, UsageIteration, UsageIterationType};
use language_model_core::{
ANTHROPIC_PROVIDER_NAME, LanguageModelImage, LanguageModelRequestMessage, MessageContent,
};
#[test]
fn test_caching_uses_top_level_auto_and_long_lived_prefix() {
@ -573,6 +607,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let anthropic_request = into_anthropic(
@ -593,7 +628,8 @@ mod tests {
| RequestContent::Thinking { cache_control, .. }
| RequestContent::Image { cache_control, .. }
| RequestContent::ToolUse { cache_control, .. }
| RequestContent::ToolResult { cache_control, .. } => *cache_control,
| RequestContent::ToolResult { cache_control, .. }
| RequestContent::Compaction { cache_control, .. } => *cache_control,
RequestContent::RedactedThinking { .. } => None,
};
assert!(
@ -677,6 +713,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let anthropic_request = into_anthropic(
@ -734,6 +771,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: Some("xhigh".into()),
speed: None,
compact_at_tokens: None,
};
let anthropic_request = into_anthropic(
@ -785,6 +823,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let anthropic_request = into_anthropic(
@ -822,6 +861,7 @@ mod tests {
tool_choice: None,
thinking_allowed: true,
speed: None,
compact_at_tokens: None,
};
request.messages.push(LanguageModelRequestMessage {
role: Role::Assistant,
@ -916,4 +956,145 @@ mod tests {
should be omitted entirely"
);
}
#[test]
fn test_compact_at_tokens_maps_to_context_management() {
let request = LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
cache: false,
reasoning_details: None,
}],
compact_at_tokens: Some(100_000),
..Default::default()
};
let anthropic_request = into_anthropic(
request,
"claude-sonnet-4-5".to_string(),
1.0,
4096,
AnthropicModelMode::Default,
AnthropicPromptCacheMode::Disabled,
);
assert_eq!(
serde_json::to_value(&anthropic_request.context_management).unwrap(),
serde_json::json!({
"edits": [{
"type": "compact_20260112",
"trigger": { "type": "input_tokens", "value": 100_000 }
}]
})
);
}
#[test]
fn test_no_context_management_without_compact_at_tokens() {
let result =
request_with_assistant_content(vec![MessageContent::Text("Response".to_string())]);
assert!(result.context_management.is_none());
}
#[test]
fn test_compaction_content_replayed_as_compaction_block() {
let result = request_with_assistant_content(vec![
MessageContent::Compaction(CompactionContent::Summary {
content: Some("Summary of the conversation so far.".into()),
}),
MessageContent::Text("Response".to_string()),
]);
let assistant_message = result
.messages
.iter()
.find(|m| m.role == crate::Role::Assistant)
.expect("assistant message should exist");
assert_eq!(
serde_json::to_value(&assistant_message.content[0]).unwrap(),
serde_json::json!({
"type": "compaction",
"content": "Summary of the conversation so far."
})
);
}
#[test]
fn test_event_mapper_maps_compaction_block_and_deltas() {
let mut mapper = AnthropicEventMapper::new(ANTHROPIC_PROVIDER_NAME);
let start_event: Event = serde_json::from_value(serde_json::json!({
"type": "content_block_start",
"index": 0,
"content_block": { "type": "compaction", "content": null }
}))
.unwrap();
let delta_event: Event = serde_json::from_value(serde_json::json!({
"type": "content_block_delta",
"index": 0,
"delta": { "type": "compaction_delta", "content": "Summary chunk" }
}))
.unwrap();
let mut events = Vec::new();
events.extend(mapper.map_event(start_event));
events.extend(mapper.map_event(delta_event));
let events = events
.into_iter()
.collect::<Result<Vec<_>, _>>()
.expect("all events should map successfully");
assert_eq!(
events,
vec![
LanguageModelCompletionEvent::Compaction(CompactionContent::Summary {
content: None
}),
LanguageModelCompletionEvent::Compaction(CompactionContent::Summary {
content: Some("Summary chunk".into())
}),
]
);
}
#[test]
fn test_usage_iterations_parsed_from_message_delta() {
let event: Event = serde_json::from_value(serde_json::json!({
"type": "message_delta",
"delta": { "stop_reason": "end_turn", "stop_sequence": null },
"usage": {
"input_tokens": 100,
"output_tokens": 39,
"iterations": [
{ "type": "compaction", "input_tokens": 180000, "output_tokens": 1200 },
{ "type": "message", "input_tokens": 100, "output_tokens": 39 }
]
}
}))
.unwrap();
let Event::MessageDelta { usage, .. } = event else {
panic!("expected message_delta event");
};
let iterations = usage.iterations.as_deref().expect("iterations expected");
assert!(matches!(
iterations[0],
UsageIteration {
iteration_type: UsageIterationType::Compaction,
input_tokens: Some(180000),
..
}
));
assert!(matches!(
iterations[1],
UsageIteration {
iteration_type: UsageIterationType::Message,
input_tokens: Some(100),
..
}
));
}
}

2
crates/cloud_llm_client/src/cloud_llm_client.rs
View file

@ -306,6 +306,8 @@ pub struct LanguageModel {
pub supports_disabling_thinking: bool,
#[serde(default)]
pub supports_fast_mode: bool,
#[serde(default)]
pub supports_server_side_compaction: bool,
pub supported_effort_levels: Vec<SupportedEffortLevel>,
#[serde(default)]
pub supports_streaming_tools: bool,

3
crates/edit_prediction_cli/src/anthropic_client.rs
View file

@ -48,6 +48,7 @@ impl PlainLlmClient {
tool_choice: None,
system: None,
cache_control: None,
context_management: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),
@ -90,6 +91,7 @@ impl PlainLlmClient {
tool_choice: None,
system: None,
cache_control: None,
context_management: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),
@ -587,6 +589,7 @@ impl BatchingLlmClient {
tool_choice: None,
system: None,
cache_control: None,
context_management: None,
metadata: None,
output_config: None,
stop_sequences: Vec::new(),

1
crates/git_ui/src/git_panel.rs
View file

@ -2843,6 +2843,7 @@ impl GitPanel {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let stream = model.stream_completion_text(request, cx);

2
crates/google_ai/src/completion.rs
View file

@ -53,7 +53,7 @@ pub fn into_google(
MessageContent::Thinking { .. } => {
vec![]
}
MessageContent::RedactedThinking(_) => vec![],
MessageContent::RedactedThinking(_) | MessageContent::Compaction(_) => vec![],
MessageContent::Image(image) => {
vec![Part::InlineDataPart(InlineDataPart {
inline_data: GenerativeContentBlob {

10
crates/language_model/src/fake_provider.rs
View file

@ -127,6 +127,7 @@ pub struct FakeLanguageModel {
supports_disabling_thinking: AtomicBool,
supports_streaming_tools: AtomicBool,
supports_images: AtomicBool,
supports_server_side_compaction: AtomicBool,
max_token_count: AtomicU64,
max_output_tokens: AtomicU64,
}
@ -144,6 +145,7 @@ impl Default for FakeLanguageModel {
supports_disabling_thinking: AtomicBool::new(true),
supports_streaming_tools: AtomicBool::new(false),
supports_images: AtomicBool::new(false),
supports_server_side_compaction: AtomicBool::new(false),
max_token_count: AtomicU64::new(1_000_000),
max_output_tokens: AtomicU64::new(0),
}
@ -190,6 +192,10 @@ impl FakeLanguageModel {
self.supports_images.store(supports, SeqCst);
}
pub fn set_supports_server_side_compaction(&self, supports: bool) {
self.supports_server_side_compaction.store(supports, SeqCst);
}
pub fn set_max_token_count(&self, count: u64) {
self.max_token_count.store(count, SeqCst);
}
@ -308,6 +314,10 @@ impl LanguageModel for FakeLanguageModel {
self.supports_images.load(SeqCst)
}
fn supports_server_side_compaction(&self) -> bool {
self.supports_server_side_compaction.load(SeqCst)
}
fn supports_thinking(&self) -> bool {
self.supports_thinking.load(SeqCst)
}

7
crates/language_model/src/language_model.rs
View file

@ -109,6 +109,12 @@ pub trait LanguageModel: Send + Sync {
.find(|effort_level| effort_level.is_default)
}
/// Whether this model supports provider-side automatic context
/// compaction (requested via `LanguageModelRequest::compact_at_tokens`).
fn supports_server_side_compaction(&self) -> bool {
false
}
/// Whether this model supports images
fn supports_images(&self) -> bool;
@ -195,6 +201,7 @@ pub trait LanguageModel: Send + Sync {
Ok(LanguageModelCompletionEvent::ToolUseJsonParseError {
..
}) => None,
Ok(LanguageModelCompletionEvent::Compaction(_)) => None,
Ok(LanguageModelCompletionEvent::UsageUpdate(token_usage)) => {
*last_token_usage.lock() = token_usage;
None

1
crates/language_model_core/src/language_model_core.rs
View file

@ -56,6 +56,7 @@ pub enum LanguageModelCompletionEvent {
},
ReasoningDetails(serde_json::Value),
UsageUpdate(TokenUsage),
Compaction(CompactionContent),
}
impl LanguageModelCompletionEvent {

16
crates/language_model_core/src/provider.rs
View file

@ -19,3 +19,19 @@ pub const X_AI_PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderN
pub const ZED_CLOUD_PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("zed.dev");
pub const ZED_CLOUD_PROVIDER_NAME: LanguageModelProviderName =
LanguageModelProviderName::new("Zed");
pub fn provider_name_for_id(provider_id: &LanguageModelProviderId) -> LanguageModelProviderName {
if provider_id == &OPEN_AI_PROVIDER_ID {
OPEN_AI_PROVIDER_NAME
} else if provider_id == &ANTHROPIC_PROVIDER_ID {
ANTHROPIC_PROVIDER_NAME
} else if provider_id == &GOOGLE_PROVIDER_ID {
GOOGLE_PROVIDER_NAME
} else if provider_id == &X_AI_PROVIDER_ID {
X_AI_PROVIDER_NAME
} else if provider_id == &ZED_CLOUD_PROVIDER_ID {
ZED_CLOUD_PROVIDER_NAME
} else {
LanguageModelProviderName(provider_id.0.clone())
}
}

21
crates/language_model_core/src/request.rs
View file

@ -260,6 +260,19 @@ pub enum MessageContent {
Image(LanguageModelImage),
ToolUse(LanguageModelToolUse),
ToolResult(LanguageModelToolResult),
Compaction(CompactionContent),
}
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Hash)]
pub enum CompactionContent {
Pending,
Summary {
content: Option<Arc<str>>,
},
Encrypted {
id: Option<Arc<str>>,
encrypted_content: Arc<str>,
},
}
impl MessageContent {
@ -270,7 +283,8 @@ impl MessageContent {
MessageContent::ToolResult(tool_result) => tool_result.is_content_empty(),
MessageContent::RedactedThinking(_)
| MessageContent::ToolUse(_)
| MessageContent::Image(_) => false,
| MessageContent::Image(_)
| MessageContent::Compaction(_) => false,
}
}
}
@ -316,7 +330,8 @@ impl LanguageModelRequestMessage {
}
MessageContent::RedactedThinking(_)
| MessageContent::ToolUse(_)
| MessageContent::Image(_) => {}
| MessageContent::Image(_)
| MessageContent::Compaction(_) => {}
}
}
buffer
@ -370,6 +385,8 @@ pub struct LanguageModelRequest {
pub thinking_allowed: bool,
pub thinking_effort: Option<String>,
pub speed: Option<Speed>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub compact_at_tokens: Option<u64>,
}
#[derive(

5
crates/language_models/src/provider/anthropic.rs
View file

@ -350,6 +350,7 @@ fn available_model_to_anthropic_model(available: &AvailableModel) -> anthropic::
supports_adaptive_thinking,
supports_images: true,
supports_speed: false,
supports_compaction: false,
supported_effort_levels: if supports_adaptive_thinking {
vec![
anthropic::Effort::Low,
@ -471,6 +472,10 @@ impl LanguageModel for AnthropicModel {
}
}
fn supports_server_side_compaction(&self) -> bool {
self.model.supports_compaction
}
fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
self.model
.supported_effort_levels

1
crates/language_models/src/provider/anthropic_compatible.rs
View file

@ -70,6 +70,7 @@ fn available_model_to_anthropic_model(available: &AvailableModel) -> anthropic::
supports_adaptive_thinking: false,
supports_images: available.capabilities.images,
supports_speed: false,
supports_compaction: false,
supported_effort_levels: Vec::new(),
tool_override: available.tool_override.clone(),
extra_beta_headers: available.extra_beta_headers.clone(),

1
crates/language_models/src/provider/bedrock.rs
View file

@ -882,6 +882,7 @@ pub fn into_bedrock(
None
}
}
MessageContent::Compaction(_) => None,
MessageContent::Thinking { text, signature } => {
if model.contains(Model::DeepSeekR1.request_id()) {
// DeepSeekR1 doesn't support thinking blocks

1
crates/language_models/src/provider/cloud.rs
View file

@ -776,6 +776,7 @@ mod tests {
supports_thinking: false,
supports_disabling_thinking: false,
supports_fast_mode: false,
supports_server_side_compaction: false,
supported_effort_levels: Vec::new(),
supports_streaming_tools: false,
supports_parallel_tool_calls: false,

4
crates/language_models/src/provider/copilot_chat.rs
View file

@ -1062,7 +1062,8 @@ fn into_copilot_chat(
| MessageContent::ToolUse(_)
| MessageContent::RedactedThinking(_)
| MessageContent::ToolResult(_)
| MessageContent::Image(_) => None,
| MessageContent::Image(_)
| MessageContent::Compaction(_) => None,
}) {
buffer.push_str(string);
}
@ -1189,6 +1190,7 @@ fn into_copilot_responses(
thinking_allowed,
thinking_effort,
speed: _,
compact_at_tokens: _,
} = request;
let mut input_items: Vec<responses::ResponseInputItem> = Vec::new();

1
crates/language_models/src/provider/deepseek.rs
View file

@ -392,6 +392,7 @@ pub fn into_deepseek(
}
MessageContent::RedactedThinking(_) => {}
MessageContent::Image(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::ToolUse(tool_use) => {
let tool_call = deepseek::ToolCall {
id: tool_use.id.to_string(),

1
crates/language_models/src/provider/lmstudio.rs
View file

@ -354,6 +354,7 @@ impl LmStudioLanguageModel {
),
MessageContent::Thinking { .. } => {}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::Image(image) => {
add_message_content_part(
lmstudio::MessagePart::Image {

5
crates/language_models/src/provider/mistral.rs
View file

@ -391,6 +391,7 @@ pub fn into_mistral(
}
}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::ToolUse(_) => {
// Tool use is not supported in User messages for Mistral
}
@ -450,6 +451,7 @@ pub fn into_mistral(
}
MessageContent::RedactedThinking(_) => {}
MessageContent::Image(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::ToolUse(tool_use) => {
let tool_call = mistral::ToolCall {
id: tool_use.id.to_string(),
@ -503,6 +505,7 @@ pub fn into_mistral(
}
}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::Image(_)
| MessageContent::ToolUse(_)
| MessageContent::ToolResult(_) => {
@ -987,6 +990,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: Default::default(),
compact_at_tokens: None,
};
let (mistral_request, affinity) =
@ -1023,6 +1027,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let (mistral_request, _) = into_mistral(request, mistral::Model::MistralSmallLatest, None);

4
crates/language_models/src/provider/open_ai.rs
View file

@ -476,6 +476,10 @@ impl LanguageModel for OpenAiLanguageModel {
self.model.supports_priority()
}
fn supports_server_side_compaction(&self) -> bool {
self.model.supports_compaction()
}
fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
supported_thinking_effort_levels(&self.model)
}

1
crates/language_models/src/provider/open_router.rs
View file

@ -444,6 +444,7 @@ pub fn into_open_router(
),
MessageContent::Thinking { .. } => {}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
MessageContent::Image(image) => {
add_message_content_part(
open_router::MessagePart::Image {

4
crates/language_models_cloud/src/language_models_cloud.rs
View file

@ -362,6 +362,10 @@ impl<TP: CloudLlmTokenProvider + 'static> LanguageModel for CloudLanguageModel<T
self.model.supports_fast_mode
}
fn supports_server_side_compaction(&self) -> bool {
self.model.supports_server_side_compaction
}
fn supported_effort_levels(&self) -> Vec<LanguageModelEffortLevel> {
self.model
.supported_effort_levels

180
crates/open_ai/src/completion.rs
View file

@ -2,8 +2,8 @@ use anyhow::{Result, anyhow};
use collections::HashMap;
use futures::{Stream, StreamExt};
use language_model_core::{
LanguageModelCompletionError, LanguageModelCompletionEvent, LanguageModelImage,
LanguageModelRequest, LanguageModelRequestMessage, LanguageModelToolChoice,
CompactionContent, LanguageModelCompletionError, LanguageModelCompletionEvent,
LanguageModelImage, LanguageModelRequest, LanguageModelRequestMessage, LanguageModelToolChoice,
LanguageModelToolResultContent, LanguageModelToolUse, LanguageModelToolUseId, MessageContent,
Role, StopReason, TokenUsage,
util::{fix_streamed_json, parse_tool_arguments},
@ -12,10 +12,10 @@ use std::pin::Pin;
use std::sync::Arc;
use crate::responses::{
Request as ResponseRequest, ResponseError, ResponseFunctionCallItem,
ResponseFunctionCallOutputContent, ResponseFunctionCallOutputItem, ResponseIncludable,
ResponseInputContent, ResponseInputItem, ResponseMessageItem, ResponseOutputItem,
ResponseOutputMessage, ResponseReasoningInputItem, ResponseReasoningItem,
ContextManagement, Request as ResponseRequest, ResponseCompactionItem, ResponseError,
ResponseFunctionCallItem, ResponseFunctionCallOutputContent, ResponseFunctionCallOutputItem,
ResponseIncludable, ResponseInputContent, ResponseInputItem, ResponseMessageItem,
ResponseOutputItem, ResponseOutputMessage, ResponseReasoningInputItem, ResponseReasoningItem,
ResponseReasoningSummaryPart, ResponseSummary as ResponsesSummary,
ResponseUsage as ResponsesUsage, StreamEvent as ResponsesStreamEvent,
};
@ -82,7 +82,7 @@ pub fn into_open_ai(
}
}
}
MessageContent::RedactedThinking(_) => {}
MessageContent::RedactedThinking(_) | MessageContent::Compaction(_) => {}
MessageContent::Image(image) => {
add_message_content_part(
MessagePart::Image {
@ -211,6 +211,7 @@ pub fn into_open_ai_response(
thinking_allowed,
thinking_effort,
speed,
compact_at_tokens,
} = request;
let service_tier = service_tier_for(speed);
@ -299,6 +300,8 @@ pub fn into_open_ai_response(
},
reasoning,
service_tier,
context_management: compact_at_tokens
.map(|compact_threshold| vec![ContextManagement::Compaction { compact_threshold }]),
}
}
@ -336,6 +339,27 @@ fn append_message_to_response_items(
push_response_text_part(&role, text, &mut content_parts);
}
MessageContent::Thinking { .. } | MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(CompactionContent::Encrypted {
id,
encrypted_content,
}) => {
flush_response_parts(
&role,
index,
phase.as_deref(),
&mut content_parts,
input_items,
);
input_items.push(ResponseInputItem::Compaction(ResponseCompactionItem {
id,
encrypted_content,
}));
}
// Summary compaction blocks come from other providers, and a
// Pending block is a streaming-only UI signal; neither is replayed.
MessageContent::Compaction(
CompactionContent::Summary { .. } | CompactionContent::Pending,
) => {}
MessageContent::Image(image) => {
push_response_image_part(&role, image, &mut content_parts);
}
@ -757,6 +781,11 @@ impl OpenAiResponseEventMapper {
self.function_calls_by_item.insert(item_id, entry);
}
}
ResponseOutputItem::Compaction(_) => {
events.push(Ok(LanguageModelCompletionEvent::Compaction(
CompactionContent::Pending,
)));
}
ResponseOutputItem::Reasoning(_) | ResponseOutputItem::Unknown => {}
}
events
@ -905,6 +934,14 @@ impl OpenAiResponseEventMapper {
ResponsesStreamEvent::OutputItemDone { item, .. } => match item {
ResponseOutputItem::Reasoning(reasoning) => self.capture_reasoning_item(&reasoning),
ResponseOutputItem::Message(message) => self.capture_message_phase(&message),
ResponseOutputItem::Compaction(compaction) => {
vec![Ok(LanguageModelCompletionEvent::Compaction(
CompactionContent::Encrypted {
id: compaction.id,
encrypted_content: compaction.encrypted_content,
},
))]
}
ResponseOutputItem::FunctionCall(_) | ResponseOutputItem::Unknown => Vec::new(),
},
ResponsesStreamEvent::OutputTextDone { .. }
@ -1411,6 +1448,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1532,6 +1570,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1614,6 +1653,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let response =
@ -1670,6 +1710,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1713,6 +1754,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: None,
speed,
compact_at_tokens: None,
};
let response = into_open_ai_response(request, "gpt-5.4", true, true, None, None, true);
@ -1754,6 +1796,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: None,
speed,
compact_at_tokens: None,
};
let chat = into_open_ai(request, "gpt-5.4", true, true, None, None, false);
@ -1789,6 +1832,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: Some("high".into()),
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1827,6 +1871,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: Some("none".into()),
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1877,6 +1922,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -1966,6 +2012,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let response = into_open_ai_response(
@ -2053,6 +2100,7 @@ mod tests {
thinking_allowed: false,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let response =
@ -3109,6 +3157,7 @@ mod tests {
thinking_allowed: true,
thinking_effort: None,
speed: None,
compact_at_tokens: None,
};
let result = into_open_ai(request.clone(), "model", false, false, None, None, true);
@ -3271,4 +3320,121 @@ mod tests {
)
}));
}
#[test]
fn into_open_ai_response_maps_compact_at_tokens_to_context_management() {
let request = LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".into())],
cache: false,
reasoning_details: None,
}],
compact_at_tokens: Some(100_000),
..Default::default()
};
let response = into_open_ai_response(request, "gpt-5.1", true, true, None, None, false);
assert_eq!(
serde_json::to_value(&response).unwrap()["context_management"],
json!([{ "type": "compaction", "compact_threshold": 100_000 }])
);
}
#[test]
fn into_open_ai_response_omits_context_management_without_compact_at_tokens() {
let request = LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".into())],
cache: false,
reasoning_details: None,
}],
..Default::default()
};
let response = into_open_ai_response(request, "gpt-5.1", true, true, None, None, false);
assert!(
serde_json::to_value(&response)
.unwrap()
.get("context_management")
.is_none()
);
}
#[test]
fn into_open_ai_response_replays_encrypted_compaction_block() {
let request = LanguageModelRequest {
messages: vec![LanguageModelRequestMessage {
role: Role::Assistant,
content: vec![
MessageContent::Compaction(CompactionContent::Encrypted {
id: Some("cmp_1".into()),
encrypted_content: "encrypted-blob".into(),
}),
MessageContent::Text("Done.".into()),
],
cache: false,
reasoning_details: None,
}],
..Default::default()
};
let response = into_open_ai_response(request, "gpt-5.1", true, true, None, None, false);
assert_eq!(
serde_json::to_value(&response).unwrap()["input"],
json!([
{
"type": "compaction",
"id": "cmp_1",
"encrypted_content": "encrypted-blob"
},
{
"type": "message",
"role": "assistant",
"content": [
{ "type": "output_text", "text": "Done.", "annotations": [] }
]
}
])
);
}
#[test]
fn responses_stream_maps_compaction_output_item() {
let item: ResponseOutputItem = serde_json::from_value(json!({
"type": "compaction",
"id": "cmp_1",
"encrypted_content": "encrypted-blob"
}))
.unwrap();
let events = vec![
ResponsesStreamEvent::OutputItemAdded {
output_index: 0,
sequence_number: None,
item: item.clone(),
},
ResponsesStreamEvent::OutputItemDone {
output_index: 0,
sequence_number: None,
item,
},
];
let mapped = map_response_events(events);
assert_eq!(
mapped,
vec![
LanguageModelCompletionEvent::Compaction(CompactionContent::Pending),
LanguageModelCompletionEvent::Compaction(CompactionContent::Encrypted {
id: Some("cmp_1".into()),
encrypted_content: "encrypted-blob".into(),
}),
]
);
}
}

28
crates/open_ai/src/open_ai.rs
View file

@ -344,6 +344,34 @@ impl Model {
true
}
/// Whether this model supports server-side compaction via the
/// `context_management` request parameter. OpenAI doesn't publish a
/// support matrix, but the GPT-5.5 guide notes compaction is a feature
/// shared with GPT-5.4, and the compaction docs exercise the GPT-5.3
/// Codex line, so we treat everything from GPT-5.3 onward as supported.
///
/// <https://developers.openai.com/api/docs/guides/compaction>
pub fn supports_compaction(&self) -> bool {
match self {
Self::FivePointThreeCodex
| Self::FivePointFourNano
| Self::FivePointFourMini
| Self::FivePointFour
| Self::FivePointFourPro
| Self::FivePointFive
| Self::FivePointFivePro => true,
Self::Four
| Self::FourOmniMini
| Self::O3
| Self::Five
| Self::FiveMini
| Self::FiveNano
| Self::FivePointOne
| Self::FivePointTwo
| Self::Custom { .. } => false,
}
}
/// Whether OpenAI's Priority processing tier is available for this model.
/// Sourced from <https://openai.com/api-priority-processing/>. The `*-pro`,
/// `*-nano`, and legacy `gpt-4` variants are not eligible.

24
crates/open_ai/src/responses.rs
View file

@ -5,6 +5,7 @@ use http_client::{
};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::sync::Arc;
use crate::{ReasoningEffort, RequestError, Role, ServiceTier, ToolChoice};
@ -39,6 +40,17 @@ pub struct Request {
pub store: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub service_tier: Option<ServiceTier>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub context_management: Option<Vec<ContextManagement>>,
}
/// Server-side context management configuration.
///
/// <https://developers.openai.com/api/docs/guides/compaction>
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ContextManagement {
Compaction { compact_threshold: u64 },
}
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
@ -55,6 +67,14 @@ pub enum ResponseInputItem {
FunctionCall(ResponseFunctionCallItem),
FunctionCallOutput(ResponseFunctionCallOutputItem),
Reasoning(ResponseReasoningInputItem),
Compaction(ResponseCompactionItem),
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ResponseCompactionItem {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<Arc<str>>,
pub encrypted_content: Arc<str>,
}
#[derive(Debug, Serialize, Deserialize)]
@ -379,6 +399,7 @@ pub enum ResponseOutputItem {
Message(ResponseOutputMessage),
FunctionCall(ResponseFunctionToolCall),
Reasoning(ResponseReasoningItem),
Compaction(ResponseCompactionItem),
#[serde(other)]
Unknown,
}
@ -557,6 +578,9 @@ pub async fn stream_response(
}
}
}
// No synthesized deltas; the `OutputItemDone`
// event pushed below carries the full item.
ResponseOutputItem::Compaction(_) => {}
ResponseOutputItem::Unknown => {}
}