vrr/zed
2
0
Fork 0
mirror of https://github.com/zed-industries/zed.git synced 2026-07-10 00:13:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 18

open_router: Add prompt caching for Anthropic models (#57498)

Browse source 
Closes https://github.com/zed-industries/zed/issues/52576

## Overview

Adds prompt caching support for Anthropic Claude models when accessed
via OpenRouter.

This mirrors Zed's native Anthropic provider by using explicit per-block
`cache_control` breakpoints:

- A long-lived 1-hour breakpoint on the system message's last text
block, covering the tools and system prefix.
- A default 5-minute breakpoint on the last `cache: true` conversation
message.

The implementation intentionally avoids OpenRouter's top-level automatic
`cache_control` field so routing remains available across
Anthropic-compatible providers including Anthropic, Bedrock, and Vertex
AI. It also adds `session_id` sticky routing from the request thread ID
and maps OpenRouter cache usage fields into Zed's token usage
accounting.

Self-Review Checklist:

- [x] I've reviewed my own diff for quality, security, and reliability
- [x] Unsafe blocks (if any) have justifying comments
- [x] The content is consistent with the [UI/UX
checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist)
- [x] Tests cover the new/changed behavior
- [x] Performance impact has been considered and is acceptable

Release Notes:

- Fixed Anthropic prompt caching when using OpenRouter.

---------

Co-authored-by: Anant Goel <anant@zed.dev>
This commit is contained in:
James St-Pierre 2026-06-23 19:01:58 -04:00 • committed by GitHub
parent 7187d65774
commit f844c93c94
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 495 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

457
crates/language_models/src/provider/open_router.rs
View file

@ -30,6 +30,7 @@ const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new(
const API_KEY_ENV_VAR_NAME: &str = "OPENROUTER_API_KEY";
static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME);
pub(crate) const RESERVED_HEADER_NAMES: &[&str] = &["HTTP-Referer", "X-Title"];
const MAX_OPEN_ROUTER_SESSION_ID_LENGTH: usize = 256;
#[derive(Default, Clone, Debug, PartialEq)]
pub struct OpenRouterSettings {
@ -449,23 +450,39 @@ pub fn into_open_router(
// If we ever have a more formal distionction between the models in the future,
// we should revise this to use that instead.
let is_anthropic_model = model.id().starts_with("anthropic/");
let session_id = open_router_session_id(request.thread_id);
let mut messages = Vec::new();
let mut any_message_wants_cache = false;
let mut last_cache_message_index: Option<usize> = None;
for message in request.messages {
let mut message_added_content = false;
let reasoning_details_for_message = if is_anthropic_model {
None
} else {
message.reasoning_details.clone()
};
let message_wants_cache = message.cache;
if message_wants_cache {
any_message_wants_cache = true;
}
for content in message.content {
match content {
MessageContent::Text(text) => add_message_content_part(
open_router::MessagePart::Text { text },
message.role,
&mut messages,
reasoning_details_for_message.clone(),
),
MessageContent::Text(text) => {
add_message_content_part(
open_router::MessagePart::Text {
text,
cache_control: None,
},
message.role,
&mut messages,
reasoning_details_for_message.clone(),
);
message_added_content = true;
}
MessageContent::Thinking { .. } => {}
MessageContent::RedactedThinking(_) => {}
MessageContent::Compaction(_) => {}
@ -478,6 +495,7 @@ pub fn into_open_router(
&mut messages,
reasoning_details_for_message.clone(),
);
message_added_content = true;
}
MessageContent::ToolUse(tool_use) => {
let tool_call = open_router::ToolCall {
@ -503,6 +521,7 @@ pub fn into_open_router(
reasoning_details: reasoning_details_for_message.clone(),
});
}
message_added_content = true;
}
MessageContent::ToolResult(tool_result) => {
let content: Vec<open_router::MessagePart> = tool_result
@ -512,6 +531,7 @@ pub fn into_open_router(
LanguageModelToolResultContent::Text(text) => {
open_router::MessagePart::Text {
text: text.to_string(),
cache_control: None,
}
}
LanguageModelToolResultContent::Image(image) => {
@ -526,15 +546,42 @@ pub fn into_open_router(
content: content.into(),
tool_call_id: tool_result.tool_use_id.to_string(),
});
message_added_content = true;
}
}
}
if message_wants_cache && message_added_content {
last_cache_message_index = messages.len().checked_sub(1);
}
}
if is_anthropic_model && any_message_wants_cache {
// OpenRouter's top-level automatic cache_control restricts routing to
// Anthropic direct; explicit block breakpoints also work on Bedrock and Vertex.
if let Some(content) = last_cache_message_index
.and_then(|index| messages.get_mut(index))
.and_then(request_message_content_mut)
{
set_last_text_cache_control(content, cache_control(None));
}
if let Some(content) = messages.iter_mut().find_map(|message| match message {
open_router::RequestMessage::System { content } => Some(content),
_ => None,
}) {
set_last_text_cache_control(
content,
cache_control(Some(open_router::CacheTtl::OneHour)),
);
}
}
open_router::Request {
model: model.id().into(),
messages,
stream: true,
session_id,
stop: request.stop,
temperature: request.temperature.unwrap_or(0.4),
max_tokens: max_output_tokens,
@ -576,6 +623,65 @@ pub fn into_open_router(
}
}
fn open_router_session_id(thread_id: Option<String>) -> Option<String> {
thread_id.map(|thread_id| {
thread_id
.chars()
.take(MAX_OPEN_ROUTER_SESSION_ID_LENGTH)
.collect()
})
}
fn cache_control(ttl: Option<open_router::CacheTtl>) -> open_router::CacheControl {
open_router::CacheControl {
cache_type: open_router::CacheControlType::Ephemeral,
ttl,
}
}
fn request_message_content_mut(
message: &mut open_router::RequestMessage,
) -> Option<&mut open_router::MessageContent> {
match message {
open_router::RequestMessage::User { content }
| open_router::RequestMessage::System { content }
| open_router::RequestMessage::Tool { content, .. } => Some(content),
open_router::RequestMessage::Assistant {
content: Some(content),
..
} => Some(content),
open_router::RequestMessage::Assistant { content: None, .. } => None,
}
}
fn set_last_text_cache_control(
content: &mut open_router::MessageContent,
cache_control: open_router::CacheControl,
) {
match content {
open_router::MessageContent::Plain(text) => {
let text = std::mem::take(text);
*content =
open_router::MessageContent::Multipart(vec![open_router::MessagePart::Text {
text,
cache_control: Some(cache_control),
}]);
}
open_router::MessageContent::Multipart(parts) => {
for part in parts.iter_mut().rev() {
if let open_router::MessagePart::Text {
cache_control: target,
..
} = part
{
*target = Some(cache_control);
break;
}
}
}
}
}
fn add_message_content_part(
new_part: open_router::MessagePart,
role: Role,
@ -651,11 +757,23 @@ impl OpenRouterEventMapper {
let mut events = Vec::new();
if let Some(usage) = event.usage {
let cache_creation_input_tokens = usage
.prompt_tokens_details
.as_ref()
.map_or(0, |details| details.cache_write_tokens);
let cache_read_input_tokens = usage
.prompt_tokens_details
.as_ref()
.map_or(0, |details| details.cached_tokens);
let input_tokens = usage.prompt_tokens.saturating_sub(
cache_creation_input_tokens.saturating_add(cache_read_input_tokens),
);
events.push(Ok(LanguageModelCompletionEvent::UsageUpdate(TokenUsage {
input_tokens: usage.prompt_tokens,
input_tokens,
output_tokens: usage.completion_tokens,
cache_creation_input_tokens: 0,
cache_read_input_tokens: 0,
cache_creation_input_tokens,
cache_read_input_tokens,
})));
}
@ -1114,19 +1232,57 @@ mod tests {
prompt_tokens: 12,
completion_tokens: 7,
total_tokens: 19,
prompt_tokens_details: Some(open_router::PromptTokensDetails {
cached_tokens: 5,
cache_write_tokens: 3,
}),
}),
});
assert_eq!(events.len(), 1);
match events.into_iter().next().unwrap() {
Ok(LanguageModelCompletionEvent::UsageUpdate(usage)) => {
assert_eq!(usage.input_tokens, 12);
match events.into_iter().next() {
Some(Ok(LanguageModelCompletionEvent::UsageUpdate(usage))) => {
assert_eq!(usage.input_tokens, 4);
assert_eq!(usage.output_tokens, 7);
assert_eq!(usage.cache_creation_input_tokens, 3);
assert_eq!(usage.cache_read_input_tokens, 5);
assert_eq!(usage.total_tokens(), 19);
}
other => panic!("Expected usage update event, got: {other:?}"),
}
}
#[gpui::test]
async fn test_session_id_uses_thread_id() {
let model = open_router::Model::new(
"openai/gpt-4o",
Some("GPT-4o"),
Some(128000),
Some(true),
Some(false),
None,
None,
);
let expected_session_id = "a".repeat(MAX_OPEN_ROUTER_SESSION_ID_LENGTH);
let request = LanguageModelRequest {
thread_id: Some(format!("{expected_session_id}extra")),
messages: vec![language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
cache: false,
reasoning_details: None,
}],
..Default::default()
};
let result = into_open_router(request, &model, None);
assert_eq!(
result.session_id.as_deref(),
Some(expected_session_id.as_str())
);
}
#[gpui::test]
async fn test_agent_prevents_empty_reasoning_details_overwrite() {
// This test verifies that the agent layer prevents empty reasoning_details
@ -1168,4 +1324,281 @@ mod tests {
panic!("Expected array");
}
}
#[gpui::test]
async fn test_anthropic_model_caching_two_tier() {
let model = open_router::Model::new(
"anthropic/claude-sonnet-4-5",
Some("Claude Sonnet"),
Some(200000),
Some(true),
Some(false),
None,
None,
);
let request = LanguageModelRequest {
messages: vec![
language_model::LanguageModelRequestMessage {
role: Role::System,
content: vec![MessageContent::Text("You are helpful.".to_string())],
cache: false,
reasoning_details: None,
},
language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
cache: false,
reasoning_details: None,
},
language_model::LanguageModelRequestMessage {
role: Role::Assistant,
content: vec![MessageContent::Text("Hi there!".to_string())],
cache: false,
reasoning_details: None,
},
language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("What is 2+2?".to_string())],
cache: true,
reasoning_details: None,
},
],
stop: vec![],
temperature: None,
tools: vec![],
tool_choice: None,
thinking_allowed: false,
thinking_effort: None,
speed: None,
thread_id: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
};
let result = into_open_router(request, &model, None);
let system_cache = result.messages.iter().find_map(|m| {
if let open_router::RequestMessage::System { content } = m {
if let open_router::MessageContent::Multipart(parts) = content {
parts.iter().last().and_then(|p| {
if let open_router::MessagePart::Text { cache_control, .. } = p {
*cache_control
} else {
None
}
})
} else {
None
}
} else {
None
}
});
assert!(
matches!(
system_cache,
Some(open_router::CacheControl {
cache_type: open_router::CacheControlType::Ephemeral,
ttl: Some(open_router::CacheTtl::OneHour),
})
),
"System message should have 1h cache_control, got: {system_cache:?}"
);
let tail_cache = result.messages.last().and_then(|last_message| {
if let open_router::RequestMessage::User { content } = last_message {
if let open_router::MessageContent::Multipart(parts) = content {
parts.iter().last().and_then(|part| {
if let open_router::MessagePart::Text { cache_control, .. } = part {
*cache_control
} else {
None
}
})
} else {
None
}
} else {
None
}
});
assert!(
matches!(
tail_cache,
Some(open_router::CacheControl {
cache_type: open_router::CacheControlType::Ephemeral,
ttl: None,
})
),
"Last cache:true message should have 5min cache_control, got: {tail_cache:?}"
);
for (i, message) in result.messages.iter().enumerate() {
let is_system = matches!(message, open_router::RequestMessage::System { .. });
let is_last = i == result.messages.len() - 1;
if is_system || is_last {
continue;
}
let parts: Option<&Vec<open_router::MessagePart>> = match message {
open_router::RequestMessage::User { content }
| open_router::RequestMessage::System { content }
| open_router::RequestMessage::Tool { content, .. } => {
if let open_router::MessageContent::Multipart(parts) = content {
Some(parts)
} else {
None
}
}
open_router::RequestMessage::Assistant {
content: Some(content),
..
} => {
if let open_router::MessageContent::Multipart(parts) = content {
Some(parts)
} else {
None
}
}
_ => None,
};
if let Some(parts) = parts {
for part in parts {
if let open_router::MessagePart::Text { cache_control, .. } = part {
assert!(
cache_control.is_none(),
"Message {i} should not have cache_control"
);
}
}
}
}
}
#[gpui::test]
async fn test_anthropic_model_no_cache_when_no_cache_flag() {
let model = open_router::Model::new(
"anthropic/claude-sonnet-4-5",
Some("Claude Sonnet"),
Some(200000),
Some(true),
Some(false),
None,
None,
);
let request = LanguageModelRequest {
messages: vec![
language_model::LanguageModelRequestMessage {
role: Role::System,
content: vec![MessageContent::Text("You are helpful.".to_string())],
cache: false,
reasoning_details: None,
},
language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
cache: false,
reasoning_details: None,
},
],
stop: vec![],
temperature: None,
tools: vec![],
tool_choice: None,
thinking_allowed: false,
thinking_effort: None,
speed: None,
thread_id: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
};
let result = into_open_router(request, &model, None);
for message in &result.messages {
let content = match message {
open_router::RequestMessage::User { content }
| open_router::RequestMessage::System { content } => Some(content),
_ => None,
};
if let Some(content) = content {
if let open_router::MessageContent::Multipart(parts) = content {
for part in parts {
if let open_router::MessagePart::Text { cache_control, .. } = part {
assert!(
cache_control.is_none(),
"No message should have cache_control when no cache:true flags"
);
}
}
}
}
}
}
#[gpui::test]
async fn test_non_anthropic_model_no_cache_control() {
let model = open_router::Model::new(
"openai/gpt-4o",
Some("GPT-4o"),
Some(128000),
Some(true),
Some(false),
None,
None,
);
let request = LanguageModelRequest {
messages: vec![
language_model::LanguageModelRequestMessage {
role: Role::System,
content: vec![MessageContent::Text("You are helpful.".to_string())],
cache: false,
reasoning_details: None,
},
language_model::LanguageModelRequestMessage {
role: Role::User,
content: vec![MessageContent::Text("Hello".to_string())],
cache: true,
reasoning_details: None,
},
],
stop: vec![],
temperature: None,
tools: vec![],
tool_choice: None,
thinking_allowed: false,
thinking_effort: None,
speed: None,
thread_id: None,
prompt_id: None,
intent: None,
compact_at_tokens: None,
};
let result = into_open_router(request, &model, None);
for message in &result.messages {
let content = match message {
open_router::RequestMessage::User { content }
| open_router::RequestMessage::System { content } => Some(content),
_ => None,
};
if let Some(content) = content {
if let open_router::MessageContent::Multipart(parts) = content {
for part in parts {
if let open_router::MessagePart::Text { cache_control, .. } = part {
assert!(
cache_control.is_none(),
"Non-Anthropic model should never have cache_control"
);
}
}
}
}
}
}
}

56
crates/open_router/src/open_router.rs
View file

@ -147,6 +147,8 @@ pub struct Request {
pub messages: Vec<RequestMessage>,
pub stream: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub session_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub stop: Vec<String>,
@ -248,6 +250,7 @@ impl MessageContent {
Self::Plain(text) => {
let text_part = MessagePart::Text {
text: std::mem::take(text),
cache_control: None,
};
*self = Self::Multipart(vec![text_part, part]);
}
@ -259,7 +262,11 @@ impl MessageContent {
impl From<Vec<MessagePart>> for MessageContent {
fn from(parts: Vec<MessagePart>) -> Self {
if parts.len() == 1
&& let MessagePart::Text { text } = &parts[0]
&& let MessagePart::Text {
text,
cache_control,
} = &parts[0]
&& cache_control.is_none()
{
return Self::Plain(text.clone());
}
@ -284,7 +291,7 @@ impl MessageContent {
match self {
Self::Plain(text) => Some(text),
Self::Multipart(parts) if parts.len() == 1 => {
if let MessagePart::Text { text } = &parts[0] {
if let MessagePart::Text { text, .. } = &parts[0] {
Some(text)
} else {
None
@ -300,7 +307,7 @@ impl MessageContent {
Self::Multipart(parts) => parts
.iter()
.filter_map(|part| {
if let MessagePart::Text { text } = part {
if let MessagePart::Text { text, .. } = part {
Some(text.as_str())
} else {
None
@ -312,16 +319,43 @@ impl MessageContent {
}
}
#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum CacheControlType {
Ephemeral,
}
#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
pub enum CacheTtl {
/// Anthropic's default ephemeral TTL (currently 5 minutes). Refreshes for
/// free on every cache hit.
#[serde(rename = "5m")]
FiveMinutes,
/// Anthropic's extended ephemeral TTL (currently 1 hour). Costs 2x base
/// input tokens to write, but persists across longer idle gaps.
#[serde(rename = "1h")]
OneHour,
}
#[derive(Debug, Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
pub struct CacheControl {
#[serde(rename = "type")]
pub cache_type: CacheControlType,
/// Omitting this field uses the API's default 5-minute TTL.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ttl: Option<CacheTtl>,
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum MessagePart {
Text {
text: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<CacheControl>,
},
#[serde(rename = "image_url")]
Image {
image_url: String,
},
Image { image_url: String },
}
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq)]
@ -371,11 +405,21 @@ pub struct FunctionChunk {
pub thought_signature: Option<String>,
}
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct PromptTokensDetails {
#[serde(default)]
pub cached_tokens: u64,
#[serde(default)]
pub cache_write_tokens: u64,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Usage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_tokens_details: Option<PromptTokensDetails>,
}
#[derive(Serialize, Deserialize, Debug)]