mirror of
https://github.com/zed-industries/zed.git
synced 2026-07-10 00:13:29 +00:00
language_models: Avoid sending Bedrock cache-point-only messages (#59436)
When prompt caching is enabled, `into_bedrock` pushes a `CachePoint`
block
onto a message's content whenever `message.cache && supports_caching` is
true.
This push happens before the `if bedrock_message_content.is_empty() {
continue; }`
guard. As a result, a message whose content filters down to empty (for
example,
a message that contained only content stripped during conversion) still
gets
appended to the request carrying nothing but a `cachePoint`. Bedrock
rejects such
a message with a `ValidationException`, breaking the whole request.
The Anthropic path is already internally consistent here: in
`crates/language_models/src/provider/anthropic.rs` (around the message
assembly
in `completion.rs`) the empty-content check runs first, so an empty
message is
dropped before any cache marker is attached. The Bedrock path should
behave the
same way.
This change gates the `CachePoint` push on non-empty content
(`&& !bedrock_message_content.is_empty()`), so an empty message is left
empty and
then skipped by the existing `continue`, exactly mirroring the Anthropic
ordering.
The fix is minimal and touches only the cache-point condition.
Release Notes:
- Fixed Bedrock requests failing with ValidationException when the last
message filtered to empty content while prompt caching was enabled.
---------
Signed-off-by: Yi LIU <yi@quantstamp.com>
Co-authored-by: Smit Barmase <heysmitbarmase@gmail.com>
This commit is contained in:
parent
814b152a0f
commit
6eaad52c29
1 changed files with 89 additions and 1 deletions
|
|
@ -1025,7 +1025,7 @@ pub fn into_bedrock(
|
|||
}
|
||||
})
|
||||
.collect();
|
||||
if message.cache && supports_caching {
|
||||
if message.cache && supports_caching && !bedrock_message_content.is_empty() {
|
||||
bedrock_message_content.push(BedrockInnerContent::CachePoint(
|
||||
CachePointBlock::builder()
|
||||
.r#type(CachePointType::Default)
|
||||
|
|
@ -1747,3 +1747,91 @@ impl ConfigurationView {
|
|||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use language_model::LanguageModelRequestMessage;
|
||||
|
||||
fn into_bedrock_request(messages: Vec<LanguageModelRequestMessage>) -> bedrock::Request {
|
||||
into_bedrock(
|
||||
LanguageModelRequest {
|
||||
messages,
|
||||
..Default::default()
|
||||
},
|
||||
"claude-sonnet-4-5".to_string(),
|
||||
1.0,
|
||||
4096,
|
||||
BedrockModelMode::Default,
|
||||
true,
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_marked_message_that_filters_to_empty_is_dropped() {
|
||||
let request = into_bedrock_request(vec![
|
||||
LanguageModelRequestMessage {
|
||||
role: Role::User,
|
||||
content: vec![MessageContent::Text("What's the weather?".into())],
|
||||
cache: false,
|
||||
reasoning_details: None,
|
||||
},
|
||||
LanguageModelRequestMessage {
|
||||
role: Role::Assistant,
|
||||
content: vec![MessageContent::Thinking {
|
||||
text: "Let me think about this...".into(),
|
||||
signature: None,
|
||||
}],
|
||||
cache: true,
|
||||
reasoning_details: None,
|
||||
},
|
||||
LanguageModelRequestMessage {
|
||||
role: Role::User,
|
||||
content: vec![MessageContent::Text("Summarize this conversation.".into())],
|
||||
cache: false,
|
||||
reasoning_details: None,
|
||||
},
|
||||
]);
|
||||
|
||||
for message in &request.messages {
|
||||
assert!(
|
||||
message
|
||||
.content()
|
||||
.iter()
|
||||
.any(|block| !matches!(block, BedrockInnerContent::CachePoint(_))),
|
||||
"message must not consist solely of cache points: {:?}",
|
||||
message
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
request
|
||||
.messages
|
||||
.iter()
|
||||
.all(|message| *message.role() == bedrock::BedrockRole::User),
|
||||
"the assistant message stripped to empty content should be dropped entirely"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cache_marked_message_with_content_gets_cache_point() {
|
||||
let request = into_bedrock_request(vec![LanguageModelRequestMessage {
|
||||
role: Role::User,
|
||||
content: vec![MessageContent::Text("What's the weather?".into())],
|
||||
cache: true,
|
||||
reasoning_details: None,
|
||||
}]);
|
||||
|
||||
assert_eq!(request.messages.len(), 1);
|
||||
assert!(
|
||||
matches!(
|
||||
request.messages[0].content().last(),
|
||||
Some(BedrockInnerContent::CachePoint(_))
|
||||
),
|
||||
"a cache-marked message with content should end with a cache point"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue