Fix user message text silently dropped when message contains both text and image (#8071)
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli (push) Blocked by required conditions
Canary / Upload Install Script (push) Blocked by required conditions
Canary / bundle-desktop (push) Blocked by required conditions
Canary / bundle-desktop-intel (push) Blocked by required conditions
Canary / bundle-desktop-linux (push) Blocked by required conditions
Canary / bundle-desktop-windows (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / changes (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check OpenAPI Schema is Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Live Provider Tests / goose server HTTP integration tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Publish to npm / Build native binaries (push) Waiting to run
Publish to npm / Release (push) Blocked by required conditions
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run

Signed-off-by: Vincenzo Palazzo <vincenzopalazzodev@gmail.com>
Signed-off-by: Douwe Osinga <douwe@squareup.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
Vincenzo Palazzo 2026-03-25 00:25:53 +01:00 committed by GitHub
parent 18bc030e48
commit 9849671413
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -126,7 +126,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
let mut output = Vec::new();
let mut content_array = Vec::new();
let mut text_array = Vec::new();
let mut has_non_text_content = false;
let mut reasoning_text = String::new();
for content in &message.content {
@ -136,16 +136,17 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
if message.role == Role::User {
if let Some(image_path) = detect_image_path(&text.text) {
if let Ok(image) = load_image_file(image_path) {
has_non_text_content = true;
content_array.push(json!({"type": "text", "text": text.text}));
content_array.push(convert_image(&image, image_format));
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
} else {
text_array.push(text.text.clone());
content_array.push(json!({"type": "text", "text": text.text}));
}
}
}
@ -259,6 +260,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
MessageContent::ActionRequired(_) => {}
MessageContent::Image(image) => {
if message.role == Role::User {
has_non_text_content = true;
content_array.push(convert_image(image, image_format));
} else {
content_array.push(json!({
@ -304,9 +306,15 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
}
if !content_array.is_empty() {
converted["content"] = json!(content_array);
} else if !text_array.is_empty() {
converted["content"] = json!(text_array.join("\n"));
if has_non_text_content {
converted["content"] = json!(content_array);
} else {
let texts: Vec<String> = content_array
.iter()
.filter_map(|v| v["text"].as_str().map(|s| s.to_string()))
.collect();
converted["content"] = json!(texts.join("\n"));
}
}
// Some strict OpenAI-compatible providers require "content" to be present
@ -1210,6 +1218,40 @@ mod tests {
Ok(())
}
#[test]
fn test_format_messages_with_text_and_image_preserves_order() {
// Text before image: order should be [text, image]
let msg_text_first = Message::user()
.with_text("Describe this image")
.with_image("aW1hZ2VkYXRh", "image/png");
let spec = format_messages(&[msg_text_first], &ImageFormat::OpenAi);
assert_eq!(spec.len(), 1);
assert_eq!(spec[0]["role"], "user");
let content = spec[0]["content"]
.as_array()
.expect("content should be an array");
assert_eq!(content.len(), 2, "expected text + image entries");
assert_eq!(content[0]["type"], "text");
assert_eq!(content[0]["text"], "Describe this image");
assert_eq!(content[1]["type"], "image_url");
// Image before text: order should be [image, text]
let msg_image_first = Message::user()
.with_image("aW1hZ2VkYXRh", "image/png")
.with_text("What do you see?");
let spec2 = format_messages(&[msg_image_first], &ImageFormat::OpenAi);
let content2 = spec2[0]["content"]
.as_array()
.expect("content should be an array");
assert_eq!(content2.len(), 2, "expected image + text entries");
assert_eq!(content2[0]["type"], "image_url");
assert_eq!(content2[1]["type"], "text");
assert_eq!(content2[1]["text"], "What do you see?");
}
#[test]
fn test_response_to_message_text() -> anyhow::Result<()> {
let response = json!({