mirror of
https://github.com/block/goose.git
synced 2026-07-09 16:09:22 +00:00
Fix user message text silently dropped when message contains both text and image (#8071)
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli (push) Blocked by required conditions
Canary / Upload Install Script (push) Blocked by required conditions
Canary / bundle-desktop (push) Blocked by required conditions
Canary / bundle-desktop-intel (push) Blocked by required conditions
Canary / bundle-desktop-linux (push) Blocked by required conditions
Canary / bundle-desktop-windows (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / changes (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check OpenAPI Schema is Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Live Provider Tests / goose server HTTP integration tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Publish to npm / Build native binaries (push) Waiting to run
Publish to npm / Release (push) Blocked by required conditions
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli (push) Blocked by required conditions
Canary / Upload Install Script (push) Blocked by required conditions
Canary / bundle-desktop (push) Blocked by required conditions
Canary / bundle-desktop-intel (push) Blocked by required conditions
Canary / bundle-desktop-linux (push) Blocked by required conditions
Canary / bundle-desktop-windows (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / changes (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check OpenAPI Schema is Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Live Provider Tests / goose server HTTP integration tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Publish to npm / Build native binaries (push) Waiting to run
Publish to npm / Release (push) Blocked by required conditions
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Signed-off-by: Vincenzo Palazzo <vincenzopalazzodev@gmail.com> Signed-off-by: Douwe Osinga <douwe@squareup.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com> Co-authored-by: Douwe Osinga <douwe@squareup.com>
This commit is contained in:
parent
18bc030e48
commit
9849671413
1 changed files with 49 additions and 7 deletions
|
|
@ -126,7 +126,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
|
|||
|
||||
let mut output = Vec::new();
|
||||
let mut content_array = Vec::new();
|
||||
let mut text_array = Vec::new();
|
||||
let mut has_non_text_content = false;
|
||||
let mut reasoning_text = String::new();
|
||||
|
||||
for content in &message.content {
|
||||
|
|
@ -136,16 +136,17 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
|
|||
if message.role == Role::User {
|
||||
if let Some(image_path) = detect_image_path(&text.text) {
|
||||
if let Ok(image) = load_image_file(image_path) {
|
||||
has_non_text_content = true;
|
||||
content_array.push(json!({"type": "text", "text": text.text}));
|
||||
content_array.push(convert_image(&image, image_format));
|
||||
} else {
|
||||
text_array.push(text.text.clone());
|
||||
content_array.push(json!({"type": "text", "text": text.text}));
|
||||
}
|
||||
} else {
|
||||
text_array.push(text.text.clone());
|
||||
content_array.push(json!({"type": "text", "text": text.text}));
|
||||
}
|
||||
} else {
|
||||
text_array.push(text.text.clone());
|
||||
content_array.push(json!({"type": "text", "text": text.text}));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -259,6 +260,7 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
|
|||
MessageContent::ActionRequired(_) => {}
|
||||
MessageContent::Image(image) => {
|
||||
if message.role == Role::User {
|
||||
has_non_text_content = true;
|
||||
content_array.push(convert_image(image, image_format));
|
||||
} else {
|
||||
content_array.push(json!({
|
||||
|
|
@ -304,9 +306,15 @@ pub fn format_messages(messages: &[Message], image_format: &ImageFormat) -> Vec<
|
|||
}
|
||||
|
||||
if !content_array.is_empty() {
|
||||
converted["content"] = json!(content_array);
|
||||
} else if !text_array.is_empty() {
|
||||
converted["content"] = json!(text_array.join("\n"));
|
||||
if has_non_text_content {
|
||||
converted["content"] = json!(content_array);
|
||||
} else {
|
||||
let texts: Vec<String> = content_array
|
||||
.iter()
|
||||
.filter_map(|v| v["text"].as_str().map(|s| s.to_string()))
|
||||
.collect();
|
||||
converted["content"] = json!(texts.join("\n"));
|
||||
}
|
||||
}
|
||||
|
||||
// Some strict OpenAI-compatible providers require "content" to be present
|
||||
|
|
@ -1210,6 +1218,40 @@ mod tests {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_format_messages_with_text_and_image_preserves_order() {
|
||||
// Text before image: order should be [text, image]
|
||||
let msg_text_first = Message::user()
|
||||
.with_text("Describe this image")
|
||||
.with_image("aW1hZ2VkYXRh", "image/png");
|
||||
|
||||
let spec = format_messages(&[msg_text_first], &ImageFormat::OpenAi);
|
||||
assert_eq!(spec.len(), 1);
|
||||
assert_eq!(spec[0]["role"], "user");
|
||||
|
||||
let content = spec[0]["content"]
|
||||
.as_array()
|
||||
.expect("content should be an array");
|
||||
assert_eq!(content.len(), 2, "expected text + image entries");
|
||||
assert_eq!(content[0]["type"], "text");
|
||||
assert_eq!(content[0]["text"], "Describe this image");
|
||||
assert_eq!(content[1]["type"], "image_url");
|
||||
|
||||
// Image before text: order should be [image, text]
|
||||
let msg_image_first = Message::user()
|
||||
.with_image("aW1hZ2VkYXRh", "image/png")
|
||||
.with_text("What do you see?");
|
||||
|
||||
let spec2 = format_messages(&[msg_image_first], &ImageFormat::OpenAi);
|
||||
let content2 = spec2[0]["content"]
|
||||
.as_array()
|
||||
.expect("content should be an array");
|
||||
assert_eq!(content2.len(), 2, "expected image + text entries");
|
||||
assert_eq!(content2[0]["type"], "image_url");
|
||||
assert_eq!(content2[1]["type"], "text");
|
||||
assert_eq!(content2[1]["text"], "What do you see?");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_response_to_message_text() -> anyhow::Result<()> {
|
||||
let response = json!({
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue