agent: Stream new_text in StreamingEditFileTool (#50240)

We now stream the new text into the buffer as soon as we receive partial
chunks of `new_text`. This is pretty much a full re-write of the way
streaming worked, which is now much closer to how the edit agent works:
- `ToolEditParser` buffers chunks as they stream in, and emits relevant
events (`OldTextChunk`,`NewTextChunk`, ...) that we use to power the
`EditSession` pipeline.
- `EditSession::process_events` takes care of consuming these events and
applying the edits incrementally as chunks stream in. `EditPipeline`
maintains the underlying state machine for each edit.
- We handle whitespace mismatches similar to the edit agent, the code is
shared by moving that logic to `reindent.rs`

Before you mark this PR as ready for review, make sure that you have:
- [x] Added a solid test coverage and/or screenshots from doing manual
testing
- [x] Done a self-review taking into account security and performance
aspects
- [x] Aligned any UI changes with the [UI
checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist)

Release Notes:

- N/A
This commit is contained in:
Bennet Bo Fenner 2026-02-27 13:24:57 +01:00 committed by GitHub
parent 78878e514e
commit c9425f2a90
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 2059 additions and 1063 deletions

View file

@ -2,6 +2,7 @@ mod create_file_parser;
mod edit_parser;
#[cfg(test)]
mod evals;
pub mod reindent;
pub mod streaming_fuzzy_matcher;
use crate::{Template, Templates};
@ -24,9 +25,10 @@ use language_model::{
LanguageModelToolChoice, MessageContent, Role,
};
use project::{AgentLocation, Project};
use reindent::{IndentDelta, Reindenter};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use std::{cmp, iter, mem, ops::Range, pin::Pin, sync::Arc, task::Poll};
use std::{mem, ops::Range, pin::Pin, sync::Arc, task::Poll};
use streaming_diff::{CharOperation, StreamingDiff};
use streaming_fuzzy_matcher::StreamingFuzzyMatcher;
@ -553,15 +555,8 @@ impl EditAgent {
let compute_edits = cx.background_spawn(async move {
let buffer_start_indent = snapshot
.line_indent_for_row(snapshot.offset_to_point(resolved_old_text.range.start).row);
let indent_delta = if buffer_start_indent.tabs > 0 {
IndentDelta::Tabs(
buffer_start_indent.tabs as isize - resolved_old_text.indent.tabs as isize,
)
} else {
IndentDelta::Spaces(
buffer_start_indent.spaces as isize - resolved_old_text.indent.spaces as isize,
)
};
let indent_delta =
reindent::compute_indent_delta(buffer_start_indent, resolved_old_text.indent);
let old_text = snapshot
.text_for_range(resolved_old_text.range.clone())
@ -608,8 +603,7 @@ impl EditAgent {
delta: IndentDelta,
mut stream: impl Unpin + Stream<Item = Result<EditParserEvent>>,
) -> impl Stream<Item = Result<String>> {
let mut buffer = String::new();
let mut in_leading_whitespace = true;
let mut reindenter = Reindenter::new(delta);
let mut done = false;
futures::stream::poll_fn(move |cx| {
while !done {
@ -622,55 +616,10 @@ impl EditAgent {
_ => return Poll::Ready(None),
};
buffer.push_str(&chunk);
let mut indented_new_text = String::new();
let mut start_ix = 0;
let mut newlines = buffer.match_indices('\n').peekable();
loop {
let (line_end, is_pending_line) = match newlines.next() {
Some((ix, _)) => (ix, false),
None => (buffer.len(), true),
};
let line = &buffer[start_ix..line_end];
if in_leading_whitespace {
if let Some(non_whitespace_ix) = line.find(|c| delta.character() != c) {
// We found a non-whitespace character, adjust
// indentation based on the delta.
let new_indent_len =
cmp::max(0, non_whitespace_ix as isize + delta.len()) as usize;
indented_new_text
.extend(iter::repeat(delta.character()).take(new_indent_len));
indented_new_text.push_str(&line[non_whitespace_ix..]);
in_leading_whitespace = false;
} else if is_pending_line {
// We're still in leading whitespace and this line is incomplete.
// Stop processing until we receive more input.
break;
} else {
// This line is entirely whitespace. Push it without indentation.
indented_new_text.push_str(line);
}
} else {
indented_new_text.push_str(line);
}
if is_pending_line {
start_ix = line_end;
break;
} else {
in_leading_whitespace = true;
indented_new_text.push('\n');
start_ix = line_end + 1;
}
}
buffer.replace_range(..start_ix, "");
let mut indented_new_text = reindenter.push(&chunk);
// This was the last chunk, push all the buffered content as-is.
if is_last_chunk {
indented_new_text.push_str(&buffer);
buffer.clear();
indented_new_text.push_str(&reindenter.finish());
done = true;
}
@ -761,28 +710,6 @@ struct ResolvedOldText {
indent: LineIndent,
}
#[derive(Copy, Clone, Debug)]
enum IndentDelta {
Spaces(isize),
Tabs(isize),
}
impl IndentDelta {
fn character(&self) -> char {
match self {
IndentDelta::Spaces(_) => ' ',
IndentDelta::Tabs(_) => '\t',
}
}
fn len(&self) -> isize {
match self {
IndentDelta::Spaces(n) => *n,
IndentDelta::Tabs(n) => *n,
}
}
}
#[cfg(test)]
mod tests {
use super::*;

View file

@ -0,0 +1,214 @@
use language::LineIndent;
use std::{cmp, iter};
#[derive(Copy, Clone, Debug)]
pub enum IndentDelta {
Spaces(isize),
Tabs(isize),
}
impl IndentDelta {
pub fn character(&self) -> char {
match self {
IndentDelta::Spaces(_) => ' ',
IndentDelta::Tabs(_) => '\t',
}
}
pub fn len(&self) -> isize {
match self {
IndentDelta::Spaces(n) => *n,
IndentDelta::Tabs(n) => *n,
}
}
}
pub fn compute_indent_delta(buffer_indent: LineIndent, query_indent: LineIndent) -> IndentDelta {
if buffer_indent.tabs > 0 {
IndentDelta::Tabs(buffer_indent.tabs as isize - query_indent.tabs as isize)
} else {
IndentDelta::Spaces(buffer_indent.spaces as isize - query_indent.spaces as isize)
}
}
/// Synchronous re-indentation adapter. Buffers incomplete lines and applies
/// an `IndentDelta` to each line's leading whitespace before emitting it.
pub struct Reindenter {
delta: IndentDelta,
buffer: String,
in_leading_whitespace: bool,
}
impl Reindenter {
pub fn new(delta: IndentDelta) -> Self {
Self {
delta,
buffer: String::new(),
in_leading_whitespace: true,
}
}
/// Feed a chunk of text and return the re-indented portion that is
/// ready to emit. Incomplete trailing lines are buffered internally.
pub fn push(&mut self, chunk: &str) -> String {
self.buffer.push_str(chunk);
self.drain(false)
}
/// Flush any remaining buffered content (call when the stream is done).
pub fn finish(&mut self) -> String {
self.drain(true)
}
fn drain(&mut self, is_final: bool) -> String {
let mut indented = String::new();
let mut start_ix = 0;
let mut newlines = self.buffer.match_indices('\n');
loop {
let (line_end, is_pending_line) = match newlines.next() {
Some((ix, _)) => (ix, false),
None => (self.buffer.len(), true),
};
let line = &self.buffer[start_ix..line_end];
if self.in_leading_whitespace {
if let Some(non_whitespace_ix) = line.find(|c| self.delta.character() != c) {
// We found a non-whitespace character, adjust indentation
// based on the delta.
let new_indent_len =
cmp::max(0, non_whitespace_ix as isize + self.delta.len()) as usize;
indented.extend(iter::repeat(self.delta.character()).take(new_indent_len));
indented.push_str(&line[non_whitespace_ix..]);
self.in_leading_whitespace = false;
} else if is_pending_line && !is_final {
// We're still in leading whitespace and this line is incomplete.
// Stop processing until we receive more input.
break;
} else {
// This line is entirely whitespace. Push it without indentation.
indented.push_str(line);
}
} else {
indented.push_str(line);
}
if is_pending_line {
start_ix = line_end;
break;
} else {
self.in_leading_whitespace = true;
indented.push('\n');
start_ix = line_end + 1;
}
}
self.buffer.replace_range(..start_ix, "");
if is_final {
indented.push_str(&self.buffer);
self.buffer.clear();
}
indented
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_indent_single_chunk() {
let mut r = Reindenter::new(IndentDelta::Spaces(2));
let out = r.push(" abc\n def\n ghi");
// All three lines are emitted: "ghi" starts with spaces but
// contains non-whitespace, so it's processed immediately.
assert_eq!(out, " abc\n def\n ghi");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_outdent_tabs() {
let mut r = Reindenter::new(IndentDelta::Tabs(-2));
let out = r.push("\t\t\t\tabc\n\t\tdef\n\t\t\t\t\t\tghi");
assert_eq!(out, "\t\tabc\ndef\n\t\t\t\tghi");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_incremental_chunks() {
let mut r = Reindenter::new(IndentDelta::Spaces(2));
// Feed " ab" — the `a` is non-whitespace, so the line is
// processed immediately even without a trailing newline.
let out = r.push(" ab");
assert_eq!(out, " ab");
// Feed "c\n" — appended to the already-processed line (no longer
// in leading whitespace).
let out = r.push("c\n");
assert_eq!(out, "c\n");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_zero_delta() {
let mut r = Reindenter::new(IndentDelta::Spaces(0));
let out = r.push(" hello\n world\n");
assert_eq!(out, " hello\n world\n");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_clamp_negative_indent() {
let mut r = Reindenter::new(IndentDelta::Spaces(-10));
let out = r.push(" abc\n");
// max(0, 2 - 10) = 0, so no leading spaces.
assert_eq!(out, "abc\n");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_whitespace_only_lines() {
let mut r = Reindenter::new(IndentDelta::Spaces(2));
let out = r.push(" \n code\n");
// First line is all whitespace — emitted verbatim. Second line is indented.
assert_eq!(out, " \n code\n");
let out = r.finish();
assert_eq!(out, "");
}
#[test]
fn test_compute_indent_delta_spaces() {
let buffer = LineIndent {
tabs: 0,
spaces: 8,
line_blank: false,
};
let query = LineIndent {
tabs: 0,
spaces: 4,
line_blank: false,
};
let delta = compute_indent_delta(buffer, query);
assert_eq!(delta.len(), 4);
assert_eq!(delta.character(), ' ');
}
#[test]
fn test_compute_indent_delta_tabs() {
let buffer = LineIndent {
tabs: 2,
spaces: 0,
line_blank: false,
};
let query = LineIndent {
tabs: 3,
spaces: 0,
line_blank: false,
};
let delta = compute_indent_delta(buffer, query);
assert_eq!(delta.len(), -1);
assert_eq!(delta.character(), '\t');
}
}

View file

@ -17,6 +17,7 @@ mod save_file_tool;
mod spawn_agent_tool;
mod streaming_edit_file_tool;
mod terminal_tool;
mod tool_edit_parser;
mod tool_permissions;
mod web_search_tool;

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,941 @@
use smallvec::SmallVec;
use crate::{Edit, PartialEdit};
/// Events emitted by `ToolEditParser` as tool call input streams in.
#[derive(Debug, PartialEq, Eq)]
pub enum ToolEditEvent {
/// A chunk of `old_text` for an edit operation.
OldTextChunk {
edit_index: usize,
chunk: String,
done: bool,
},
/// A chunk of `new_text` for an edit operation.
NewTextChunk {
edit_index: usize,
chunk: String,
done: bool,
},
/// A chunk of content for write/overwrite mode.
ContentChunk { chunk: String },
}
/// Tracks the streaming state of a single edit to detect deltas.
#[derive(Default, Debug)]
struct EditStreamState {
old_text_emitted_len: usize,
old_text_done: bool,
new_text_emitted_len: usize,
new_text_done: bool,
}
/// Converts incrementally-growing tool call JSON into a stream of chunk events.
///
/// The tool call streaming infrastructure delivers partial JSON objects where
/// string fields grow over time. This parser compares consecutive partials,
/// computes the deltas, and emits `ToolEditEvent`s that downstream pipeline
/// stages (`StreamingFuzzyMatcher` for old_text, `StreamingDiff` for new_text)
/// can consume incrementally.
///
/// Because partial JSON comes through a fixer (`partial-json-fixer`) that
/// closes incomplete escape sequences, a string can temporarily contain wrong
/// trailing characters (e.g. a literal `\` instead of `\n`). We handle this
/// by holding back trailing backslash characters in non-finalized chunks: if
/// a partial string ends with `\` (0x5C), that byte is not emitted until the
/// next partial confirms or corrects it. This avoids feeding corrupted bytes
/// to downstream consumers.
#[derive(Default, Debug)]
pub struct ToolEditParser {
edit_states: Vec<EditStreamState>,
content_emitted_len: usize,
}
impl ToolEditParser {
/// Push a new set of partial edits (from edit mode) and return any events.
///
/// Each call should pass the *entire current* edits array as seen in the
/// latest partial input. The parser will diff it against its internal state
/// to produce only the new events.
pub fn push_edits(&mut self, edits: &[PartialEdit]) -> SmallVec<[ToolEditEvent; 4]> {
let mut events = SmallVec::new();
for (index, partial) in edits.iter().enumerate() {
if index >= self.edit_states.len() {
// A new edit appeared — finalize the previous one if there was one.
if let Some(previous) = self.finalize_previous_edit(index) {
events.extend(previous);
}
self.edit_states.push(EditStreamState::default());
}
let state = &mut self.edit_states[index];
// Process old_text changes.
if let Some(old_text) = &partial.old_text
&& !state.old_text_done
{
if partial.new_text.is_some() {
// new_text appeared, so old_text is done — emit everything.
let start = state.old_text_emitted_len.min(old_text.len());
let chunk = old_text[start..].to_string();
state.old_text_done = true;
state.old_text_emitted_len = old_text.len();
events.push(ToolEditEvent::OldTextChunk {
edit_index: index,
chunk,
done: true,
});
} else {
let safe_end = safe_emit_end(old_text);
if safe_end > state.old_text_emitted_len {
let chunk = old_text[state.old_text_emitted_len..safe_end].to_string();
state.old_text_emitted_len = safe_end;
events.push(ToolEditEvent::OldTextChunk {
edit_index: index,
chunk,
done: false,
});
}
}
}
// Process new_text changes.
if let Some(new_text) = &partial.new_text
&& !state.new_text_done
{
let safe_end = safe_emit_end(new_text);
if safe_end > state.new_text_emitted_len {
let chunk = new_text[state.new_text_emitted_len..safe_end].to_string();
state.new_text_emitted_len = safe_end;
events.push(ToolEditEvent::NewTextChunk {
edit_index: index,
chunk,
done: false,
});
}
}
}
events
}
/// Push new content and return any events.
///
/// Each call should pass the *entire current* content string. The parser
/// will diff it against its internal state to emit only the new chunk.
pub fn push_content(&mut self, content: &str) -> SmallVec<[ToolEditEvent; 1]> {
let mut events = SmallVec::new();
let safe_end = safe_emit_end(content);
if safe_end > self.content_emitted_len {
let chunk = content[self.content_emitted_len..safe_end].to_string();
self.content_emitted_len = safe_end;
events.push(ToolEditEvent::ContentChunk { chunk });
}
events
}
/// Finalize all edits with the complete input. This emits `done: true`
/// events for any in-progress old_text or new_text that hasn't been
/// finalized yet.
///
/// `final_edits` should be the fully deserialized final edits array. The
/// parser compares against its tracked state and emits any remaining deltas
/// with `done: true`.
pub fn finalize_edits(&mut self, edits: &[Edit]) -> SmallVec<[ToolEditEvent; 4]> {
let mut events = SmallVec::new();
for (index, edit) in edits.iter().enumerate() {
if index >= self.edit_states.len() {
// This edit was never seen in partials — emit it fully.
if let Some(previous) = self.finalize_previous_edit(index) {
events.extend(previous);
}
self.edit_states.push(EditStreamState::default());
}
let state = &mut self.edit_states[index];
if !state.old_text_done {
let start = state.old_text_emitted_len.min(edit.old_text.len());
let chunk = edit.old_text[start..].to_string();
state.old_text_done = true;
state.old_text_emitted_len = edit.old_text.len();
events.push(ToolEditEvent::OldTextChunk {
edit_index: index,
chunk,
done: true,
});
}
if !state.new_text_done {
let start = state.new_text_emitted_len.min(edit.new_text.len());
let chunk = edit.new_text[start..].to_string();
state.new_text_done = true;
state.new_text_emitted_len = edit.new_text.len();
events.push(ToolEditEvent::NewTextChunk {
edit_index: index,
chunk,
done: true,
});
}
}
events
}
/// Finalize content with the complete input.
pub fn finalize_content(&mut self, content: &str) -> SmallVec<[ToolEditEvent; 1]> {
let mut events = SmallVec::new();
let start = self.content_emitted_len.min(content.len());
if content.len() > start {
let chunk = content[start..].to_string();
self.content_emitted_len = content.len();
events.push(ToolEditEvent::ContentChunk { chunk });
}
events
}
/// When a new edit appears at `index`, finalize the edit at `index - 1`
/// by emitting a `NewTextChunk { done: true }` if it hasn't been finalized.
fn finalize_previous_edit(&mut self, new_index: usize) -> Option<SmallVec<[ToolEditEvent; 2]>> {
if new_index == 0 || self.edit_states.is_empty() {
return None;
}
let previous_index = new_index - 1;
if previous_index >= self.edit_states.len() {
return None;
}
let state = &mut self.edit_states[previous_index];
let mut events = SmallVec::new();
// If old_text was never finalized, finalize it now with an empty done chunk.
if !state.old_text_done {
state.old_text_done = true;
events.push(ToolEditEvent::OldTextChunk {
edit_index: previous_index,
chunk: String::new(),
done: true,
});
}
// Emit a done event for new_text if not already finalized.
if !state.new_text_done {
state.new_text_done = true;
events.push(ToolEditEvent::NewTextChunk {
edit_index: previous_index,
chunk: String::new(),
done: true,
});
}
Some(events)
}
}
/// Returns the byte position up to which it is safe to emit from a partial
/// string. If the string ends with a backslash (`\`, 0x5C), that byte is
/// held back because it may be an artifact of the partial JSON fixer closing
/// an incomplete escape sequence (e.g. turning a half-received `\n` into `\\`).
/// The next partial will reveal the correct character.
fn safe_emit_end(text: &str) -> usize {
if text.as_bytes().last() == Some(&b'\\') {
text.len() - 1
} else {
text.len()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_single_edit_streamed_incrementally() {
let mut parser = ToolEditParser::default();
// old_text arrives in chunks: "hell" → "hello w" → "hello world"
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hell".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "hell".into(),
done: false,
}]
);
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello w".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "o w".into(),
done: false,
}]
);
// new_text appears → old_text finalizes
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello world".into()),
new_text: Some("good".into()),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "orld".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "good".into(),
done: false,
},
]
);
// new_text grows
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello world".into()),
new_text: Some("goodbye world".into()),
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "bye world".into(),
done: false,
}]
);
// Finalize
let events = parser.finalize_edits(&[Edit {
old_text: "hello world".into(),
new_text: "goodbye world".into(),
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "".into(),
done: true,
}]
);
}
#[test]
fn test_multiple_edits_sequential() {
let mut parser = ToolEditParser::default();
// First edit streams in
let events = parser.push_edits(&[PartialEdit {
old_text: Some("first old".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "first old".into(),
done: false,
}]
);
let events = parser.push_edits(&[PartialEdit {
old_text: Some("first old".into()),
new_text: Some("first new".into()),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "first new".into(),
done: false,
},
]
);
// Second edit appears → first edit's new_text is finalized
let events = parser.push_edits(&[
PartialEdit {
old_text: Some("first old".into()),
new_text: Some("first new".into()),
},
PartialEdit {
old_text: Some("second".into()),
new_text: None,
},
]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "".into(),
done: true,
},
ToolEditEvent::OldTextChunk {
edit_index: 1,
chunk: "second".into(),
done: false,
},
]
);
// Finalize everything
let events = parser.finalize_edits(&[
Edit {
old_text: "first old".into(),
new_text: "first new".into(),
},
Edit {
old_text: "second old".into(),
new_text: "second new".into(),
},
]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 1,
chunk: " old".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 1,
chunk: "second new".into(),
done: true,
},
]
);
}
#[test]
fn test_content_streamed_incrementally() {
let mut parser = ToolEditParser::default();
let events = parser.push_content("hello");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk {
chunk: "hello".into(),
}]
);
let events = parser.push_content("hello world");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk {
chunk: " world".into(),
}]
);
// No change
let events = parser.push_content("hello world");
assert!(events.is_empty());
let events = parser.push_content("hello world!");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk { chunk: "!".into() }]
);
// Finalize with no additional content
let events = parser.finalize_content("hello world!");
assert!(events.is_empty());
}
#[test]
fn test_finalize_content_with_remaining() {
let mut parser = ToolEditParser::default();
parser.push_content("partial");
let events = parser.finalize_content("partial content here");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk {
chunk: " content here".into(),
}]
);
}
#[test]
fn test_content_trailing_backslash_held_back() {
let mut parser = ToolEditParser::default();
// Partial JSON fixer turns incomplete \n into \\ (literal backslash).
// The trailing backslash is held back.
let events = parser.push_content("hello,\\");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk {
chunk: "hello,".into(),
}]
);
// Next partial corrects the escape to an actual newline.
// The held-back byte was wrong; the correct newline is emitted.
let events = parser.push_content("hello,\n");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk { chunk: "\n".into() }]
);
// Normal growth.
let events = parser.push_content("hello,\nworld");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk {
chunk: "world".into(),
}]
);
}
#[test]
fn test_content_finalize_with_trailing_backslash() {
let mut parser = ToolEditParser::default();
// Stream a partial with a fixer-corrupted trailing backslash.
// The backslash is held back.
parser.push_content("abc\\");
// Finalize reveals the correct character.
let events = parser.finalize_content("abc\n");
assert_eq!(
events.as_slice(),
&[ToolEditEvent::ContentChunk { chunk: "\n".into() }]
);
}
#[test]
fn test_no_partials_direct_finalize() {
let mut parser = ToolEditParser::default();
let events = parser.finalize_edits(&[Edit {
old_text: "old".into(),
new_text: "new".into(),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "old".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "new".into(),
done: true,
},
]
);
}
#[test]
fn test_no_partials_direct_finalize_multiple() {
let mut parser = ToolEditParser::default();
let events = parser.finalize_edits(&[
Edit {
old_text: "first old".into(),
new_text: "first new".into(),
},
Edit {
old_text: "second old".into(),
new_text: "second new".into(),
},
]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "first old".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "first new".into(),
done: true,
},
ToolEditEvent::OldTextChunk {
edit_index: 1,
chunk: "second old".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 1,
chunk: "second new".into(),
done: true,
},
]
);
}
#[test]
fn test_old_text_no_growth() {
let mut parser = ToolEditParser::default();
let events = parser.push_edits(&[PartialEdit {
old_text: Some("same".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "same".into(),
done: false,
}]
);
// Same old_text, no new_text → no events
let events = parser.push_edits(&[PartialEdit {
old_text: Some("same".into()),
new_text: None,
}]);
assert!(events.is_empty());
}
#[test]
fn test_old_text_none_then_appears() {
let mut parser = ToolEditParser::default();
// Edit exists but old_text is None (field hasn't arrived yet)
let events = parser.push_edits(&[PartialEdit {
old_text: None,
new_text: None,
}]);
assert!(events.is_empty());
// old_text appears
let events = parser.push_edits(&[PartialEdit {
old_text: Some("text".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "text".into(),
done: false,
}]
);
}
#[test]
fn test_empty_old_text_with_new_text() {
let mut parser = ToolEditParser::default();
// old_text is empty, new_text appears immediately
let events = parser.push_edits(&[PartialEdit {
old_text: Some("".into()),
new_text: Some("inserted".into()),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "inserted".into(),
done: false,
},
]
);
}
#[test]
fn test_three_edits_streamed() {
let mut parser = ToolEditParser::default();
// Stream first edit
parser.push_edits(&[PartialEdit {
old_text: Some("a".into()),
new_text: Some("A".into()),
}]);
// Second edit appears
parser.push_edits(&[
PartialEdit {
old_text: Some("a".into()),
new_text: Some("A".into()),
},
PartialEdit {
old_text: Some("b".into()),
new_text: Some("B".into()),
},
]);
// Third edit appears
let events = parser.push_edits(&[
PartialEdit {
old_text: Some("a".into()),
new_text: Some("A".into()),
},
PartialEdit {
old_text: Some("b".into()),
new_text: Some("B".into()),
},
PartialEdit {
old_text: Some("c".into()),
new_text: None,
},
]);
// Should finalize edit 1 (index=1) and start edit 2 (index=2)
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::NewTextChunk {
edit_index: 1,
chunk: "".into(),
done: true,
},
ToolEditEvent::OldTextChunk {
edit_index: 2,
chunk: "c".into(),
done: false,
},
]
);
// Finalize
let events = parser.finalize_edits(&[
Edit {
old_text: "a".into(),
new_text: "A".into(),
},
Edit {
old_text: "b".into(),
new_text: "B".into(),
},
Edit {
old_text: "c".into(),
new_text: "C".into(),
},
]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 2,
chunk: "".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 2,
chunk: "C".into(),
done: true,
},
]
);
}
#[test]
fn test_finalize_with_unseen_old_text() {
let mut parser = ToolEditParser::default();
// Only saw partial old_text, never saw new_text in partials
parser.push_edits(&[PartialEdit {
old_text: Some("partial".into()),
new_text: None,
}]);
let events = parser.finalize_edits(&[Edit {
old_text: "partial old text".into(),
new_text: "replacement".into(),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: " old text".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "replacement".into(),
done: true,
},
]
);
}
#[test]
fn test_finalize_with_partially_seen_new_text() {
let mut parser = ToolEditParser::default();
parser.push_edits(&[PartialEdit {
old_text: Some("old".into()),
new_text: Some("partial".into()),
}]);
let events = parser.finalize_edits(&[Edit {
old_text: "old".into(),
new_text: "partial new text".into(),
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: " new text".into(),
done: true,
}]
);
}
#[test]
fn test_repeated_pushes_with_no_change() {
let mut parser = ToolEditParser::default();
let events = parser.push_edits(&[PartialEdit {
old_text: Some("stable".into()),
new_text: Some("also stable".into()),
}]);
assert_eq!(events.len(), 2); // old done + new chunk
// Push the exact same data again
let events = parser.push_edits(&[PartialEdit {
old_text: Some("stable".into()),
new_text: Some("also stable".into()),
}]);
assert!(events.is_empty());
// And again
let events = parser.push_edits(&[PartialEdit {
old_text: Some("stable".into()),
new_text: Some("also stable".into()),
}]);
assert!(events.is_empty());
}
#[test]
fn test_old_text_trailing_backslash_held_back() {
let mut parser = ToolEditParser::default();
// Partial-json-fixer produces a literal backslash when the JSON stream
// cuts in the middle of an escape sequence like \n. The parser holds
// back the trailing backslash instead of emitting it.
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello,\\".into()), // fixer closed incomplete \n as \\
new_text: None,
}]);
// The trailing `\` is held back — only "hello," is emitted.
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "hello,".into(),
done: false,
}]
);
// Next partial: the fixer corrects the escape to \n.
// The held-back byte was wrong, but we never emitted it. Now the
// correct newline at that position is emitted normally.
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello,\n".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "\n".into(),
done: false,
}]
);
// Continue normally.
let events = parser.push_edits(&[PartialEdit {
old_text: Some("hello,\nworld".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "world".into(),
done: false,
}]
);
}
#[test]
fn test_multiline_old_and_new_text() {
let mut parser = ToolEditParser::default();
let events = parser.push_edits(&[PartialEdit {
old_text: Some("line1\nline2".into()),
new_text: None,
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "line1\nline2".into(),
done: false,
}]
);
let events = parser.push_edits(&[PartialEdit {
old_text: Some("line1\nline2\nline3".into()),
new_text: Some("LINE1\n".into()),
}]);
assert_eq!(
events.as_slice(),
&[
ToolEditEvent::OldTextChunk {
edit_index: 0,
chunk: "\nline3".into(),
done: true,
},
ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "LINE1\n".into(),
done: false,
},
]
);
let events = parser.push_edits(&[PartialEdit {
old_text: Some("line1\nline2\nline3".into()),
new_text: Some("LINE1\nLINE2\nLINE3".into()),
}]);
assert_eq!(
events.as_slice(),
&[ToolEditEvent::NewTextChunk {
edit_index: 0,
chunk: "LINE2\nLINE3".into(),
done: false,
}]
);
}
}