mirror of
https://github.com/QwenLM/qwen-code.git
synced 2026-08-31 10:16:57 +00:00
docs(core): cover recovery-dedup line-boundary + normalization branches
Add JSDoc to getRecoveryContinuationSuffix calling out that its empty-input guard is defensive-only (the production caller already filters both sides), and document appendRecoveryContinuationParts' implicit coupling with processStreamResponse's text-part consolidation plus its return-shape convention that coalesceRecoveryPairs relies on for multi-iteration recovery. Add two regression tests: - mid-paragraph match rejection: a structural anchor that appears in the previous tail but is not preceded by a newline must NOT trigger the contained-prefix strip, so legitimate continuation survives verbatim. - newline-normalization branch: when the replayed prefix ends with \n but the previous tail does not and the suffix does not start with \n, the helper must insert a separator so the coalesced text keeps its block boundary. Generated with AI Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
parent
08f2abd49e
commit
556b015de7
2 changed files with 128 additions and 0 deletions
|
|
@ -2765,6 +2765,97 @@ describe('GeminiChat', async () => {
|
|||
expect(text).toBe(previous + continuation);
|
||||
});
|
||||
|
||||
it('should preserve continuation when its structural prefix appears mid-paragraph in the previous tail (line-boundary rejection)', async () => {
|
||||
// Regression: `previousTailContainsAtLineBoundary` must reject matches
|
||||
// that land mid-paragraph in `previousTail` even when a structural
|
||||
// anchor at the start of `continuationText` would otherwise pass the
|
||||
// contained-prefix gate. Without that check, a plain substring match
|
||||
// (e.g. inside a code block that quotes the literal string
|
||||
// `"### Heading\n..."` as prose) would silently strip legitimate
|
||||
// continuation. The only `"### Heading"` occurrence here is preceded
|
||||
// by `"some text"`, not a newline, so the contained-prefix path MUST
|
||||
// reject the match and pass the continuation through verbatim.
|
||||
const previous =
|
||||
'some text ### Heading and then more inline prose follows';
|
||||
const continuation =
|
||||
'### Heading\nfresh continuation that should not be stripped';
|
||||
const streams = [
|
||||
makeStream([makeChunk([{ text: 'discarded initial' }], 'MAX_TOKENS')]),
|
||||
makeStream([makeChunk([{ text: previous }], 'MAX_TOKENS')]),
|
||||
makeStream([makeChunk([{ text: continuation }], 'STOP')]),
|
||||
];
|
||||
let callIndex = 0;
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockImplementation(
|
||||
async () => streams[callIndex++]!,
|
||||
);
|
||||
|
||||
const stream = await chat.sendMessageStream(
|
||||
'gemini-3-pro',
|
||||
{ message: 'write something with a heading' },
|
||||
'prompt-recovery-line-boundary-reject',
|
||||
);
|
||||
for await (const _event of stream) {
|
||||
// consume
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
const lastEntry = history[history.length - 1]!;
|
||||
const text = lastEntry.parts
|
||||
?.map((part) => ('text' in part ? part.text : ''))
|
||||
.join('');
|
||||
// No silent strip: the full continuation must follow the previous tail
|
||||
// verbatim because the only `"### Heading"` occurrence in `previous`
|
||||
// is mid-paragraph (not preceded by `\n`).
|
||||
expect(text).toBe(previous + continuation);
|
||||
});
|
||||
|
||||
it('should insert a newline separator when the replayed prefix ends with newline but previous tail does not', async () => {
|
||||
// Covers the three-condition normalization branch in
|
||||
// `getRecoveryContinuationSuffix`: when `replayedPrefix` ends with
|
||||
// `\n`, `previousText` does NOT, and `suffix` does NOT start with
|
||||
// `\n`, the helper prepends a `\n` so the coalesced text keeps the
|
||||
// block-level boundary intact. Without normalization, the suffix
|
||||
// would butt up against the previous tail with no separator.
|
||||
//
|
||||
// Setup: previous tail ends with `### Section` (no trailing newline,
|
||||
// because the truncation cut the response immediately after the
|
||||
// heading). Continuation replays `### Section\n` followed by body
|
||||
// prose. The contained-prefix path strips the replayed heading +
|
||||
// newline, leaving a suffix that starts with prose. The
|
||||
// normalization branch must restore a `\n` between `### Section` in
|
||||
// history and the body prose.
|
||||
const previous = 'Intro paragraph.\n### Section';
|
||||
const replayedBlock = '### Section\n';
|
||||
const continuation = `${replayedBlock}body prose continuation`;
|
||||
const streams = [
|
||||
makeStream([makeChunk([{ text: 'discarded initial' }], 'MAX_TOKENS')]),
|
||||
makeStream([makeChunk([{ text: previous }], 'MAX_TOKENS')]),
|
||||
makeStream([makeChunk([{ text: continuation }], 'STOP')]),
|
||||
];
|
||||
let callIndex = 0;
|
||||
vi.mocked(mockContentGenerator.generateContentStream).mockImplementation(
|
||||
async () => streams[callIndex++]!,
|
||||
);
|
||||
|
||||
const stream = await chat.sendMessageStream(
|
||||
'gemini-3-pro',
|
||||
{ message: 'write a structured answer' },
|
||||
'prompt-recovery-newline-normalization',
|
||||
);
|
||||
for await (const _event of stream) {
|
||||
// consume
|
||||
}
|
||||
|
||||
const history = chat.getHistory();
|
||||
const lastEntry = history[history.length - 1]!;
|
||||
const text = lastEntry.parts
|
||||
?.map((part) => ('text' in part ? part.text : ''))
|
||||
.join('');
|
||||
// No duplicated `### Section`, and the heading is separated from the
|
||||
// body prose by exactly one newline — the normalization branch fired.
|
||||
expect(text).toBe(`${previous}\nbody prose continuation`);
|
||||
});
|
||||
|
||||
it('should drop continuation entirely when it exactly replays the previous tail', async () => {
|
||||
// Covers the full-overlap guard in getRecoveryContinuationSuffix:
|
||||
// previousText.endsWith(continuationText) AND the overlap is significant.
|
||||
|
|
|
|||
|
|
@ -297,6 +297,20 @@ function previousTailContainsAtLineBoundary(
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the portion of `continuationText` that should be appended to
|
||||
* `previousText` after a MAX_TOKENS recovery, stripping any overlap that the
|
||||
* provider replayed at the boundary.
|
||||
*
|
||||
* The empty-input guard (`previousText.length === 0 ||
|
||||
* continuationText.length === 0`) is *defensive only*. The sole production
|
||||
* caller is {@link appendRecoveryContinuationParts}, which already short-
|
||||
* circuits when either side has no plain-text part — neither branch of the
|
||||
* guard can fire from production code. It exists so that anyone reusing this
|
||||
* helper directly (e.g. a future unit test, a refactor that bypasses the
|
||||
* caller's filter) cannot crash or read out of bounds. We deliberately leave
|
||||
* the guard in place rather than rely on the caller's invariant alone.
|
||||
*/
|
||||
function getRecoveryContinuationSuffix(
|
||||
previousText: string,
|
||||
continuationText: string,
|
||||
|
|
@ -432,6 +446,29 @@ function buildOutputRecoveryMessage(previousModelTurn: Content | undefined) {
|
|||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Coalesce a recovery continuation turn into the preceding (truncated) model
|
||||
* turn, dropping any replayed overlap.
|
||||
*
|
||||
* Coupling with `processStreamResponse`. This function assumes the parts
|
||||
* arrays it receives were produced by {@link GeminiChat.processStreamResponse}
|
||||
* — i.e. all plain-text streaming chunks from a given turn have been
|
||||
* consolidated in place into a single text part via `lastPart.text +=
|
||||
* part.text`. The dedup logic only inspects the *last* plain-text part of
|
||||
* `previousParts` and the *first* plain-text part of `continuationParts`, so
|
||||
* if a future refactor of `processStreamResponse` ever emits multiple adjacent
|
||||
* unconsolidated text parts per turn, this function would compare the
|
||||
* continuation against only the trailing fragment and miss real overlaps with
|
||||
* earlier fragments. Both functions live in this file precisely so the
|
||||
* coupling is reviewable in a single window.
|
||||
*
|
||||
* Return-value shape. The returned array preserves the *shape convention* of
|
||||
* `processStreamResponse` output: `[thoughtPart?, ...consolidatedTextParts,
|
||||
* ...nonTextParts]`. {@link GeminiChat.coalesceRecoveryPairs} relies on this
|
||||
* by feeding the merged result back as `previousParts` on the next recovery
|
||||
* iteration; if the shape ever diverges, multi-iteration recovery dedup would
|
||||
* fail silently against the wrong part.
|
||||
*/
|
||||
function appendRecoveryContinuationParts(
|
||||
previousParts: Part[] | undefined,
|
||||
continuationParts: Part[] | undefined,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue