From 6cf4b810c00d5d0eb9265874ecfb44105e57f209 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 23 Sep 2026 22:04:38 -0700 Subject: [PATCH] test(qa): select compaction requests by their session identity After mock request ownership moved to transport affinity, the mutating-tool compaction scenario still searched prompt text for the session identifier. The provider recorded the real context overflow, but those stale selectors found no owning requests. A copied foreign identifier in prompt content could also match the wrong session. Select overflow, write, and continuation requests using their existing request.sessionId field. Exercise the actual catalog expressions with the owner identifier absent from prompt text and a foreign identifier copied into it. Preserve the mutation, pruning, ordering, count, and failure checks. No shared helper or production session contract changes. The real scenario failed before the selector repair and passed all three after samples: request size fell from 333954 to 119338 bytes, with exactly one logical write, one authenticated wire success, and one compaction. The changed catalog test passed 20 pressure runs at eight workers on two CPUs plus a CPU contender. All 58 catalog consumers passed across the four canonical CI shards (312 files, 4485 cases); the affected 78-file, 1062-case shard passed three times. Literal one-worker unit cost: 2.794s. A fresh real scenario replay on main, including the guarded session observer, passed in 21.309s. Proof runs: 35954197301 and 35957749551. The changed checker completed all type graphs, guards and dead-code scans. Remaining global lint was stopped in favor of scoped type-aware lint with a rejecting negative canary; that substitute, import-cycle checks, formatting, whitespace checks and independent P2 review passed. --- .../src/scenario-catalog-compaction.test.ts | 53 +++++++++++++++++-- .../compaction-retry-mutating-tool.yaml | 6 +-- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts index 2cde963fb5a0..bc6a3de07c92 100644 --- a/extensions/qa-lab/src/scenario-catalog-compaction.test.ts +++ b/extensions/qa-lab/src/scenario-catalog-compaction.test.ts @@ -137,6 +137,55 @@ describe("qa compaction scenario catalog", () => { const writeTranscriptToolCallIdExpr = readSetExpression("writeTranscriptToolCallId"); const continuationChainExpr = readSetExpression("continuationChain"); const compactionSummaryRequestsExpr = readSetExpression("compactionSummaryRequests"); + const sessionId = "seeded-transcript"; + const allInputText = "Compaction retry mutating tool check QA-COMPACTION-DURABLE-MARKER"; + const overflow = { + cursor: 1, + sessionId, + allInputText, + requestKind: "agent-initial", + outcome: "error", + errorCode: "context_length_exceeded", + }; + const write = { cursor: 2, sessionId, plannedToolName: "write", allInputText }; + const failedWrite = { ...write, cursor: 3, outcome: "error", toolOutput: "failed" }; + const continuation = { cursor: 4, sessionId, requestKind: "tool-continuation", allInputText }; + const failedContinuation = { ...continuation, cursor: 5, outcome: "error" }; + const foreign = (request: T) => ({ + ...request, + sessionId: `${sessionId}-other`, + allInputText: `${allInputText} ${sessionId}`, + }); + const scope = { + sessionId, + config: scenario.execution.config, + overflowRequest: overflow, + writeRequest: write, + scenarioRequests: [ + overflow, + foreign(overflow), + { ...overflow, outcome: "success" }, + { ...overflow, errorCode: "other" }, + write, + failedWrite, + foreign(write), + { ...write, cursor: 0 }, + { ...write, plannedToolName: "read" }, + { ...write, allInputText: "QA-COMPACTION-DURABLE-MARKER" }, + { ...write, allInputText: "Compaction retry mutating tool check" }, + continuation, + failedContinuation, + foreign(continuation), + { ...continuation, cursor: 1 }, + { ...continuation, requestKind: "agent-initial" }, + ], + }; + expect(runInNewContext(readSetExpression("overflowRequests"), scope)).toEqual([overflow]); + expect(runInNewContext(writeRequestsExpr, scope)).toEqual([write, failedWrite]); + expect(runInNewContext(postWriteContinuationsExpr, scope)).toEqual([ + continuation, + failedContinuation, + ]); const continuationAssertIndex = actionIndex((action) => readFlowAssertExpression(action).includes("continuationChain.valid === true"), ); @@ -235,7 +284,6 @@ describe("qa compaction scenario catalog", () => { expect(flow).toContain("writeRequests.length === 1"); expect(writeRequestsExpr).toContain("request.plannedToolName === 'write'"); expect(writeRequestsExpr).toContain("request.cursor > overflowRequest.cursor"); - expect(writeRequestsExpr).toContain("String(request.allInputText ?? '').includes(sessionId)"); expect(writeRequestsExpr).toContain( "String(request.allInputText ?? '').includes(config.promptSnippet)", ); @@ -282,9 +330,6 @@ describe("qa compaction scenario catalog", () => { expect(flow).not.toContain("transcript.successfulToolCallCounts.write === 1"); expect(postWriteContinuationsExpr).toContain("request.requestKind === 'tool-continuation'"); expect(postWriteContinuationsExpr).toContain("request.cursor > writeRequest.cursor"); - expect(postWriteContinuationsExpr).toContain( - "String(request.allInputText ?? '').includes(sessionId)", - ); expect(postWriteContinuationsExpr).not.toContain("request.outcome"); expect(postWriteContinuationsExpr).not.toContain("request.plannedToolName"); expect(postWriteContinuationsExpr).not.toContain("request.toolOutputCallId"); diff --git a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml index caee852269a5..3e7567dfc265 100644 --- a/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml +++ b/qa/scenarios/runtime/compaction-retry-mutating-tool.yaml @@ -108,7 +108,7 @@ flow: expr: "scenarioRequests.map((request) => ({ cursor: request.cursor, kind: request.requestKind, outcome: request.outcome, code: request.errorCode ?? null, bytes: request.rawByteLength, inputChars: String(request.allInputText ?? '').length, tailBlocks: [...new Set(Array.from({ length: 16 }, (_, index) => String(index).padStart(2, '0')).filter((id) => String(request.allInputText ?? '').includes(`tail_${id}_${String(config.tailTokenCount - 1).padStart(4, '0')}`)))].sort().slice(0, 16), prompt: String(request.allInputText ?? '').includes(config.promptSnippet), durable: String(request.allInputText ?? '').includes(config.durableMarker), bulky: String(request.allInputText ?? '').includes(config.bulkyMarker), qualityRetry: String(request.allInputText ?? '').includes('Previous summary failed quality checks'), tool: request.plannedToolName ?? null, resolvedWireTool: request.plannedWireToolName ?? request.plannedToolName ?? null, callId: request.plannedToolCallId ?? null, itemId: request.plannedToolItemId ?? null, transcriptId: typeof request.plannedToolItemId === 'string' && request.plannedToolItemId.length > 0 ? `${request.plannedToolCallId}|${request.plannedToolItemId}` : request.plannedToolCallId ?? null }))" - set: overflowRequests value: - expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'error' && request.errorCode === 'context_length_exceeded' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" + expr: "scenarioRequests.filter((request) => request.requestKind === 'agent-initial' && request.outcome === 'error' && request.errorCode === 'context_length_exceeded' && request.sessionId === sessionId && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" - assert: expr: "overflowRequests.length === 1" message: @@ -118,7 +118,7 @@ flow: expr: "overflowRequests[0]" - set: writeRequests value: - expr: "scenarioRequests.filter((request) => request.cursor > overflowRequest.cursor && request.plannedToolName === 'write' && String(request.allInputText ?? '').includes(sessionId) && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" + expr: "scenarioRequests.filter((request) => request.cursor > overflowRequest.cursor && request.plannedToolName === 'write' && request.sessionId === sessionId && String(request.allInputText ?? '').includes(config.promptSnippet) && String(request.allInputText ?? '').includes(config.durableMarker))" - assert: expr: "writeRequests.length === 1" message: @@ -168,7 +168,7 @@ flow: expr: "`compacted retry did not retain durable current context: ${JSON.stringify({ overflow: overflowEvidence, write: writeEvidence })}`" - set: postWriteContinuations value: - expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.cursor > writeRequest.cursor && String(request.allInputText ?? '').includes(sessionId)).toSorted((left, right) => left.cursor - right.cursor)" + expr: "scenarioRequests.filter((request) => request.requestKind === 'tool-continuation' && request.cursor > writeRequest.cursor && request.sessionId === sessionId).toSorted((left, right) => left.cursor - right.cursor)" - set: continuationChain value: expr: "(() => { const requests = []; const waits = []; const followedCallIds = new Set(); let currentCallId = writeRequest.plannedToolCallId; let previousCursor = writeRequest.cursor; let terminal; let valid = typeof currentCallId === 'string' && currentCallId.length > 0; while (valid) { if (followedCallIds.has(currentCallId)) { valid = false; break; } followedCallIds.add(currentCallId); const matches = postWriteContinuations.filter((request) => request.cursor > previousCursor && request.toolOutputCallId === currentCallId); if (matches.length !== 1) { valid = false; break; } const request = matches[0]; requests.push(request); previousCursor = request.cursor; if (request.plannedToolName === 'wait') { if (typeof request.plannedToolCallId !== 'string' || request.plannedToolCallId.length === 0) { valid = false; break; } waits.push(request); currentCallId = request.plannedToolCallId; continue; } if (request.plannedToolName === undefined) { terminal = request; break; } valid = false; } return { valid, requests, waits, terminal }; })()"