From 39cceeb1433296588e50b5abdc1e58f48bb0a0de Mon Sep 17 00:00:00 2001 From: Aiden Cline <63023139+rekram1-node@users.noreply.github.com> Date: Fri, 10 Jul 2026 00:33:21 -0500 Subject: [PATCH] fix(codemode): return promises from combinators (#35782) --- packages/codemode/README.md | 44 +- packages/codemode/codemode.md | 45 +- packages/codemode/interpreter-support.md | 14 +- packages/codemode/src/codemode.ts | 15 +- packages/codemode/src/interpreter/runtime.ts | 518 +++++----- packages/codemode/src/tool-runtime.ts | 1 + packages/codemode/src/values.ts | 8 +- packages/codemode/test/codemode.test.ts | 20 + .../codemode/test/promise-test262.test.ts | 906 ++++++++++++++++++ packages/codemode/test/promise.test.ts | 506 +++++++++- packages/codemode/test/test262-array.md | 77 -- packages/codemode/test/test262-string.md | 69 -- packages/core/src/tool/execute.ts | 9 +- packages/core/test/tool-execute.test.ts | 47 + 14 files changed, 1797 insertions(+), 482 deletions(-) create mode 100644 packages/codemode/test/promise-test262.test.ts delete mode 100644 packages/codemode/test/test262-array.md delete mode 100644 packages/codemode/test/test262-string.md create mode 100644 packages/core/test/tool-execute.test.ts diff --git a/packages/codemode/README.md b/packages/codemode/README.md index ee2c4162e6f..2b764075aee 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -130,6 +130,7 @@ type Result = Success | Failure interface Success { readonly ok: true readonly value: CodeMode.DataValue + readonly warnings?: ReadonlyArray readonly logs?: ReadonlyArray readonly truncated?: boolean readonly toolCalls: ReadonlyArray @@ -144,7 +145,7 @@ interface Failure { } ``` -`toolCalls` contains the names of calls admitted by the runtime in call order. It is retained on failure so hosts can audit partial execution without exposing inputs or host failures. `truncated` is present when the value or logs were cut to fit `maxOutputBytes` (see Execution Limits). +`toolCalls` contains the names of calls admitted by the runtime in call order. It is retained on failure so hosts can audit partial execution without exposing inputs or host failures. A successful execution may also contain `warnings`: runtime-authored, non-fatal diagnostics alongside a valid value - unhandled rejections from promises that failed, un-awaited, before the program returned, or background work interrupted by the timeout after the program returned. Anything still running when the program returns is interrupted - race losers and fire-and-forget calls alike - so a program must await every call whose completion matters. Failure has an `error`; success may have `warnings`; program-authored console output stays in `logs`. Keeping the value on an unhandled rejection is a deliberate divergence from Node's crash-on-unhandled-rejection default: the computed value and the background failure are independently useful to the model, so the result carries both. When warnings are cut by `maxOutputBytes`, a final `Truncated` diagnostic marks the omission in-band, and `truncated` marks any result, warning, or log truncation (see Execution Limits). ### Tool-call hooks @@ -257,11 +258,11 @@ CodeMode is an orchestration language, not a general JavaScript runtime. The limits are exactly three knobs: -| Limit | Default | Bounds | -| ---------------- | -------------------: | -------------------------------------------------------------------- | -| `timeoutMs` | none - no timeout | Wall-clock execution time. | -| `maxToolCalls` | none - unlimited | Tool calls admitted during the execution. | -| `maxOutputBytes` | none - no truncation | Model-facing output: the serialized result value plus captured logs. | +| Limit | Default | Bounds | +| ---------------- | -------------------: | ---------------------------------------------------- | +| `timeoutMs` | none - no timeout | Wall-clock execution time. | +| `maxToolCalls` | none - unlimited | Tool calls admitted during the execution. | +| `maxOutputBytes` | none - no truncation | Retained result value and logs; warnings separately. | No limit has a default, on purpose: execution budgets are host policy, not library policy - a host that wants a bound sets one; a host that can interrupt the execution fiber (as OpenCode does on user cancel) may set no timeout, and a host with its own tool-output truncation (as OpenCode has) may leave `maxOutputBytes` unset. A host with neither should set `maxOutputBytes`, or oversized results silently flood model context. @@ -279,9 +280,11 @@ const runtime = CodeMode.make({ Limits are safe integers. `timeoutMs` must be at least `1`; the others may be `0`. Invalid configuration throws a `RangeError` when `CodeMode.make` or `CodeMode.execute` is called. An explicitly `undefined` value is the same as leaving the limit unset. -Exceeding a configured `maxOutputBytes` never fails the execution. An oversized result value is replaced by its truncated serialized text plus an explanatory marker, logs are kept from the start until the remaining budget is exhausted (with a final marker line noting the cut), and the result carries `truncated: true`. +`maxOutputBytes` is a payload budget, not a strict byte cap on the final rendered tool message. It counts the serialized result value and retained log lines; warning diagnostics are bounded by a separate budget of the same size, so a large value never silences runtime diagnostics. Fixed truncation notices and framing added by a host when it renders the structured result are additional and may make the final message exceed the configured number. -When configured, the timeout interrupts in-flight tool Effects, including eagerly started calls the program has not awaited (their fibers are supervised by the execution). The interpreter yields cooperatively between steps, so the timeout also interrupts pure busy loops (`while (true) {}`) - no separate work budget exists. Tool implementations remain responsible for making their external operations interruptible or independently bounded. +Exceeding a configured `maxOutputBytes` never fails the execution. An oversized result value is replaced by its truncated serialized text plus an explanatory marker, logs are kept within the remaining budget, warnings are kept within their own budget, omitted entries receive a summary marker, and the result carries `truncated: true`. + +When configured, the timeout interrupts in-flight tool Effects, including eagerly started calls the program has not awaited (their fibers are supervised by the execution). The interpreter yields cooperatively between steps, so the timeout also interrupts pure busy loops (`while (true) {}`) - no separate work budget exists. Tool implementations remain responsible for making their external operations interruptible or independently bounded: the timeout bounds when interruption begins, not when the result is delivered, which waits for tool interruption cleanup to finish. If the timeout fires after the program has already returned a valid value - while the runtime is interrupting leftover work and waiting for its cleanup - the result stays successful: the computed value is returned with a `TimeoutExceeded` warning instead of being discarded. Two interpreter internals are fixed constants rather than knobs: at most 8 tool calls run concurrently, and values crossing a data boundary may nest at most 32 levels deep (deeper values fail as `InvalidDataValue`, which reads better than a native stack-overflow error). Neither is part of the public contract. @@ -289,18 +292,19 @@ Two interpreter internals are fixed constants rather than knobs: at most 8 tool Failures are data: -| Kind | Meaning | -| ----------------------- | -------------------------------------------------------------------------------------------------------- | -| `ParseError` | Source is empty or cannot be parsed. | -| `UnsupportedSyntax` | Parsed JavaScript is outside the supported subset. | -| `UnknownTool` | A program referenced a tool the host did not provide. | -| `InvalidToolInput` | Tool input failed schema decoding or safe-data copying. | -| `InvalidToolOutput` | Tool output failed schema decoding or safe-data copying. | -| `InvalidDataValue` | Program data violated the plain-data contract (depth, circularity, blocked properties, non-data values). | -| `ToolCallLimitExceeded` | Calls exceeded `maxToolCalls`. | -| `TimeoutExceeded` | Execution exceeded `timeoutMs`. | -| `ToolFailure` | A tool refused or failed. | -| `ExecutionFailure` | The program threw or another execution error occurred. | +| Kind | Meaning | +| ----------------------- | --------------------------------------------------------------------------------------------------------- | +| `ParseError` | Source is empty or cannot be parsed. | +| `UnsupportedSyntax` | Parsed JavaScript is outside the supported subset. | +| `UnknownTool` | A program referenced a tool the host did not provide. | +| `InvalidToolInput` | Tool input failed schema decoding or safe-data copying. | +| `InvalidToolOutput` | Tool output failed schema decoding or safe-data copying. | +| `InvalidDataValue` | Program data violated the plain-data contract (depth, circularity, blocked properties, non-data values). | +| `ToolCallLimitExceeded` | Calls exceeded `maxToolCalls`. | +| `TimeoutExceeded` | Execution exceeded `timeoutMs`; as a warning, background work was interrupted after the program returned. | +| `ToolFailure` | A tool refused or failed. | +| `ExecutionFailure` | The program threw or another execution error occurred. | +| `Truncated` | Warning-only marker: additional warnings were omitted by `maxOutputBytes`. | Unknown host failures, defects, invalid outputs, and copying failures are sanitized. To return a safe operational refusal, fail with `toolError`: diff --git a/packages/codemode/codemode.md b/packages/codemode/codemode.md index 8a93beca510..0428ff02517 100644 --- a/packages/codemode/codemode.md +++ b/packages/codemode/codemode.md @@ -63,13 +63,24 @@ path lookup, namespace browsing, deterministic ranking, and pagination. ### Tool execution -Calling a tool starts its Effect eagerly on a supervised fiber. The returned sandbox promise is run-once and can be -awaited directly or through the supported `Promise` combinators. At most eight tool calls execute concurrently. -Unfinished calls are drained before successful program completion, and an unhandled call failure becomes a diagnostic. +Every sandbox promise starts eagerly on a run-once fiber owned by the whole CodeMode execution, including tool calls, +async functions, `Promise.all`, `Promise.allSettled`, `Promise.race`, `Promise.resolve`, and `Promise.reject`. Nested +functions therefore cannot end the lifetime of work they started. Independent aggregate batches overlap, and rejection +is observed at the eventual `await`. `Promise.race` uses native non-cancelling settlement semantics: its first result +wins while losers continue running. At normal completion CodeMode interrupts everything still running - race losers, +fail-fast `Promise.all` stragglers, and fire-and-forget calls alike: the program has returned, so no future await can +exist, and work whose completion matters must be awaited by the program. Waiting for any class of leftover instead +would let it hold the execution open, or deadlock it when queued work needs tool-call permits the leftovers occupy. +Rejections that settled un-awaited before the return become `Success.warnings` diagnostics. A fatal program failure or +host interruption closes the execution promise scope and interrupts its active fibers instead. A timeout does the +same, except that a value the program already returned is preserved alongside a `TimeoutExceeded` warning rather than +discarded. At most eight tool calls execute concurrently. The public execution-policy knobs are `timeoutMs`, `maxToolCalls`, and `maxOutputBytes`. The package supplies no defaults because budgets are host policy. The interpreter also enforces fixed internal boundaries for tool-call -concurrency and data nesting depth. +concurrency and data nesting depth. `maxOutputBytes` bounds retained payload bytes, not the complete rendered message; +warning diagnostics have an equal separate budget so a large value cannot starve them, and fixed truncation notices and +host-added framing are intentionally outside the budgets. ### Data, files, and failures @@ -79,7 +90,7 @@ boundary. Unknown host failures and invalid outputs are sanitized. `ToolError` is the explicit channel for a safe message that a tool wants the model to see. Diagnostic categories distinguish parsing, unsupported syntax, unknown tools, invalid -data, tool failures, limits, timeouts, and execution failures. +data, tool failures, limits, timeouts, execution failures, and warning truncation. Files and other attachment content stay outside the interpreter. A host may collect them while child tools execute and attach them to the outer result, but the program receives only the structured tool output. @@ -127,18 +138,18 @@ represent accurately rather than guessing semantics. ## Decisions and Rationale -| Decision | Rationale | -| -------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| Keep an owned tree-walking interpreter. | The product need is bounded tool orchestration, not arbitrary JavaScript. Owning the language surface keeps authority and behavior explicit. | -| Treat schemas as the model-facing interface. | Signatures drive correct calls; Effect Schema also provides the runtime validation boundary, while JSON Schema supports adapter interoperability. | -| Keep authority host-owned. | CodeMode can only confine programs to supplied tools. The host chooses those tools, and each tool enforces its own authorization and side-effect policy. | -| Use progressive catalog disclosure plus search. | Large tool sets should not consume the prompt, but every namespace must remain discoverable and speculative search calls should remain valid. | -| Start tool promises eagerly and supervise them. | This preserves normal call-time parallelism while giving each call run-once settlement and interruption safety. | -| Keep files outside the sandbox value space. | Models should compose structured data without routing binary payloads through generated code or context. | -| Treat `execute` as the model-facing invocation boundary. | Nested calls are implementation details of one orchestration program. Reusing the outer context and bounding only the final result preserves complete intermediate data without inventing durable child-call identities. | -| Return expected failures as data. | Models need actionable diagnostics without exposing private host causes; host interruption and defects must still propagate correctly. | -| Leave execution-limit defaults to hosts. | Appropriate budgets depend on the surrounding product and its own cancellation, retention, and output-bounding policies. | -| Skip unsupported OpenAPI operations. | Incorrect parameter encoding, authentication, or transport behavior is worse than a precise `skipped` reason. | +| Decision | Rationale | +| ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Keep an owned tree-walking interpreter. | The product need is bounded tool orchestration, not arbitrary JavaScript. Owning the language surface keeps authority and behavior explicit. | +| Treat schemas as the model-facing interface. | Signatures drive correct calls; Effect Schema also provides the runtime validation boundary, while JSON Schema supports adapter interoperability. | +| Keep authority host-owned. | CodeMode can only confine programs to supplied tools. The host chooses those tools, and each tool enforces its own authorization and side-effect policy. | +| Use progressive catalog disclosure plus search. | Large tool sets should not consume the prompt, but every namespace must remain discoverable and speculative search calls should remain valid. | +| Start promises eagerly and supervise them for the execution. | This preserves normal call-time parallelism and run-once settlement while allowing pending work to be interrupted when the program returns. | +| Keep files outside the sandbox value space. | Models should compose structured data without routing binary payloads through generated code or context. | +| Treat `execute` as the model-facing invocation boundary. | Nested calls are implementation details of one orchestration program. Reusing the outer context and bounding only the final result preserves complete intermediate data without inventing durable child-call identities. | +| Return expected failures as data. | Models need actionable diagnostics without exposing private host causes; host interruption and defects must still propagate correctly. | +| Leave execution-limit defaults to hosts. | Appropriate budgets depend on the surrounding product and its own cancellation, retention, and output-bounding policies. | +| Skip unsupported OpenAPI operations. | Incorrect parameter encoding, authentication, or transport behavior is worse than a precise `skipped` reason. | ## Remaining Work diff --git a/packages/codemode/interpreter-support.md b/packages/codemode/interpreter-support.md index aac4ac938cd..6887fdaed94 100644 --- a/packages/codemode/interpreter-support.md +++ b/packages/codemode/interpreter-support.md @@ -111,13 +111,15 @@ ultimate source of truth. - [x] `Promise.resolve` and `Promise.reject`. - [x] `Promise.all`, `Promise.allSettled`, and `Promise.race` over supported collections containing promises and plain values. -- [x] `Promise.all` preserves result order and rejects on the first observed failure. +- [x] `Promise.all` preserves result order and rejects on the first observed failure without cancelling siblings. - [x] `Promise.allSettled` returns plain fulfilled/rejected outcome records. -- [x] `Promise.race` interrupts losing in-flight tool calls. -- [x] Un-awaited calls are drained before execution ends; unhandled failures become diagnostics. +- [x] `Promise.race` settles from the first result without cancelling losers at settlement time. +- [x] Real promise values from `Promise.all`, `Promise.allSettled`, and `Promise.race`; separately constructed + combinator batches overlap as in normal JavaScript. +- [x] All still-pending work (race losers, fail-fast `Promise.all` stragglers, and un-awaited calls alike) is + interrupted when the program returns; rejections that settled un-awaited become `Success.warnings` + diagnostics. - [x] `try`/`catch` can handle awaited tool and promise failures. -- [ ] Real promise values from `Promise.all`, `Promise.allSettled`, and `Promise.race`. These calls currently settle - before returning, so separately constructed combinator batches do not overlap as normal JavaScript promises do. - [ ] `Promise.any`. - [ ] Promise chaining with `.then`, `.catch`, and `.finally`. - [ ] Custom promise construction with `new Promise(...)`. @@ -269,7 +271,7 @@ ultimate source of truth. These are actionable implementation items. Check them off only when behavior and direct tests land. -- [ ] Return real promises from `Promise.all`, `Promise.allSettled`, and `Promise.race`. +- [x] Return real promises from `Promise.all`, `Promise.allSettled`, and `Promise.race`. - [ ] Bound pending tool-call admission/allocation in addition to execution concurrency. - [ ] Guarantee every advertised tool path is executable, including dotted and blocked path segments. - [ ] Define safe outbound handling for non-finite numbers and `undefined` so invalid values cannot silently become diff --git a/packages/codemode/src/codemode.ts b/packages/codemode/src/codemode.ts index 842209dac57..f35294a2274 100644 --- a/packages/codemode/src/codemode.ts +++ b/packages/codemode/src/codemode.ts @@ -13,11 +13,17 @@ export type { ToolCall, ToolCallEnded, ToolCallHooks, ToolCallStarted, ToolDescr /** Resource budgets enforced independently during each CodeMode program execution. */ export type ExecutionLimits = { - /** Maximum wall-clock execution time in milliseconds. No default: absent means no timeout. */ + /** + * Wall-clock milliseconds before execution is interrupted; result delivery additionally + * waits for tool interruption cleanup. No default: absent means no timeout. + */ readonly timeoutMs?: number /** Maximum number of tool calls admitted by the runtime. No default: absent means unlimited. */ readonly maxToolCalls?: number - /** Maximum UTF-8 bytes of model-facing output. No default: absent means no truncation. */ + /** + * Maximum UTF-8 bytes retained from the result value and logs; warnings have a separate + * budget of the same size. Fixed truncation notices and host formatting are additional. + */ readonly maxOutputBytes?: number } @@ -75,8 +81,9 @@ export const DiagnosticKind = Schema.Literals([ "TimeoutExceeded", "ToolFailure", "ExecutionFailure", + "Truncated", ]) -/** Stable categories produced by program, schema, tool, and limit failures. */ +/** Stable categories produced by program, schema, tool, limit, and truncation diagnostics. */ export type DiagnosticKind = typeof DiagnosticKind.Type export const Diagnostic = Schema.Struct({ @@ -92,6 +99,8 @@ const ToolCallSchema = Schema.Struct({ name: Schema.String }) export const Success = Schema.Struct({ ok: Schema.Literal(true), value: Schema.Json, + // Runtime-authored non-fatal diagnostics; program console output stays in `logs`. + warnings: Schema.optionalKey(Schema.Array(Diagnostic)), logs: Schema.optionalKey(Schema.Array(Schema.String)), truncated: Schema.optionalKey(Schema.Boolean), toolCalls: Schema.Array(ToolCallSchema), diff --git a/packages/codemode/src/interpreter/runtime.ts b/packages/codemode/src/interpreter/runtime.ts index 7f5d015d18d..ba095e74d52 100644 --- a/packages/codemode/src/interpreter/runtime.ts +++ b/packages/codemode/src/interpreter/runtime.ts @@ -1,5 +1,5 @@ import { parse } from "acorn" -import { Cause, Effect, Exit, Fiber, Semaphore } from "effect" +import { Cause, Effect, Exit, Fiber, Scope, Semaphore } from "effect" import { DiagnosticCategory, ModuleKind, ScriptTarget, flattenDiagnosticMessageText, transpileModule } from "typescript" import { copyIn, @@ -219,7 +219,7 @@ const normalizeError = (error: unknown): Diagnostic => { } } -// Shared by catch bindings, Promise.allSettled rejection reasons, and Promise.race losers. +// Shared by catch bindings and Promise.allSettled rejection reasons. const caughtErrorValue = (thrown: unknown): unknown => { if (thrown instanceof ProgramThrow) return thrown.value if (thrown instanceof InterpreterRuntimeError) return createErrorValue(thrown.errorName, thrown.message) @@ -611,6 +611,80 @@ const collectPatternNames = (pattern: AstNode, out: Array = []): Array { + private readonly active = new Set() + private readonly ids = new WeakMap() + private readonly observed = new WeakSet() + private readonly failures = new Map() + private nextID = 0 + + constructor(private readonly scope: Scope.Scope) {} + + create(effect: Effect.Effect): Effect.Effect { + return Effect.suspend(() => { + // Allocated at execution time (not construction) so re-run effects cannot share an id, + // and before the fork so diagnostics order by creation: a forked body that immediately + // creates promises of its own must sequence after its creator. + const id = this.nextID++ + return Effect.map(Effect.forkIn(effect, this.scope, { startImmediately: true }), (fiber) => { + const promise = new SandboxPromise(fiber) + this.active.add(promise) + this.ids.set(promise, id) + fiber.addObserver((exit) => { + this.active.delete(promise) + if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause) || this.observed.has(promise)) { + this.ids.delete(promise) + return + } + const failure = normalizeError(Cause.squash(exit.cause)) + this.failures.set(id, { + ...failure, + message: `Unhandled rejection from an un-awaited promise: ${failure.message}`, + }) + }) + return promise + }) + }) + } + + // Synchronous on purpose: JS makes a promise "handled" the moment a construct takes + // responsibility for it (await, or membership in a combinator call), not when the + // consuming fiber later runs. Call sites must invoke this at that moment. + markObserved(promise: SandboxPromise): void { + this.observed.add(promise) + const id = this.ids.get(promise) + this.ids.delete(promise) + if (id !== undefined) this.failures.delete(id) + } + + // Pure settlement subscription: never re-runs work and never affects rejection reporting. + await(promise: SandboxPromise): Effect.Effect> { + return Fiber.await(promise.fiber) + } + + // Unobserved rejections that already settled, in creation order. + diagnostics(): Array { + return [...this.failures].sort(([left], [right]) => left - right).map(([, failure]) => failure) + } + + // Normal-completion lifecycle: interrupts everything still running and reports the + // rejections that already settled un-awaited. interruptAll signals every fiber + // synchronously before awaiting termination, so no straggler can spawn new work between + // interrupts; the loop re-checks as a backstop because a straggler can create promises + // before its interrupt lands. + interrupt(): Effect.Effect> { + const self = this + return Effect.gen(function* () { + while (self.active.size > 0) { + yield* Fiber.interruptAll([...self.active].map((promise) => promise.fiber)) + } + return self.diagnostics() + }) + } +} + class Interpreter { private scopes: Array> private readonly invokeTool: (path: ReadonlyArray, args: Array) => Effect.Effect @@ -620,24 +694,22 @@ class Interpreter { private readonly logs: Array // Caps how many eagerly forked tool calls run at once (the parallel-call concurrency cap). private readonly callPermits: Semaphore.Semaphore - // Fiber-backed promises whose settlement no program construct has observed yet. Successful - // program completion drains these (like a runtime waiting on in-flight work at exit) and - // surfaces a never-awaited failure as an unhandled-rejection diagnostic. - private readonly pendingSettlements: Set + private readonly promises: PromiseRuntime constructor( invokeTool: (path: ReadonlyArray, args: Array) => Effect.Effect, toolKeys: (path: ReadonlyArray) => ReadonlyArray, + promises: PromiseRuntime, logs: Array = [], - shared?: { callPermits: Semaphore.Semaphore; pendingSettlements: Set }, + callPermits: Semaphore.Semaphore = Semaphore.makeUnsafe(TOOL_CALL_CONCURRENCY), ) { const globalScope = new Map() this.scopes = [globalScope] this.invokeTool = invokeTool this.toolKeys = toolKeys this.logs = logs - this.callPermits = shared?.callPermits ?? Semaphore.makeUnsafe(TOOL_CALL_CONCURRENCY) - this.pendingSettlements = shared?.pendingSettlements ?? new Set() + this.callPermits = callPermits + this.promises = promises globalScope.set("tools", { mutable: false, value: new ToolReference([]) }) globalScope.set("Promise", { mutable: false, value: new PromiseNamespace() }) globalScope.set("undefined", { mutable: false, value: undefined }) @@ -703,36 +775,12 @@ class Interpreter { // resolves before crossing the data boundary - `return tools.ns.tool(...)` works // without an explicit await, exactly as in JS. if (value instanceof SandboxPromise) value = yield* self.settlePromise(value) - yield* self.drainPendingSettlements() return value }).pipe(Effect.ensuring(Effect.sync(() => self.popScope()))) } - // Awaits every fiber-backed promise the program abandoned (fire-and-forget tool calls), so - // their work completes before the execution ends - mirroring a JS runtime waiting on - // in-flight I/O at exit. A failure nobody could have handled becomes an unhandled-rejection - // diagnostic (interrupted calls, e.g. Promise.race losers, are ignored). - private drainPendingSettlements(): Effect.Effect { - const self = this - return Effect.gen(function* () { - while (self.pendingSettlements.size > 0) { - const promise = self.pendingSettlements.values().next().value - if (promise === undefined) break - const exit = yield* self.observePromise(promise) - if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) continue - const failure = normalizeError(Cause.squash(exit.cause)) - throw new InterpreterRuntimeError( - `Unhandled rejection from an un-awaited promise: ${failure.message}`, - undefined, - failure.kind, - ["Await promises so failures can be caught and handled."], - ) - } - }) - } - - // Eagerly starts a tool call on a supervised child fiber (so the execution timeout and - // scope teardown interrupt it) gated by the concurrency semaphore, and wraps the fiber in a + // Eagerly starts a tool call in the execution's promise scope (so timeout and teardown + // interrupt it) gated by the concurrency semaphore, and wraps the fiber in a // first-class promise value. `startImmediately` makes the runtime admit the call - charging // the tool-call budget and firing onToolCallStart - at the call site, before any await. private createToolCallPromise( @@ -743,46 +791,20 @@ class Interpreter { } private createPromise(effect: Effect.Effect): Effect.Effect { - return Effect.map(Effect.forkChild(effect, { startImmediately: true }), (fiber) => { - const promise = new SandboxPromise(fiber) - this.pendingSettlements.add(promise) - return promise - }) - } - - // The promise's settlement as an Exit, marking it observed for unhandled-rejection tracking. - // Fiber settlement is idempotent, so observing the same promise repeatedly (await twice, - // Promise.all([p, p])) never re-runs the underlying call. - private observePromise(promise: SandboxPromise): Effect.Effect> { - this.pendingSettlements.delete(promise) - return promise.fiber !== undefined ? Fiber.await(promise.fiber) : Effect.exit(promise.immediate ?? Effect.void) + return this.promises.create(effect) } // `await promise`: succeed with the fulfilled value or re-raise the failure so try/catch - // observes it exactly like a synchronous throw at the await site. - private settlePromise(promise: SandboxPromise, node?: AstNode): Effect.Effect { - const self = this - return Effect.flatMap(this.observePromise(promise), (exit) => self.unwrapPromiseExit(promise, exit, node)) - } - - private unwrapPromiseExit( - promise: SandboxPromise | undefined, - exit: Exit.Exit, - node?: AstNode, - ): Effect.Effect { - if (Exit.isSuccess(exit)) return Effect.succeed(exit.value) - // A call Promise.race interrupted after losing settles as a catchable program failure; - // any other interruption is execution teardown (timeout/host) and must keep propagating - // as interruption rather than becoming program-visible data. - if (promise?.interrupted === true && Cause.hasInterruptsOnly(exit.cause)) { - return Effect.fail( - new InterpreterRuntimeError( - "This tool call was interrupted because another value settled a Promise.race first.", - node, - ), + // observes it exactly like a synchronous throw at the await site. Settlement is idempotent + // (fiber exits replay), so awaiting the same promise repeatedly never re-runs the call. + private settlePromise(promise: SandboxPromise): Effect.Effect { + const promises = this.promises + return Effect.suspend(() => { + promises.markObserved(promise) + return Effect.flatMap(promises.await(promise), (exit) => + Exit.isSuccess(exit) ? Effect.succeed(exit.value) : Effect.failCause(exit.cause), ) - } - return Effect.failCause(exit.cause) + }) } private evaluateStatement(node: AstNode): Effect.Effect { @@ -1532,7 +1554,7 @@ class Interpreter { // matching real JS semantics for non-thenables. const self = this return Effect.flatMap(this.evaluateExpression(getNode(node, "argument")), (value) => - value instanceof SandboxPromise ? self.settlePromise(value, node) : Effect.succeed(value), + value instanceof SandboxPromise ? self.settlePromise(value) : Effect.succeed(value), ) } case "NewExpression": @@ -2199,145 +2221,116 @@ class Interpreter { // Promise.* over ordinary runtime values. Combinators accept ANY array (or spreadable // collection) mixing promise values and plain data - built inline, beforehand, via spread, - // whatever - because tool calls already run eagerly on their own fibers; the combinators - // only observe settlements. Joining is therefore sequential (no extra fibers) without - // costing parallelism, and the concurrency cap stays where the work is: the fork semaphore. + // whatever - because tool calls already run eagerly on their own fibers. Each combinator + // returns a real promise whose join runs on its own scope-owned fiber, observing member + // settlements; the concurrency cap stays where the work is: the fork semaphore. private invokePromiseMethod( ref: PromiseMethodReference, args: Array, node: AstNode, ): Effect.Effect { - const self = this if (ref.name === "resolve") { // Promise.resolve of a promise is that promise (JS flattens); anything else is a - // promise already fulfilled with the value. + // promise already fulfilled with the value. Pre-settled values still fork a scope-owned + // fiber so every promise shares one lifecycle (an abandoned reject is reported, teardown + // is uniform). const value = args[0] - return Effect.succeed( - value instanceof SandboxPromise ? value : new SandboxPromise(undefined, Effect.succeed(value)), - ) + return value instanceof SandboxPromise ? Effect.succeed(value) : this.createPromise(Effect.succeed(value)) } if (ref.name === "reject") { - return Effect.sync(() => new SandboxPromise(undefined, Effect.fail(new ProgramThrow(args[0])))) + return this.createPromise(Effect.fail(new ProgramThrow(args[0]))) } const items = Array.isArray(args[0]) ? args[0] : spreadItems(args[0]) if (items === undefined) { - throw new InterpreterRuntimeError( - `Promise.${ref.name} expects an array of promises or plain values (e.g. Promise.${ref.name}(items.map((item) => tools.ns.tool(item)))).`, - node, + return this.createPromise( + Effect.fail( + new InterpreterRuntimeError( + `Promise.${ref.name} expects an array of promises or plain values (e.g. Promise.${ref.name}(items.map((item) => tools.ns.tool(item)))).`, + node, + ), + ), ) } + // JS makes combinator members "handled" synchronously at the call - their rejections + // belong to the aggregate from this moment, even ones settling before it runs. + for (const item of items) { + if (item instanceof SandboxPromise) this.promises.markObserved(item) + } + switch (ref.name) { case "all": { - // Mark every promise element observed up-front (Promise.all handles all of its - // members' failures, as in JS), race their settlements for fail-fast rejection, and - // preserve input order when they all fulfill. Rejected calls keep draining siblings. - const observations = items.map((item, index) => + // Each observation re-raises its member's failure, so Effect.all rejects on the first + // failure without waiting for the rest and preserves input order when all fulfill. + // Its failure-time interruption only unsubscribes the sibling waiters: the underlying + // fibers stay execution-owned and keep running, as in JS. + const observations = items.map((item) => item instanceof SandboxPromise - ? Effect.map(this.observePromise(item), (exit) => ({ index, item, exit })) - : Effect.succeed({ index, item: undefined, exit: Exit.succeed(item) }), + ? Effect.flatMap(this.promises.await(item), (exit) => + Exit.isSuccess(exit) ? Effect.succeed(exit.value) : Effect.failCause(exit.cause), + ) + : Effect.succeed(item), ) - return Effect.gen(function* () { - const remaining = [...observations] - const values: Array = [] - values.length = items.length - while (remaining.length > 0) { - const winner = yield* Effect.raceAll(remaining) - const position = remaining.indexOf(observations[winner.index]) - if (position >= 0) remaining.splice(position, 1) - if (Exit.isSuccess(winner.exit)) { - values[winner.index] = winner.exit.value - continue - } - yield* self.createPromise( - Effect.asVoid( - Effect.forEach( - items, - (item) => (item instanceof SandboxPromise ? self.observePromise(item) : Effect.void), - { concurrency: "unbounded" }, - ), - ), - ) - return yield* self.unwrapPromiseExit(winner.item, winner.exit, node) - } - return values - }) + return this.createPromise(Effect.all(observations, { concurrency: "unbounded" })) } case "allSettled": { const observations = items.map((item) => - item instanceof SandboxPromise - ? Effect.map(this.observePromise(item), (exit) => ({ promise: item as SandboxPromise | undefined, exit })) - : Effect.succeed({ promise: undefined as SandboxPromise | undefined, exit: Exit.succeed(item as unknown) }), + item instanceof SandboxPromise ? this.promises.await(item) : Effect.succeed(Exit.succeed(item as unknown)), ) - return Effect.gen(function* () { - const outcomes: Array = [] - for (const observation of observations) { - const { exit, promise } = yield* observation - if (Exit.isSuccess(exit)) { - outcomes.push( - Object.assign(Object.create(null) as SafeObject, { status: "fulfilled", value: exit.value }), - ) - continue - } - const raceInterrupted = promise?.interrupted === true && Cause.hasInterruptsOnly(exit.cause) - if (Cause.hasInterruptsOnly(exit.cause) && !raceInterrupted) { - // Execution teardown (timeout/host interruption), not a program-level rejection. - return yield* Effect.failCause(exit.cause) - } - const thrown = raceInterrupted - ? new InterpreterRuntimeError( - "This tool call was interrupted because another value settled a Promise.race first.", - node, + return this.createPromise( + Effect.gen(function* () { + const outcomes: Array = [] + for (const observation of observations) { + const exit = yield* observation + if (Exit.isSuccess(exit)) { + outcomes.push( + Object.assign(Object.create(null) as SafeObject, { status: "fulfilled", value: exit.value }), ) - : Cause.squash(exit.cause) - outcomes.push( - Object.assign(Object.create(null) as SafeObject, { - status: "rejected", - reason: caughtErrorValue(thrown), - }), - ) - } - return outcomes - }) + continue + } + if (Cause.hasInterruptsOnly(exit.cause)) { + // Execution teardown (timeout/host interruption), not a program-level rejection. + return yield* Effect.failCause(exit.cause) + } + outcomes.push( + Object.assign(Object.create(null) as SafeObject, { + status: "rejected", + reason: caughtErrorValue(Cause.squash(exit.cause)), + }), + ) + } + return outcomes + }), + ) } case "race": { if (items.length === 0) { - throw new InterpreterRuntimeError( - "Promise.race([]) would never settle; provide at least one promise or value.", - node, + return this.createPromise( + Effect.fail( + new InterpreterRuntimeError( + "Promise.race([]) would never settle; provide at least one promise or value.", + node, + ), + ), ) } - const observations = items.map((item, index) => - item instanceof SandboxPromise - ? Effect.map(this.observePromise(item), (exit) => ({ index, exit })) - : Effect.succeed({ index, exit: Exit.succeed(item as unknown) }), + const observations = items.map((item) => + item instanceof SandboxPromise ? this.promises.await(item) : Effect.succeed(Exit.succeed(item as unknown)), + ) + // First settlement (fulfilled OR rejected) wins; losing work stays execution-owned + // and is interrupted at normal completion (already observed) or by teardown. + return this.createPromise( + Effect.flatMap(Effect.raceAll(observations), (exit) => + Exit.isSuccess(exit) ? Effect.succeed(exit.value) : Effect.failCause(exit.cause), + ), ) - return Effect.gen(function* () { - // First settlement (fulfilled OR rejected) wins; the observations never fail, so - // racing them yields exactly that. Losing in-flight calls are then interrupted. - const winner = yield* Effect.raceAll(observations) - for (const [index, item] of items.entries()) { - if (index === winner.index || !(item instanceof SandboxPromise) || item.fiber === undefined) continue - item.interrupted = true - yield* Fiber.interrupt(item.fiber) - } - const winningItem = items[winner.index] - return yield* self.unwrapPromiseExit( - winningItem instanceof SandboxPromise ? winningItem : undefined, - winner.exit, - node, - ) - }) } } } private invokeFunction(fn: CodeModeFunction, args: Array): Effect.Effect { - const invocation = new Interpreter(this.invokeTool, this.toolKeys, this.logs, { - callPermits: this.callPermits, - pendingSettlements: this.pendingSettlements, - }) + const invocation = new Interpreter(this.invokeTool, this.toolKeys, this.promises, this.logs, this.callPermits) invocation.scopes = [...fn.capturedScopes, new Map()] const run = Effect.gen(function* () { // Seed every parameter name into the scope as a TDZ slot first, so a default that @@ -3484,68 +3477,109 @@ export const executeWithLimits = >( limits: ResolvedExecutionLimits, searchIndex: ToolRuntime.DiscoveryPlan["searchIndex"], ): Effect.Effect> => { - const hooks = { - ...(options.onToolCallStart === undefined ? {} : { onToolCallStart: options.onToolCallStart }), - ...(options.onToolCallEnd === undefined ? {} : { onToolCallEnd: options.onToolCallEnd }), - } - const tools = ToolRuntime.make( - (options.tools ?? {}) as HostTools>, - limits.maxToolCalls, - searchIndex, - hooks, - ) - const logs: Array = [] - const logged = () => (logs.length > 0 ? { logs: [...logs] } : {}) - if (options.code.trim().length === 0) { return Effect.succeed({ ok: false, error: { kind: "ParseError", message: "Code cannot be empty." }, - toolCalls: tools.calls, + toolCalls: [], }) } - const operation = Effect.gen(function* () { - const program = parseProgram(options.code) - const interpreter = new Interpreter>(tools.invoke, tools.keys, logs) - const value = yield* interpreter.run(program) - const result = copyOut(copyIn(value, "Execution result"), true) as DataValue - return { - ok: true, - value: result, - ...logged(), - toolCalls: tools.calls, - } satisfies Result - }).pipe((program) => { - const timeoutMs = limits.timeoutMs - if (timeoutMs === undefined) return program - return program.pipe( - Effect.timeoutOrElse({ - duration: timeoutMs, - orElse: () => - Effect.succeed({ - ok: false, - error: { kind: "TimeoutExceeded", message: `Execution timed out after ${timeoutMs}ms.` }, + // Suspended so all per-execution state - tool-call admission budget and audit list, logs, + // and the timeout path's completed value - binds at run time: a reused Effect must start + // from a clean slate instead of observing a previous run's state. + return Effect.suspend(() => { + const hooks = { + ...(options.onToolCallStart === undefined ? {} : { onToolCallStart: options.onToolCallStart }), + ...(options.onToolCallEnd === undefined ? {} : { onToolCallEnd: options.onToolCallEnd }), + } + const tools = ToolRuntime.make( + (options.tools ?? {}) as HostTools>, + limits.maxToolCalls, + searchIndex, + hooks, + ) + const logs: Array = [] + const logged = () => (logs.length > 0 ? { logs: [...logs] } : {}) + // Set once the program body returned and its value crossed the data boundary, so a timeout + // firing during leftover interruption reports "completed with interrupted background work" + // instead of discarding the computed value as a plain timeout. + let returned: { value: DataValue; promises: PromiseRuntime> } | undefined + + const base = Effect.acquireUseRelease( + Scope.make("parallel"), + (scope) => + Effect.gen(function* () { + const program = parseProgram(options.code) + const promises = new PromiseRuntime>(scope) + const interpreter = new Interpreter>(tools.invoke, tools.keys, promises, logs) + const value = yield* interpreter.run(program) + // Validate the result first so an invalid value is a fatal completion that closes + // the promise scope directly instead of taking the normal-completion path. + const result = copyOut(copyIn(value, "Execution result"), true) as DataValue + returned = { value: result, promises } + const warnings = yield* promises.interrupt() + return { + ok: true, + value: result, + ...(warnings.length > 0 ? { warnings } : {}), ...logged(), toolCalls: tools.calls, - } satisfies Result), - }), + } satisfies Result + }), + (scope, exit) => Scope.close(scope, exit), + ) + const timeoutMs = limits.timeoutMs + const operation = + timeoutMs === undefined + ? base + : base.pipe( + Effect.timeoutOrElse({ + duration: timeoutMs, + orElse: () => + Effect.sync(() => { + if (returned === undefined) { + return { + ok: false, + error: { kind: "TimeoutExceeded", message: `Execution timed out after ${timeoutMs}ms.` }, + ...logged(), + toolCalls: tools.calls, + } satisfies Result + } + // The timeout warning leads so byte-budget truncation cuts it last. + return { + ok: true, + value: returned.value, + warnings: [ + { + kind: "TimeoutExceeded", + message: `The program returned, but background work was still running at the ${timeoutMs}ms timeout and was interrupted. Await all started promises.`, + }, + ...returned.promises.diagnostics(), + ], + ...logged(), + toolCalls: tools.calls, + } satisfies Result + }), + }), + ) + + return operation.pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.succeed({ + ok: false, + error: normalizeError(Cause.squash(cause)), + ...logged(), + toolCalls: tools.calls, + } satisfies Result), + ), + Effect.map((result) => + limits.maxOutputBytes === undefined ? result : boundOutput(result, limits.maxOutputBytes), + ), ) }) - - return operation.pipe( - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) - ? Effect.interrupt - : Effect.succeed({ - ok: false, - error: normalizeError(Cause.squash(cause)), - ...logged(), - toolCalls: tools.calls, - } satisfies Result), - ), - Effect.map((result) => (limits.maxOutputBytes === undefined ? result : boundOutput(result, limits.maxOutputBytes))), - ) } const utf8ByteLength = (value: string): number => new TextEncoder().encode(value).byteLength @@ -3560,9 +3594,10 @@ const utf8Truncate = (value: string, maxBytes: number): string => { } /** - * Bounds the model-facing output (serialized result value plus logs) to `maxOutputBytes`. - * Oversized values are replaced by their truncated serialized text with an explanatory marker, - * and logs are kept from the start until the remaining budget is exhausted. Truncation never + * Bounds retained program payload bytes (serialized result value and logs) to `maxOutputBytes`. + * Warning diagnostics are bounded by a separate budget of the same size so a large value can + * never starve runtime-authored diagnostics. Fixed truncation notices are added outside those + * budgets, as is any framing added when a host renders the structured result. Truncation never * fails the execution; `truncated: true` marks affected results. Only runs when the host set * `maxOutputBytes` - with the limit absent, output passes through unbounded. */ @@ -3584,6 +3619,23 @@ const boundOutput = (result: Result, maxOutputBytes: number): Result => { } } + const warnings = result.ok ? (result.warnings ?? []) : [] + const keptWarnings: Array = [] + let warningBytes = 0 + for (const warning of warnings) { + const bytes = utf8ByteLength(JSON.stringify(warning)) + 1 + if (warningBytes + bytes > maxOutputBytes) break + warningBytes += bytes + keptWarnings.push(warning) + } + if (keptWarnings.length < warnings.length) { + truncated = true + keptWarnings.push({ + kind: "Truncated", + message: `${warnings.length - keptWarnings.length} additional warnings omitted by the output limit.`, + }) + } + const logs = result.logs ?? [] const kept: Array = [] const logBudget = Math.max(0, maxOutputBytes - valueBytes) @@ -3600,8 +3652,16 @@ const boundOutput = (result: Result, maxOutputBytes: number): Result => { } if (!truncated) return result + const warningsPart = keptWarnings.length > 0 ? { warnings: keptWarnings } : {} const logsPart = kept.length > 0 ? { logs: kept } : {} return result.ok - ? { ok: true, value, ...logsPart, truncated: true, toolCalls: result.toolCalls } + ? { + ok: true, + value, + ...warningsPart, + ...logsPart, + truncated: true, + toolCalls: result.toolCalls, + } : { ok: false, error: result.error, ...logsPart, truncated: true, toolCalls: result.toolCalls } } diff --git a/packages/codemode/src/tool-runtime.ts b/packages/codemode/src/tool-runtime.ts index 80ce828414d..3cdf49cfe53 100644 --- a/packages/codemode/src/tool-runtime.ts +++ b/packages/codemode/src/tool-runtime.ts @@ -602,6 +602,7 @@ export const prepare = (tools: HostTools, catalogBudget = defaultCatalogBu "- Filter, aggregate, and transform collections in code - never return them raw or call a tool per item across messages.", "- A result typed `Promise` may be structured data or text. Before reading fields, check that it is a non-null object and not an array; otherwise handle the returned text or primitive directly.", '- Run independent calls in parallel: `await Promise.all(items.map((item) => tools..(item)))`, or use `tools.["tool-name"](item)` when the listed signature uses bracket notation.', + "- Execution ends when the program returns; pending promises are interrupted, so await every call whose completion matters.", "- `Object.keys(tools)` lists namespaces; `Object.keys(tools.)` lists its tools; `for...in` works on both.", ...(complete ? [] diff --git a/packages/codemode/src/values.ts b/packages/codemode/src/values.ts index 4ca305d815e..3ba2421d5d9 100644 --- a/packages/codemode/src/values.ts +++ b/packages/codemode/src/values.ts @@ -1,11 +1,7 @@ -import type { Effect, Fiber } from "effect" +import type { Fiber } from "effect" export class SandboxPromise { - interrupted = false - constructor( - readonly fiber: Fiber.Fiber | undefined, - readonly immediate?: Effect.Effect, - ) {} + constructor(readonly fiber: Fiber.Fiber) {} } export class SandboxDate { diff --git a/packages/codemode/test/codemode.test.ts b/packages/codemode/test/codemode.test.ts index 16c17ca0c2a..2e09e9d3c9e 100644 --- a/packages/codemode/test/codemode.test.ts +++ b/packages/codemode/test/codemode.test.ts @@ -506,6 +506,26 @@ describe("CodeMode public contract", () => { expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(reusable)))).toStrictEqual(reusable) }) + test("a reused execution Effect starts from a clean slate", async () => { + const echo = Tool.make({ + description: "echo", + input: Schema.Struct({}), + output: Schema.Number, + run: () => Effect.succeed(1), + }) + const effect = CodeMode.execute({ + tools: { host: { echo } }, + code: `console.log("hi"); return await tools.host.echo({})`, + limits: { maxToolCalls: 1 }, + }) + const first = await Effect.runPromise(effect) + const second = await Effect.runPromise(effect) + // Per-execution state (tool-call budget and audit list, logs, timeout bookkeeping) must + // bind at run time, so the second run neither exhausts the budget nor leaks run 1's logs. + expect(first).toStrictEqual(second) + expect(second).toStrictEqual({ ok: true, value: 1, logs: ["hi"], toolCalls: [{ name: "host.echo" }] }) + }) + test("inlines a COMPLETE small catalog and keeps search registered but unadvertised", async () => { const runtime = CodeMode.make({ tools }) expect(runtime.catalog()).toStrictEqual([ diff --git a/packages/codemode/test/promise-test262.test.ts b/packages/codemode/test/promise-test262.test.ts new file mode 100644 index 00000000000..500b7ae45df --- /dev/null +++ b/packages/codemode/test/promise-test262.test.ts @@ -0,0 +1,906 @@ +/* + * Portions adapted from Test262 at revision 250f204f23a9249ff204be2baec29600faae7b75. + * Every test names its upstream source; test.failing cases are executable conformance + * targets for intended Promise behavior that CodeMode does not implement yet. + * + * Copyright 2014 Cubane Canada, Inc. All rights reserved. + * Copyright 2015 Microsoft Corporation. All rights reserved. + * Copyright 2016 Microsoft, Inc. All rights reserved. + * Copyright 2017 Caitlin Potter. All rights reserved. + * Copyright (C) 2016-2020 the V8 project authors. All rights reserved. + * Copyright (C) 2018-2020 Rick Waldron. All rights reserved. + * Copyright (C) 2019 Leo Balter. All rights reserved. + * Test262 portions are governed by the BSD license in LICENSE.test262. + */ +import { describe, expect, test } from "bun:test" +import { Effect } from "effect" +import { CodeMode } from "../src/index.js" + +const execute = (code: string) => + Effect.runPromise(CodeMode.execute({ code, tools: {}, limits: { timeoutMs: 1_000 } })) + +const value = async (code: string) => { + const result = await execute(code) + if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`) + return result.value +} + +describe("Test262 Promise statics", () => { + test("statics are callable and return promises", async () => { + // Sources: + // test/built-ins/Promise/all/S25.4.4.1_A1.1_T1.js + // test/built-ins/Promise/allSettled/is-function.js + // test/built-ins/Promise/allSettled/returns-promise.js + // test/built-ins/Promise/race/S25.4.4.3_A1.1_T1.js + // test/built-ins/Promise/resolve/S25.4.4.5_A1.1_T1.js + // test/built-ins/Promise/reject/S25.4.4.4_A1.1_T1.js + expect( + await value(` + const values = [ + Promise.all([]), + Promise.allSettled([]), + Promise.race([undefined]), + Promise.resolve(), + Promise.reject(), + ] + const callable = [ + typeof Promise.all, + typeof Promise.allSettled, + typeof Promise.race, + typeof Promise.resolve, + typeof Promise.reject, + ] + try { await values[4] } catch {} + return [callable, values.map((item) => item instanceof Promise)] + `), + ).toEqual([ + ["function", "function", "function", "function", "function"], + [true, true, true, true, true], + ]) + }) + + test("Promise.all returns fresh arrays for empty and settled inputs", async () => { + // Sources: + // test/built-ins/Promise/all/S25.4.4.1_A2.1_T1.js + // test/built-ins/Promise/all/S25.4.4.1_A2.3_T1.js + // test/built-ins/Promise/all/S25.4.4.1_A2.3_T2.js + // test/built-ins/Promise/all/S25.4.4.1_A2.3_T3.js + // test/built-ins/Promise/all/S25.4.4.1_A7.1_T1.js + expect( + await value(` + const input = [] + const emptyPromise = Promise.all(input) + const empty = await emptyPromise + const onePromise = Promise.all([Promise.resolve(3)]) + const one = await onePromise + return [ + emptyPromise instanceof Promise, + empty instanceof Array, + empty.length, + empty !== input, + onePromise instanceof Promise, + one instanceof Array, + one.length, + one[0], + ] + `), + ).toEqual([true, true, 0, true, true, true, 1, 3]) + }) + + test("Promise.all adopts values and preserves input order and identity", async () => { + // Sources: + // test/built-ins/Promise/all/resolve-non-thenable.js + // test/built-ins/Promise/all/S25.4.4.1_A8.2_T1.js + // test/built-ins/Promise/all/S25.4.4.1_A8.2_T2.js + const result = await value(` + const first = { id: 1 } + const second = { id: 2 } + const values = await Promise.all([Promise.resolve(3), first, Promise.resolve(second)]) + const observe = async (promise) => { + try { await promise; return "fulfilled" } catch (reason) { return reason } + } + return [ + values.length, + values[0], + values[1] === first, + values[2] === second, + await observe(Promise.all([Promise.reject(1), Promise.resolve(2)])), + await observe(Promise.all([Promise.resolve(1), Promise.reject(2)])), + ] + `) + expect(result).toEqual([3, 3, true, true, 1, 2]) + }) + + test("Promise.allSettled returns fresh arrays and ordered outcome records", async () => { + // Sources: + // test/built-ins/Promise/allSettled/resolves-empty-array.js + // test/built-ins/Promise/allSettled/resolves-to-array.js + // test/built-ins/Promise/allSettled/resolved-all-fulfilled.js + // test/built-ins/Promise/allSettled/resolved-all-rejected.js + // test/built-ins/Promise/allSettled/resolved-all-mixed.js + // test/built-ins/Promise/allSettled/resolve-non-thenable.js + expect( + await value(` + const input = [] + const empty = await Promise.allSettled(input) + const reason = { id: 4 } + const object = { id: 5 } + const outcomes = await Promise.allSettled([ + Promise.resolve(1), + Promise.reject(2), + 3, + Promise.reject(reason), + object, + ]) + return [ + empty instanceof Array, + empty.length, + empty !== input, + outcomes, + outcomes[4].value === object, + outcomes.map((item) => Object.keys(item)), + ] + `), + ).toEqual([ + true, + 0, + true, + [ + { status: "fulfilled", value: 1 }, + { status: "rejected", reason: 2 }, + { status: "fulfilled", value: 3 }, + { status: "rejected", reason: { id: 4 } }, + { status: "fulfilled", value: { id: 5 } }, + ], + true, + [ + ["status", "value"], + ["status", "reason"], + ["status", "value"], + ["status", "reason"], + ["status", "value"], + ], + ]) + }) + + test("Promise.race preserves fulfillment, rejection, and iterable order", async () => { + // Sources: + // test/built-ins/Promise/race/S25.4.4.3_A6.2_T1.js + // test/built-ins/Promise/race/S25.4.4.3_A7.1_T1.js + // test/built-ins/Promise/race/S25.4.4.3_A7.2_T1.js + // test/built-ins/Promise/race/S25.4.4.3_A7.3_T1.js + // test/built-ins/Promise/race/S25.4.4.3_A7.3_T2.js + expect( + await value(` + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + return await Promise.all([ + observe(Promise.race([23])), + observe(Promise.race([Promise.reject(7)])), + observe(Promise.race([Promise.resolve(1), Promise.resolve(2)])), + observe(Promise.race([Promise.reject(3), Promise.resolve(4)])), + ]) + `), + ).toEqual([ + ["fulfilled", 23], + ["rejected", 7], + ["fulfilled", 1], + ["rejected", 3], + ]) + }) + + test("combinators consume supported string iterables", async () => { + // Sources: + // test/built-ins/Promise/all/iter-arg-is-string-resolve.js + // test/built-ins/Promise/allSettled/iter-arg-is-string-resolve.js + // test/built-ins/Promise/race/iter-arg-is-string-resolve.js + expect( + await value(` + return [ + await Promise.all("abc"), + await Promise.allSettled("ab"), + await Promise.race("abc"), + ] + `), + ).toEqual([ + ["a", "b", "c"], + [ + { status: "fulfilled", value: "a" }, + { status: "fulfilled", value: "b" }, + ], + "a", + ]) + }) + + test("Promise.resolve adopts values and preserves sandbox-promise identity", async () => { + // Sources: + // test/built-ins/Promise/resolve/S25.4.4.5_A2.1_T1.js + // test/built-ins/Promise/resolve/resolve-non-obj.js + // test/built-ins/Promise/resolve/resolve-non-thenable.js + expect( + await value(` + const object = { id: 1 } + const promise = Promise.resolve(1) + return [ + await Promise.resolve(23), + await Promise.resolve(Promise.resolve(24)), + (await Promise.resolve(object)) === object, + [promise].includes(Promise.resolve(promise)), + ] + `), + ).toEqual([23, 24, true, true]) + }) + + test("Promise.reject preserves primitive and object reasons", async () => { + // Sources: + // test/built-ins/Promise/reject/S25.4.4.4_A2.1_T1.js + const result = await value(` + const object = { reason: true } + const reasons = [undefined, null, false, true, 0, "", 42, object] + const observe = async (reason) => { + try { await Promise.reject(reason); return false } catch (caught) { return caught === reason } + } + return await Promise.all(reasons.map(observe)) + `) + expect(result).toEqual([true, true, true, true, true, true, true, true]) + }) + + test("Promise.all resolves duplicate members into every slot", async () => { + // Sources: + // test/built-ins/Promise/all/invoke-resolve-on-promises-every-iteration-of-promise.js + // test/built-ins/Promise/all/invoke-resolve-on-values-every-iteration-of-promise.js + // (adapted: CodeMode has no observable Promise.resolve hook, so per-iteration + // handling of a repeated member is asserted through the resolved slots) + expect( + await value(` + const settled = Promise.resolve(3) + const computed = (async () => "computed")() + return [ + await Promise.all([settled, settled, settled]), + await Promise.all([computed, "plain", computed]), + ] + `), + ).toEqual([ + [3, 3, 3], + ["computed", "plain", "computed"], + ]) + }) + + test("Promise.allSettled records duplicate members independently", async () => { + // Source: test/built-ins/Promise/allSettled/invoke-resolve-on-promises-every-iteration-of-promise.js + // (adapted: per-iteration handling of a repeated member is asserted through the + // outcome records instead of a Promise.resolve hook) + expect( + await value(` + const good = Promise.resolve(1) + const bad = Promise.reject(2) + return await Promise.allSettled([good, bad, good, bad]) + `), + ).toEqual([ + { status: "fulfilled", value: 1 }, + { status: "rejected", reason: 2 }, + { status: "fulfilled", value: 1 }, + { status: "rejected", reason: 2 }, + ]) + }) + + test("combinators adopt members that settled before the call", async () => { + // Sources: + // test/built-ins/Promise/all/reject-immed.js + // test/built-ins/Promise/allSettled/reject-immed.js + // test/built-ins/Promise/race/reject-immed.js + // (adapted: immediately-rejecting thenables become sandbox promises that settled, + // and were even observed, before the combinator call) + expect( + await value(` + const fulfilled = Promise.resolve("done") + const rejected = Promise.reject("failed") + try { await rejected } catch {} + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + return [ + await observe(Promise.all([fulfilled, rejected])), + await Promise.allSettled([rejected, fulfilled]), + await observe(Promise.race([rejected, fulfilled])), + ] + `), + ).toEqual([ + ["rejected", "failed"], + [ + { status: "rejected", reason: "failed" }, + { status: "fulfilled", value: "done" }, + ], + ["rejected", "failed"], + ]) + }) + + test("combinator results follow input order, not settlement order", async () => { + // Sources: + // test/built-ins/Promise/all/resolve-non-thenable.js + // test/built-ins/Promise/allSettled/resolved-all-mixed.js + // (adapted: members are created, and therefore settle, in reverse of input order; + // deferred settlement is not expressible without host-async work in this corpus) + expect( + await value(` + const third = Promise.resolve("c") + const failing = (async () => { throw "b" })() + try { await failing } catch {} + const second = (async () => "b")() + const first = Promise.resolve("a") + return [ + await Promise.all([first, second, third]), + await Promise.allSettled([first, failing, third]), + ] + `), + ).toEqual([ + ["a", "b", "c"], + [ + { status: "fulfilled", value: "a" }, + { status: "rejected", reason: "b" }, + { status: "fulfilled", value: "c" }, + ], + ]) + }) + + test("Promise.race ignores a rejected loser once the first contender wins", async () => { + // Source: test/built-ins/Promise/race/reject-ignored-immed.js + // (adapted: the losing rejection comes from an async function instead of a thenable; + // the exact-equality check also asserts the loser leaves no unhandled-rejection warning) + expect( + await execute(` + const loser = (async () => { throw "lost" })() + return await Promise.race([Promise.resolve("won"), loser]) + `), + ).toEqual({ ok: true, value: "won", toolCalls: [] }) + }) + + test("Promise.race([]) returns a promise whose CodeMode failure is catchable", async () => { + // Sources: + // test/built-ins/Promise/race/S25.4.4.3_A2.1_T1.js + // test/built-ins/Promise/race/S25.4.4.3_A5.1_T1.js + // (adapted: upstream requires Promise.race([]) to never settle; CodeMode intentionally + // rejects with a catchable diagnostic instead of hanging, so this asserts the sandbox + // divergence rather than the spec never-settles behavior) + expect( + await value(` + const empty = Promise.race([]) + try { + await empty + return "settled" + } catch (error) { + return [empty instanceof Promise, error instanceof Error] + } + `), + ).toEqual([true, true]) + }) + + test("Promise.resolve passes the same sandbox promise through nested chains", async () => { + // Source: test/built-ins/Promise/resolve/S25.4.4.5_A2.2_T1.js + // (adapted: no executor construction, and identity is observed with Array includes + // because promises are not comparable data values in CodeMode) + expect( + await value(` + const promise = Promise.resolve({ id: 1 }) + return [ + [promise].includes(Promise.resolve(promise)), + [promise].includes(Promise.resolve(Promise.resolve(promise))), + (await Promise.resolve(Promise.resolve(promise))).id, + ] + `), + ).toEqual([true, true, 1]) + }) + + test("Promise.resolve of a rejected promise preserves identity and reason", async () => { + // Source: test/built-ins/Promise/resolve/S25.4.4.5_A2.3_T1.js + // (adapted: the source promise is already rejected instead of rejected later) + expect( + await value(` + const rejected = Promise.reject("oops") + const adopted = Promise.resolve(rejected) + const identity = [rejected].includes(adopted) + try { + await adopted + return "fulfilled" + } catch (reason) { + return [identity, reason] + } + `), + ).toEqual([true, "oops"]) + }) + + test("Promise.reject uses a promise reason without flattening it", async () => { + // Sources: + // test/built-ins/Promise/reject-via-fn-immed.js + // test/built-ins/Promise/reject-via-fn-deferred.js + // (adapted: the promise reason goes through Promise.reject instead of executor reject) + expect( + await value(` + const observe = async (reason) => { + try { + await Promise.reject(reason) + return "fulfilled" + } catch (caught) { + const identity = [reason].includes(caught) + try { return [identity, caught instanceof Promise, await caught] } + catch (inner) { return [identity, caught instanceof Promise, "rethrew " + inner] } + } + } + return [await observe(Promise.resolve(1)), await observe(Promise.reject("inner"))] + `), + ).toEqual([ + [true, true, 1], + [true, true, "rethrew inner"], + ]) + }) +}) + +describe("Test262 async functions and await", () => { + test("declaration, expression, and arrow forms return promises", async () => { + // Sources: + // test/language/statements/async-function/declaration-returns-promise.js + // test/language/expressions/async-function/expression-returns-promise.js + // test/language/expressions/async-arrow-function/arrow-returns-promise.js + expect( + await value(` + async function declaration() { return 1 } + const expression = async function() { return 2 } + const arrow = async () => 3 + const promises = [declaration(), expression(), arrow()] + return [promises.map((item) => item instanceof Promise), await Promise.all(promises)] + `), + ).toEqual([[true, true, true], [1, 2, 3]]) + }) + + test("async bodies adopt returns and reject throws before and after await", async () => { + // Sources: + // test/language/statements/async-function/evaluation-body.js + // test/language/statements/async-function/evaluation-body-that-returns.js + // test/language/statements/async-function/evaluation-body-that-returns-after-await.js + // test/language/statements/async-function/evaluation-body-that-throws.js + // test/language/statements/async-function/evaluation-body-that-throws-after-await.js + expect( + await value(` + const order = [] + const plain = async () => { order.push("body"); return 42 } + const afterAwait = async () => { await Promise.resolve(); return 43 } + const throwsBefore = async () => { throw 1 } + const throwsAfter = async () => { await Promise.resolve(); throw 2 } + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + const first = plain() + return [ + order, + await observe(first), + await observe(afterAwait()), + await observe(throwsBefore()), + await observe(throwsAfter()), + ] + `), + ).toEqual([ + ["body"], + ["fulfilled", 42], + ["fulfilled", 43], + ["rejected", 1], + ["rejected", 2], + ]) + }) + + test("default-parameter throws reject instead of escaping the call", async () => { + // Source: test/language/statements/async-function/evaluation-default-that-throws.js + expect( + await value(` + const fail = () => { throw new Error("default") } + const run = async (value = fail()) => value + let returned = false + try { + const promise = run() + returned = promise instanceof Promise + await promise + return [returned, "fulfilled"] + } catch (error) { + return [returned, error.message] + } + `), + ).toEqual([true, "default"]) + }) + + test("async try/finally completion records override earlier completion", async () => { + // Sources: the try-{return,throw,reject}-finally-{return,throw,reject}.js matrix under + // test/language/statements/async-function, test/language/expressions/async-function, + // and test/language/expressions/async-arrow-function. + expect( + await value(` + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + const returnReturn = async () => { try { return "early" } finally { return await Promise.resolve("override") } } + const returnThrow = async () => { try { return "early" } finally { throw "override" } } + const returnReject = async () => { try { return "early" } finally { await Promise.reject("override") } } + const throwReturn = async () => { try { throw "early" } finally { return await Promise.resolve("override") } } + const throwThrow = async () => { try { throw "early" } finally { throw "override" } } + const throwReject = async () => { try { throw "early" } finally { await Promise.reject("override") } } + const rejectReturn = async () => { try { await Promise.reject("early") } finally { return await Promise.resolve("override") } } + const rejectThrow = async () => { try { await Promise.reject("early") } finally { throw "override" } } + const rejectReject = async () => { try { await Promise.reject("early") } finally { await Promise.reject("override") } } + return await Promise.all([ + observe(returnReturn()), observe(returnThrow()), observe(returnReject()), + observe(throwReturn()), observe(throwThrow()), observe(throwReject()), + observe(rejectReturn()), observe(rejectThrow()), observe(rejectReject()), + ]) + `), + ).toEqual([ + ["fulfilled", "override"], + ["rejected", "override"], + ["rejected", "override"], + ["fulfilled", "override"], + ["rejected", "override"], + ["rejected", "override"], + ["fulfilled", "override"], + ["rejected", "override"], + ["rejected", "override"], + ]) + }) + + test("await preserves an object whose then property is not callable", async () => { + // Source: test/language/expressions/await/await-awaits-thenable-not-callable.js + expect( + await value(` + const thenable = { then: 42 } + return (await thenable) === thenable + `), + ).toBe(true) + }) + + test("await returns non-promise operands unchanged", async () => { + // Source: test/language/expressions/await/await-non-promise.js + // (adapted: only value pass-through is asserted here; the spec tick ordering around + // await of non-promises is covered by the failing interleaving test below) + expect( + await value(` + const object = { id: 1 } + const array = [1, 2] + return [ + await 1, + await "text", + await true, + (await null) === null, + (await undefined) === undefined, + (await object) === object, + (await array) === array, + ] + `), + ).toEqual([1, "text", true, true, true, true, true]) + }) +}) + +describe("Test262 expected Promise conformance", () => { + for (const name of ["all", "allSettled", "race"] as const) { + test.failing(`Promise.${name} rejects invalid input with TypeError`, async () => { + // Sources: + // test/built-ins/Promise/all/S25.4.4.1_A3.1_T1.js + // test/built-ins/Promise/all/S25.4.4.1_A3.1_T2.js + // test/built-ins/Promise/allSettled/iter-arg-is-number-reject.js + // test/built-ins/Promise/race/iter-arg-is-number-reject.js + expect( + await value(` + try { + const promise = Promise.${name}(42) + const returned = promise instanceof Promise + await promise + return [returned, "fulfilled"] + } catch (error) { + return [true, error.name] + } + `), + ).toEqual([true, "TypeError"]) + }) + } + + test.failing("Promise.all consumes sparse positions as undefined", async () => { + // Source: test/built-ins/Array/from/from-array.js (array iterator hole behavior) + expect( + await value(` + const input = [] + input[1] = 1 + const result = await Promise.all(input) + return [result.length, result[0] === undefined, result[1]] + `), + ).toEqual([2, true, 1]) + }) + + test.failing("Promise.allSettled consumes sparse positions as undefined", async () => { + // Source: test/built-ins/Array/from/from-array.js (array iterator hole behavior) + expect( + await value(` + const input = [] + input[1] = 1 + const result = await Promise.allSettled(input) + return [result.length, result[0].status, result[0].value === undefined, result[1]] + `), + ).toEqual([2, "fulfilled", true, { status: "fulfilled", value: 1 }]) + }) + + test.failing("Promise.race consumes a sparse first position as undefined", async () => { + // Source: test/built-ins/Array/from/from-array.js (array iterator hole behavior) + expect( + await value(` + const input = [] + input[1] = 1 + return (await Promise.race(input)) === undefined + `), + ).toBe(true) + }) + + test.failing("Promise.all settles after reactions attached to its inputs", async () => { + // Sources: + // test/built-ins/Promise/all/S25.4.4.1_A7.2_T1.js + // test/built-ins/Promise/all/S25.4.4.1_A8.1_T1.js + expect( + await value(` + const sequence = [1] + const input = Promise.resolve(1) + const aggregate = Promise.all([input]) + aggregate.then(() => sequence.push(4)) + input.then(() => sequence.push(3)).then(() => sequence.push(5)) + sequence.push(2) + await aggregate + await Promise.resolve() + return sequence + `), + ).toEqual([1, 2, 3, 4, 5]) + }) + + test.failing("Promise.allSettled settles after reactions attached to its inputs", async () => { + // Sources: + // test/built-ins/Promise/allSettled/resolved-sequence.js + // test/built-ins/Promise/allSettled/resolved-sequence-extra-ticks.js + // test/built-ins/Promise/allSettled/resolved-sequence-mixed.js + // test/built-ins/Promise/allSettled/resolved-sequence-with-rejections.js + expect( + await value(` + const sequence = [1] + const input = Promise.resolve(1) + const aggregate = Promise.allSettled([input]) + aggregate.then(() => sequence.push(4)) + input.then(() => sequence.push(3)).then(() => sequence.push(5)) + sequence.push(2) + await aggregate + await Promise.resolve() + return sequence + `), + ).toEqual([1, 2, 3, 4, 5]) + }) + + test.failing("Promise.race settles in a reaction after its winning input", async () => { + // Sources: + // test/built-ins/Promise/race/S25.4.4.3_A6.1_T1.js + // test/built-ins/Promise/race/resolved-sequence-extra-ticks.js + expect( + await value(` + const sequence = [1] + const race = Promise.race([1]) + race.then(() => sequence.push(4)) + Promise.resolve().then(() => sequence.push(3)).then(() => sequence.push(5)) + sequence.push(2) + await race + await Promise.resolve() + return sequence + `), + ).toEqual([1, 2, 3, 4, 5]) + }) + + test.failing("then reactions route and propagate fulfillment and rejection", async () => { + // Sources: + // test/built-ins/Promise/prototype/then/prfm-fulfilled.js + // test/built-ins/Promise/prototype/then/prfm-rejected.js + // test/built-ins/Promise/prototype/then/rxn-handler-identity.js + // test/built-ins/Promise/prototype/then/rxn-handler-thrower.js + // test/built-ins/Promise/prototype/then/rxn-handler-fulfilled-return-normal.js + // test/built-ins/Promise/prototype/then/rxn-handler-fulfilled-return-abrupt.js + // test/built-ins/Promise/prototype/then/rxn-handler-rejected-return-normal.js + // test/built-ins/Promise/prototype/then/rxn-handler-rejected-return-abrupt.js + expect( + await value(` + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + return await Promise.all([ + observe(Promise.resolve(1).then((value) => value + 1)), + observe(Promise.reject(2).then(undefined, (reason) => reason + 1)), + observe(Promise.resolve(3).then(undefined)), + observe(Promise.reject(4).then(undefined)), + observe(Promise.resolve(5).then(() => { throw 6 })), + observe(Promise.reject(7).then(undefined, () => { throw 8 })), + ]) + `), + ).toEqual([ + ["fulfilled", 2], + ["fulfilled", 3], + ["fulfilled", 3], + ["rejected", 4], + ["rejected", 6], + ["rejected", 8], + ]) + }) + + test.failing("then reactions preserve breadth-first queue order", async () => { + // Source: test/built-ins/Promise/prototype/then/S25.4.4_A1.1_T1.js + expect( + await value(` + const sequence = [1] + const promise = Promise.resolve() + const first = promise.then(() => sequence.push(3)).then(() => sequence.push(5)).then(() => sequence.push(7)) + const second = promise.then(() => sequence.push(4)).then(() => sequence.push(6)).then(() => sequence.push(8)) + sequence.push(2) + await Promise.all([first, second]) + return sequence + `), + ).toEqual([1, 2, 3, 4, 5, 6, 7, 8]) + }) + + test.failing("then rejects direct self-resolution for fulfilled and rejected sources", async () => { + // Sources: + // test/built-ins/Promise/prototype/then/resolve-settled-fulfilled-self.js + // test/built-ins/Promise/prototype/then/resolve-settled-rejected-self.js + // test/built-ins/Promise/prototype/then/resolve-pending-fulfilled-self.js + // test/built-ins/Promise/prototype/then/resolve-pending-rejected-self.js + expect( + await value(` + const observe = async (promise) => { + try { await promise; return "fulfilled" } catch (reason) { return reason.name } + } + let fulfilled + let rejected + fulfilled = Promise.resolve().then(() => fulfilled) + rejected = Promise.reject().then(undefined, () => rejected) + return await Promise.all([observe(fulfilled), observe(rejected)]) + `), + ).toEqual(["TypeError", "TypeError"]) + }) + + test.failing("catch delegates rejection handling and preserves fulfillment", async () => { + // Sources: + // test/built-ins/Promise/prototype/catch/S25.4.5.1_A2.1_T1.js + // test/built-ins/Promise/prototype/catch/S25.4.5.1_A3.1_T1.js + // test/built-ins/Promise/prototype/catch/S25.4.5.1_A3.1_T2.js + expect( + await value(` + return [ + await Promise.resolve(1).catch(() => 2), + await Promise.reject(3).catch((reason) => reason + 1), + ] + `), + ).toEqual([1, 4]) + }) + + test.failing("finally preserves or replaces the original settlement", async () => { + // Sources: + // test/built-ins/Promise/prototype/finally/resolution-value-no-override.js + // test/built-ins/Promise/prototype/finally/rejection-reason-no-fulfill.js + // test/built-ins/Promise/prototype/finally/rejection-reason-override-with-throw.js + expect( + await value(` + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + return await Promise.all([ + observe(Promise.resolve(1).finally(() => 2)), + observe(Promise.reject(3).finally(() => 4)), + observe(Promise.reject(5).finally(() => { throw 6 })), + ]) + `), + ).toEqual([ + ["fulfilled", 1], + ["rejected", 3], + ["rejected", 6], + ]) + }) + + test.failing("await always resumes in a later reaction and interleaves async functions", async () => { + // Sources: + // test/language/expressions/await/async-await-interleaved.js + // test/language/expressions/await/await-non-promise.js + expect( + await value(` + const sequence = [] + const first = async () => { sequence.push("first:1"); await 0; sequence.push("first:2") } + const second = async () => { sequence.push("second:1"); await 0; sequence.push("second:2") } + await Promise.all([first(), second()]) + return sequence + `), + ).toEqual(["first:1", "second:1", "first:2", "second:2"]) + }) + + test.failing("an async function rejects when it resolves with its own promise", async () => { + // Adapted from the self-resolution requirement represented by: + // test/built-ins/Promise/resolve-self.js + // test/built-ins/Promise/resolve/S25.4.4.5_A4.1_T1.js + expect( + await value(` + let promise + const run = async () => { + await Promise.resolve() + return promise + } + promise = run() + try { + await promise + return "fulfilled" + } catch (error) { + return error.name + } + `), + ).toBe("TypeError") + }) + + test.failing("Promise.resolve recursively assimilates callable thenables", async () => { + // Source: test/built-ins/Promise/resolve/resolve-thenable.js + expect( + await value(` + const value = { id: 1 } + const nested = { then: (resolve) => resolve(value) } + const thenable = { then: (resolve) => resolve(nested) } + return (await Promise.resolve(thenable)) === value + `), + ).toBe(true) + }) + + test.failing("Promise combinators assimilate callable thenable inputs", async () => { + // Sources: + // test/built-ins/Promise/all/reject-immed.js + // test/built-ins/Promise/all/reject-ignored-immed.js + // test/built-ins/Promise/allSettled/reject-ignored-immed.js + // test/built-ins/Promise/race/resolve-thenable.js + expect( + await value(` + const fulfills = { then: (resolve) => resolve(1) } + const rejects = { then: (_, reject) => reject(2) } + const resolvesFirst = { then: (resolve, reject) => { resolve(3); reject(4) } } + const observe = async (promise) => { + try { return ["fulfilled", await promise] } catch (reason) { return ["rejected", reason] } + } + return [ + await observe(Promise.all([fulfills, rejects])), + await Promise.allSettled([fulfills, resolvesFirst]), + await observe(Promise.race([rejects])), + ] + `), + ).toEqual([ + ["rejected", 2], + [ + { status: "fulfilled", value: 1 }, + { status: "fulfilled", value: 3 }, + ], + ["rejected", 2], + ]) + }) + + test.failing("await assimilates callable thenables", async () => { + // Source: test/language/expressions/await/await-awaits-thenables.js + expect( + await value(` + const thenable = { then: (resolve) => resolve(42) } + return await thenable + `), + ).toBe(42) + }) + + test.failing("await rejects when a callable thenable throws", async () => { + // Source: test/language/expressions/await/await-awaits-thenables-that-throw.js + expect( + await value(` + const error = { id: 1 } + const thenable = { then: () => { throw error } } + try { + await thenable + return false + } catch (caught) { + return caught === error + } + `), + ).toBe(true) + }) +}) diff --git a/packages/codemode/test/promise.test.ts b/packages/codemode/test/promise.test.ts index ce5f1e03a49..847cd6fb155 100644 --- a/packages/codemode/test/promise.test.ts +++ b/packages/codemode/test/promise.test.ts @@ -48,6 +48,13 @@ const failingTool = Tool.make({ run: () => Effect.fail(toolError("Lookup refused")), }) +const interruptedTool = Tool.make({ + description: "Interrupt this call", + input: Schema.Struct({}), + output: Schema.String, + run: () => Effect.interrupt, +}) + const completedTool = (trace: Trace) => Tool.make({ description: "Return the number of completed sleepy calls", @@ -56,6 +63,25 @@ const completedTool = (trace: Trace) => run: () => Effect.succeed(trace.completed), }) +/** Never settles, and holds interruption cleanup for `cleanupMs` so completion cleanup can outlast a timeout. */ +const stubbornTool = (trace: Trace) => + Tool.make({ + description: "Never settle; clean up slowly when interrupted", + input: Schema.Struct({ cleanupMs: Schema.Number }), + output: Schema.Number, + run: ({ cleanupMs }) => + Effect.never.pipe( + Effect.onInterrupt(() => + Effect.andThen( + Effect.sleep(cleanupMs), + Effect.sync(() => { + trace.interrupted += 1 + }), + ), + ), + ), + }) + const run = ( code: string, options: { trace?: Trace; limits?: CodeMode.ExecutionLimits } = {}, @@ -63,7 +89,15 @@ const run = ( const trace = options.trace ?? makeTrace() return Effect.runPromise( CodeMode.execute({ - tools: { host: { sleepy: sleepyTool(trace), fail: failingTool, completed: completedTool(trace) } }, + tools: { + host: { + sleepy: sleepyTool(trace), + fail: failingTool, + interrupt: interruptedTool, + completed: completedTool(trace), + stubborn: stubbornTool(trace), + }, + }, code, ...(options.limits ? { limits: options.limits } : {}), }), @@ -174,8 +208,7 @@ describe("first-class promise values", () => { }) test("an awaited failure is catchable exactly like a synchronous throw", async () => { - expect( - await value(` + const result = await run(` const p = tools.host.fail({}) try { await p @@ -183,57 +216,195 @@ describe("first-class promise values", () => { } catch (e) { return e.message } - `), - ).toBe("Lookup refused") + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("Lookup refused") + expect(result.warnings).toBeUndefined() }) - test("a fire-and-forget call completes before the execution ends", async () => { + test("a fire-and-forget call is interrupted when the program returns", async () => { const trace = makeTrace() - const result = await value( + const result = await run( ` tools.host.sleepy({ id: 1, ms: 30 }) return "done" `, { trace }, ) - expect(result).toBe("done") - expect(trace.completed).toBe(1) - expect(trace.interrupted).toBe(0) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toBeUndefined() + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) }) - test("a never-awaited failing call surfaces as an unhandled-rejection diagnostic", async () => { - const diagnostic = await error(` + test("a never-awaited failing call preserves the result and reports the rejection", async () => { + const result = await run(` tools.host.fail({}) return "done" `) - expect(diagnostic.kind).toBe("ToolFailure") - expect(diagnostic.message).toContain("Unhandled rejection from an un-awaited promise") - expect(diagnostic.message).toContain("Lookup refused") - expect(diagnostic.suggestions?.join(" ")).toContain("Await promises") + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toStrictEqual([ + { kind: "ToolFailure", message: "Unhandled rejection from an un-awaited promise: Lookup refused" }, + ]) + expect(Schema.decodeUnknownSync(CodeMode.Result)(JSON.parse(JSON.stringify(result)))).toStrictEqual(result) }) - test("a never-awaited failing async function surfaces as an unhandled promise rejection", async () => { - const diagnostic = await error(` + test("a never-awaited failing async function is reported with a successful result", async () => { + const result = await run(` const fail = async () => { throw new Error("boom") } fail() return "done" `) - expect(diagnostic.kind).toBe("ExecutionFailure") - expect(diagnostic.message).toContain("Unhandled rejection from an un-awaited promise") - expect(diagnostic.message).toContain("boom") + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toStrictEqual([ + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: boom" }, + ]) }) - test("drains promises started by an async function after an await", async () => { - const diagnostic = await error(` - const run = async () => { - await tools.host.sleepy({ id: 1 }) - tools.host.fail({}) - } - run() + test("output truncation bounds warning diagnostics with an in-band marker", async () => { + const result = await run( + ` + for (let i = 0; i < 100; i += 1) Promise.reject(new Error("x".repeat(1_000))) + return "done" + `, + { limits: { maxOutputBytes: 64 } }, + ) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.truncated).toBe(true) + expect(result.warnings).toStrictEqual([ + { kind: "Truncated", message: "100 additional warnings omitted by the output limit." }, + ]) + }) + + test("a budget-consuming value does not starve warnings", async () => { + const result = await run( + ` + Promise.reject(new Error("boom")) + return "x".repeat(500) + `, + { limits: { maxOutputBytes: 128 } }, + ) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.truncated).toBe(true) + expect(typeof result.value).toBe("string") + expect(result.warnings).toStrictEqual([ + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: boom" }, + ]) + }) + + test("an un-awaited async function's pending chain is interrupted at the return", async () => { + const trace = makeTrace() + const result = await run( + ` + const run = async () => { + await tools.host.sleepy({ id: 1, ms: 60000 }) + tools.host.fail({}) + } + run() + return "done" + `, + { trace }, + ) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toBeUndefined() + expect(trace.starts).toEqual([1]) + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + + test("reports every unhandled rejection in promise creation order", async () => { + const result = await run(` + Promise.reject(new Error("first")) + tools.host.fail({}) + Promise.reject(new Error("third")) return "done" `) - expect(diagnostic.kind).toBe("ToolFailure") - expect(diagnostic.message).toContain("Lookup refused") + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.warnings).toStrictEqual([ + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: first" }, + { kind: "ToolFailure", message: "Unhandled rejection from an un-awaited promise: Lookup refused" }, + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: third" }, + ]) + }) + + test("orders an async function rejection before promises created inside its body", async () => { + const result = await run(` + const outer = async () => { + Promise.reject(new Error("inner")) + throw new Error("outer") + } + outer() + return "done" + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.warnings).toStrictEqual([ + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: outer" }, + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: inner" }, + ]) + }) + + test("un-awaited interruptions settle without becoming rejections", async () => { + const result = await run(` + tools.host.interrupt({}) + Promise.all([tools.host.interrupt({})]) + return "done" + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toBeUndefined() + }) + + test("a fatal program error cancels outstanding work without reporting unhandled rejections", async () => { + const trace = makeTrace() + const result = await run( + ` + tools.host.sleepy({ id: 1, ms: 1_000 }) + throw new Error("boom") + `, + { trace }, + ) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.message).toBe("Uncaught: boom") + expect("warnings" in result).toBe(false) + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + + test("async-function promises remain owned by the execution after the function returns", async () => { + const trace = makeTrace() + expect( + await value( + ` + const launch = async () => { + tools.host.sleepy({ id: 1, ms: 60000 }) + Promise.all([tools.host.sleepy({ id: 2, ms: 60000 })]) + return "returned" + } + return await launch() + `, + { trace }, + ), + ).toBe("returned") + // Both calls outlive launch() itself - they belong to the execution, not the function - + // and are interrupted only when the whole program returns. + expect(trace.starts).toEqual([1, 2]) + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(2) }) }) @@ -251,6 +422,22 @@ describe("promises at data boundaries", () => { expect(diagnostic.message).toContain("un-awaited Promise") }) + test("invalid returned data cancels pending work", async () => { + const trace = makeTrace() + const result = await run( + ` + const pending = tools.host.sleepy({ id: 1, ms: 60_000 }) + return { pending } + `, + { trace, limits: { timeoutMs: 100 } }, + ) + expect(result.ok).toBe(false) + if (result.ok) return + expect(result.error.kind).toBe("InvalidDataValue") + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + test("passing an un-awaited promise as a tool argument is a clear diagnostic", async () => { const diagnostic = await error(`return await tools.host.sleepy({ id: tools.host.sleepy({ id: 1 }) })`) expect(diagnostic.kind).toBe("InvalidDataValue") @@ -270,6 +457,59 @@ describe("promises at data boundaries", () => { }) describe("Promise.all over arbitrary arrays", () => { + test("combinators return promises that can be assigned and awaited later", async () => { + expect( + await value(` + const all = Promise.all([Promise.resolve(1)]) + const settled = Promise.allSettled([Promise.reject("no")]) + const race = Promise.race([Promise.resolve(2)]) + const promises = [all instanceof Promise, settled instanceof Promise, race instanceof Promise] + return [promises, await all, await settled, await race] + `), + ).toEqual([[true, true, true], [1], [{ status: "rejected", reason: "no" }], 2]) + }) + + test("separately-created aggregate batches overlap before either is awaited", async () => { + const trace = makeTrace() + expect( + await value( + ` + const first = Promise.all([tools.host.sleepy({ id: 1, ms: 40 })]) + const second = Promise.all([tools.host.sleepy({ id: 2, ms: 40 })]) + return [await first, await second] + `, + { trace }, + ), + ).toEqual([[1], [2]]) + expect(trace.starts).toEqual([1, 2]) + expect(trace.maxActive).toBeGreaterThan(1) + }) + + test("an aggregate created before a try block rejects at its later await", async () => { + expect( + await value(` + const aggregate = Promise.all([tools.host.fail({})]) + try { + await aggregate + return "no" + } catch (error) { + return error.message + } + `), + ).toBe("Lookup refused") + }) + + test("awaiting an aggregate repeatedly does not rerun its members", async () => { + const result = await run(` + const aggregate = Promise.all([tools.host.sleepy({ id: 7 })]) + return [await aggregate, await aggregate] + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toEqual([[7], [7]]) + expect(result.toolCalls).toStrictEqual([{ name: "host.sleepy" }]) + }) + test("mixes promises and plain values, preserving order", async () => { expect( await value(` @@ -340,16 +580,18 @@ describe("Promise.all over arbitrary arrays", () => { }) test("rejects with the first failure, catchable in-program", async () => { - expect( - await value(` + const result = await run(` try { await Promise.all([tools.host.sleepy({ id: 1 }), tools.host.fail({})]) return "no" } catch (e) { return e.message } - `), - ).toBe("Lookup refused") + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("Lookup refused") + expect(result.warnings).toBeUndefined() }) test("rejects before an earlier slow promise fulfills", async () => { @@ -370,10 +612,55 @@ describe("Promise.all over arbitrary arrays", () => { { trace }, ), ).toBe(0) + // The surviving member is observed (Promise.all handled it), so completion interrupts + // it instead of waiting for it. + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + + test("fail-fast does not cancel a sibling the program still holds and awaits", async () => { + const trace = makeTrace() + expect( + await value( + ` + const slow = tools.host.sleepy({ id: 1, ms: 40 }) + try { + await Promise.all([slow, tools.host.fail({})]) + return "no" + } catch {} + return await slow + `, + { trace }, + ), + ).toBe(1) expect(trace.completed).toBe(1) expect(trace.interrupted).toBe(0) }) + test("a slower observed sibling is interrupted at completion after failing fast", async () => { + const trace = makeTrace() + expect( + await value( + ` + const failLater = async () => { + await tools.host.sleepy({ id: 1, ms: 40 }) + throw new Error("later") + } + const aggregate = Promise.all([Promise.reject(new Error("first")), failLater()]) + try { + await aggregate + return "no" + } catch (error) { + return error.message + } + `, + { trace }, + ), + ).toBe("first") + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + test("a non-collection argument is a clear error", async () => { const diagnostic = await error(`return await Promise.all(42)`) expect(diagnostic.message).toContain("Promise.all expects an array") @@ -413,50 +700,64 @@ describe("Promise.allSettled", () => { return settled.filter((s) => s.status === "rejected").length `) expect(result.ok).toBe(true) - if (result.ok) expect(result.value).toBe(2) + if (!result.ok) return + expect(result.value).toBe(2) + expect(result.warnings).toBeUndefined() }) }) describe("Promise.race", () => { - test("first settlement wins and losers are interrupted", async () => { + test("first settlement wins and a direct loser is interrupted at completion", async () => { const trace = makeTrace() const result = await value( ` const fast = tools.host.sleepy({ id: 1, ms: 10 }) - const slow = tools.host.sleepy({ id: 2, ms: 5000 }) + const slow = tools.host.sleepy({ id: 2, ms: 40 }) return await Promise.race([fast, slow]) `, { trace }, ) expect(result).toBe(1) - expect(trace.interrupted).toBe(1) + // The loser is observed (the race handled it), so the execution does not wait for it. expect(trace.completed).toBe(1) + expect(trace.interrupted).toBe(1) }) - test("awaiting an interrupted loser afterwards is a catchable program failure", async () => { + test("a direct loser remains awaitable after the race settles", async () => { expect( await value(` const fast = tools.host.sleepy({ id: 1, ms: 10 }) - const slow = tools.host.sleepy({ id: 2, ms: 5000 }) + const slow = tools.host.sleepy({ id: 2, ms: 40 }) const winner = await Promise.race([fast, slow]) - try { - await slow - return "no" - } catch (e) { - return { winner, caught: e.message } - } + return { winner, loser: await slow } `), - ).toEqual({ - winner: 1, - caught: "This tool call was interrupted because another value settled a Promise.race first.", - }) + ).toEqual({ winner: 1, loser: 2 }) + }) + + test("a nested aggregate loser and its members are interrupted at completion", async () => { + const trace = makeTrace() + expect( + await value( + ` + const nested = Promise.all([ + tools.host.sleepy({ id: 1, ms: 40 }), + tools.host.sleepy({ id: 2, ms: 40 }), + ]) + return await Promise.race(["immediate", nested]) + `, + { trace }, + ), + ).toBe("immediate") + // The nested aggregate and its members are all observed, so nothing waits for them. + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(2) }) test("a rejection can win the race", async () => { expect( await value(` try { - await Promise.race([tools.host.fail({}), tools.host.sleepy({ id: 1, ms: 5000 })]) + await Promise.race([tools.host.fail({}), tools.host.sleepy({ id: 1, ms: 40 })]) return "no" } catch (e) { return e.message @@ -468,11 +769,20 @@ describe("Promise.race", () => { test("a plain value wins over pending promises", async () => { const trace = makeTrace() expect( - await value(`return await Promise.race([tools.host.sleepy({ id: 1, ms: 5000 }), "immediate"])`, { trace }), + await value(`return await Promise.race([tools.host.sleepy({ id: 1, ms: 40 }), "immediate"])`, { trace }), ).toBe("immediate") + expect(trace.completed).toBe(0) expect(trace.interrupted).toBe(1) }) + test("a rejected race loser is observed by the aggregate", async () => { + const result = await run(`return await Promise.race(["winner", tools.host.fail({})])`) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("winner") + expect(result.warnings).toBeUndefined() + }) + test("an empty race is a clear error instead of hanging", async () => { const diagnostic = await error(`return await Promise.race([])`) expect(diagnostic.message).toContain("never settle") @@ -484,6 +794,9 @@ describe("Promise.resolve / Promise.reject", () => { expect(await value(`return await Promise.resolve(42)`)).toBe(42) expect(await value(`return await Promise.resolve(Promise.resolve("nested"))`)).toBe("nested") expect(await value(`return await Promise.resolve(tools.host.sleepy({ id: 3 }))`)).toBe(3) + expect(await value(`const promise = Promise.resolve(1); return [promise].includes(Promise.resolve(promise))`)).toBe( + true, + ) }) test("reject produces a promise whose await throws the reason", async () => { @@ -498,6 +811,34 @@ describe("Promise.resolve / Promise.reject", () => { `), ).toBe("nope") }) + + test("a rejection observed after settlement is handled", async () => { + expect( + await value(` + const rejected = Promise.reject(new Error("handled")) + await tools.host.sleepy({ id: 1 }) + try { + await rejected + return "no" + } catch (error) { + return error.message + } + `), + ).toBe("handled") + }) + + test("an abandoned rejected promise is reported as unhandled", async () => { + const result = await run(` + Promise.reject(new Error("abandoned")) + return "done" + `) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toStrictEqual([ + { kind: "ExecutionFailure", message: "Unhandled rejection from an un-awaited promise: Uncaught: abandoned" }, + ]) + }) }) describe("timeout interruption of forked calls", () => { @@ -531,6 +872,67 @@ describe("timeout interruption of forked calls", () => { expect(result.error.kind).toBe("TimeoutExceeded") expect(trace.interrupted).toBe(2) }) + + test("a non-settling race loser cannot hold the execution to the timeout", async () => { + const trace = makeTrace() + const result = await run(`return await Promise.race(["winner", tools.host.sleepy({ id: 1, ms: 60000 })])`, { + trace, + limits: { timeoutMs: 100 }, + }) + // Completion interrupts the observed loser immediately; the race result survives. + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("winner") + expect(result.warnings).toBeUndefined() + expect(trace.starts).toEqual([1]) + expect(trace.completed).toBe(0) + expect(trace.interrupted).toBe(1) + }) + + test("a timeout during completion cleanup keeps the computed value and warns", async () => { + const trace = makeTrace() + const result = await run( + ` + tools.host.stubborn({ cleanupMs: 400 }) + return "done" + `, + { trace, limits: { timeoutMs: 100 } }, + ) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toStrictEqual([ + { + kind: "TimeoutExceeded", + message: + "The program returned, but background work was still running at the 100ms timeout and was interrupted. Await all started promises.", + }, + ]) + expect(trace.interrupted).toBe(1) + expect(trace.completed).toBe(0) + }) + + test("a timeout during completion cleanup reports the timeout warning before settled rejections", async () => { + const result = await run( + ` + tools.host.fail({}) + tools.host.stubborn({ cleanupMs: 400 }) + return "done" + `, + { limits: { timeoutMs: 100 } }, + ) + expect(result.ok).toBe(true) + if (!result.ok) return + expect(result.value).toBe("done") + expect(result.warnings).toStrictEqual([ + { + kind: "TimeoutExceeded", + message: + "The program returned, but background work was still running at the 100ms timeout and was interrupted. Await all started promises.", + }, + { kind: "ToolFailure", message: "Unhandled rejection from an un-awaited promise: Lookup refused" }, + ]) + }) }) describe("unsupported promise surface", () => { diff --git a/packages/codemode/test/test262-array.md b/packages/codemode/test/test262-array.md deleted file mode 100644 index b8e7e07a897..00000000000 --- a/packages/codemode/test/test262-array.md +++ /dev/null @@ -1,77 +0,0 @@ -# Test262 Array Coverage - -The Array tests adapt Test262 at revision `250f204f23a9249ff204be2baec29600faae7b75`. They cover CodeMode's 35 -exposed instance methods and three static methods using actual arrays, accepted argument types, deterministic behavior, -and CodeMode's materialized collection conventions. Each executable case names its exact upstream source path. -`LICENSE.test262` contains the upstream BSD terms. - -This is coverage of CodeMode's bounded Array surface, not a claim of ECMAScript or Test262 conformance. One upstream -file may contain both adapted and inapplicable assertions, so a cited source means only that the represented assertions -were adapted. - -## Inventory - -The 38 relevant upstream API directories contain 2,837 files. The executable suite adapts assertions from 83 distinct -sources. - -| API | Upstream files | Adapted sources | -| ------------------------------- | -------------: | --------------: | -| `Array.prototype.map` | 216 | 3 | -| `Array.prototype.filter` | 242 | 3 | -| `Array.prototype.find` | 23 | 4 | -| `Array.prototype.findIndex` | 23 | 3 | -| `Array.prototype.findLast` | 24 | 3 | -| `Array.prototype.findLastIndex` | 24 | 3 | -| `Array.prototype.some` | 219 | 2 | -| `Array.prototype.every` | 218 | 2 | -| `Array.prototype.includes` | 30 | 2 | -| `Array.prototype.join` | 23 | 2 | -| `Array.prototype.reduce` | 260 | 3 | -| `Array.prototype.reduceRight` | 260 | 3 | -| `Array.prototype.flatMap` | 24 | 2 | -| `Array.prototype.forEach` | 190 | 2 | -| `Array.prototype.sort` | 54 | 3 | -| `Array.prototype.toSorted` | 21 | 4 | -| `Array.prototype.slice` | 71 | 1 | -| `Array.prototype.concat` | 69 | 3 | -| `Array.prototype.indexOf` | 201 | 2 | -| `Array.prototype.lastIndexOf` | 198 | 2 | -| `Array.prototype.at` | 13 | 3 | -| `Array.prototype.flat` | 19 | 2 | -| `Array.prototype.reverse` | 18 | 1 | -| `Array.prototype.toReversed` | 17 | 2 | -| `Array.prototype.with` | 21 | 2 | -| `Array.prototype.push` | 24 | 1 | -| `Array.prototype.pop` | 23 | 1 | -| `Array.prototype.shift` | 20 | 1 | -| `Array.prototype.unshift` | 22 | 1 | -| `Array.prototype.splice` | 81 | 3 | -| `Array.prototype.fill` | 22 | 3 | -| `Array.prototype.copyWithin` | 39 | 2 | -| `Array.prototype.keys` | 12 | 1 | -| `Array.prototype.values` | 12 | 1 | -| `Array.prototype.entries` | 12 | 1 | -| `Array.from` | 47 | 3 | -| `Array.isArray` | 29 | 2 | -| `Array.of` | 16 | 1 | - -## Exclusions - -Assertions are not adapted when they test behavior outside CodeMode's documented Array surface: - -- Function metadata, property descriptors, constructibility, prototype mutation, species constructors, or cross-realm - identity. -- Generic receivers, detached methods, `.call`, `.apply`, boxed values, custom coercion objects, Symbols, BigInts, - proxies, accessors, frozen arrays, typed arrays, or ArrayBuffers. -- `Array.from` mappers, custom iterables, constructor substitution, and iterator-closing behavior. -- Native iterator identity, `.next()`, completion records, or live iterator mutation. CodeMode deliberately materializes - `keys`, `values`, and `entries` as arrays. -- Sparse-array assertions that depend on literal elisions or inherited indexed properties. CodeMode's confined data - model does not preserve those prototype and hole semantics at every boundary. -- Argument coercions outside the accepted schema-like surface. Numeric positions must be numbers and `join` separators - must be strings. -- Exact native error brands where CodeMode exposes a safe runtime error instead. -- Async/effectful callbacks, circular-data rejection, sandbox-value identity, diagnostics, and host-boundary behavior. - Those remain covered by CodeMode-specific tests. - -Handwritten tests remain where they specify CodeMode behavior rather than ordinary ECMAScript Array semantics. diff --git a/packages/codemode/test/test262-string.md b/packages/codemode/test/test262-string.md deleted file mode 100644 index 94d01f83d9d..00000000000 --- a/packages/codemode/test/test262-string.md +++ /dev/null @@ -1,69 +0,0 @@ -# Test262 String Coverage - -The String tests adapt Test262 at revision `250f204f23a9249ff204be2baec29600faae7b75`. They cover CodeMode's 32 -exposed instance methods and two static methods using primitive receivers, accepted argument types, and deterministic -behavior. Each executable case names its exact upstream source path. `LICENSE.test262` contains the upstream BSD terms. - -This is coverage of CodeMode's bounded String surface, not a claim of ECMAScript or Test262 conformance. One upstream -file may contain both adapted and inapplicable assertions, so a cited source means only that the represented assertions -were adapted. - -## Inventory - -The relevant upstream directories contain 1,048 files: 1,009 core built-in files, 29 Annex B files for exposed methods, -and 10 Intl `localeCompare` files. The executable suite adapts assertions from 298 distinct sources. - -| API | Upstream files | Adapted sources | -| --- | ---: | ---: | -| `String.fromCharCode` | 17 | 6 | -| `String.fromCodePoint` | 11 | 4 | -| `String.prototype.at` | 11 | 5 | -| `String.prototype.charAt` | 30 | 9 | -| `String.prototype.charCodeAt` | 25 | 4 | -| `String.prototype.codePointAt` | 16 | 6 | -| `String.prototype.concat` | 22 | 1 | -| `String.prototype.endsWith` | 27 | 13 | -| `String.prototype.includes` | 27 | 12 | -| `String.prototype.indexOf` | 47 | 8 | -| `String.prototype.lastIndexOf` | 25 | 1 | -| `String.prototype.localeCompare` | 23 | 1 | -| `String.prototype.match` | 52 | 9 | -| `String.prototype.matchAll` | 26 | 1 | -| `String.prototype.normalize` | 14 | 3 | -| `String.prototype.padEnd` | 13 | 4 | -| `String.prototype.padStart` | 13 | 4 | -| `String.prototype.repeat` | 16 | 4 | -| `String.prototype.replace` | 56 | 16 | -| `String.prototype.replaceAll` | 46 | 12 | -| `String.prototype.search` | 44 | 10 | -| `String.prototype.slice` | 38 | 11 | -| `String.prototype.split` | 121 | 50 | -| `String.prototype.startsWith` | 21 | 7 | -| `String.prototype.substr` | 15 | 6 | -| `String.prototype.substring` | 46 | 12 | -| `String.prototype.toLowerCase` | 30 | 5 | -| `String.prototype.toString` | 7 | 1 | -| `String.prototype.toUpperCase` | 26 | 3 | -| `String.prototype.trim` | 129 | 66 | -| `String.prototype.trimEnd` | 23 | 2 | -| `String.prototype.trimLeft` | 4 | 0 | -| `String.prototype.trimRight` | 4 | 0 | -| `String.prototype.trimStart` | 23 | 2 | - -## Exclusions - -Assertions are not adapted when they test behavior outside CodeMode's documented String surface: - -- Function metadata, property descriptors, constructibility, prototype mutation, or cross-realm identity. -- The `trimLeft`/`trimRight` Test262 files assert prototype function identity, which CodeMode does not expose. Their - supported call behavior remains covered by CodeMode-specific tests. -- Boxed strings, generic receivers, custom coercion objects, Symbols, BigInts, or argument types CodeMode rejects. -- Symbol-based RegExp dispatch, custom matchers, species constructors, or iterator protocol details. CodeMode materializes - `matchAll` results instead of exposing iterators. -- Locale selection and options. CodeMode deliberately uses the host default locale and ignores those arguments. -- Test262 harness behavior or setup syntax unavailable in the confined interpreter. -- Function-replacer behavior that is covered by CodeMode-specific tests for sequential callbacks, async tool calls, - result coercion, diagnostics, and sandbox boundaries. -- Assertions requiring an exact native error type when CodeMode deliberately exposes only its safe runtime error. - -Handwritten tests remain where they specify CodeMode behavior rather than ordinary ECMAScript String semantics. diff --git a/packages/core/src/tool/execute.ts b/packages/core/src/tool/execute.ts index 35990f3842d..826b1ca4fad 100644 --- a/packages/core/src/tool/execute.ts +++ b/packages/core/src/tool/execute.ts @@ -169,9 +169,12 @@ function formatResult(result: CodeMode.Result) { : [result.error.message, ...(result.error.suggestions ?? []).filter((hint) => !result.error.message.includes(hint))] .join("\n") .trim() - if (!result.logs || result.logs.length === 0) return output - const logs = `Logs:\n${result.logs.join("\n")}` - return output === "" ? logs : `${output}\n\n${logs}` + const warnings = + result.ok && result.warnings && result.warnings.length > 0 + ? `Warnings:\n${result.warnings.map((item) => `- [${item.kind}] ${item.message}`).join("\n")}` + : undefined + const logs = result.logs && result.logs.length > 0 ? `Logs:\n${result.logs.join("\n")}` : undefined + return [output, warnings, logs].filter((part) => part !== undefined && part !== "").join("\n\n") } function formatValue(value: CodeMode.DataValue) { diff --git a/packages/core/test/tool-execute.test.ts b/packages/core/test/tool-execute.test.ts new file mode 100644 index 00000000000..6a1000fec4e --- /dev/null +++ b/packages/core/test/tool-execute.test.ts @@ -0,0 +1,47 @@ +import { expect, test } from "bun:test" +import { ExecuteTool } from "@opencode-ai/core/tool/execute" +import { Tool } from "@opencode-ai/core/tool/tool" +import { Agent } from "@opencode-ai/schema/agent" +import { Session } from "@opencode-ai/schema/session" +import { SessionMessage } from "@opencode-ai/schema/session-message" +import { Effect, Schema } from "effect" + +test("execute preserves successful results with visible unhandled rejections", async () => { + const child = Tool.make({ + description: "Always fail", + input: Schema.Struct({}), + output: Schema.String, + execute: () => Effect.fail(new Tool.Failure({ message: "Lookup refused" })), + }) + const execute = ExecuteTool.create(new Map([["fail", { tool: child, name: "fail" }]])) + const result = await Effect.runPromise( + Tool.settle( + execute, + { + type: "tool-call", + id: "call_execute", + name: "execute", + input: { code: `tools.fail({}); return "done"` }, + }, + { + sessionID: Session.ID.make("ses_execute"), + agent: Agent.ID.make("build"), + assistantMessageID: SessionMessage.ID.make("msg_execute"), + toolCallID: "call_execute", + }, + ), + ) + + expect(result.structured).toEqual({ toolCalls: [{ tool: "fail", status: "error" }] }) + expect(result.content).toEqual([ + { + type: "text", + text: [ + "done", + "", + "Warnings:", + "- [ToolFailure] Unhandled rejection from an un-awaited promise: Lookup refused", + ].join("\n"), + }, + ]) +})