diff --git a/.github/workflows/ios-release-e2e.yml b/.github/workflows/ios-release-e2e.yml index dd852e819041..96e3f527c727 100644 --- a/.github/workflows/ios-release-e2e.yml +++ b/.github/workflows/ios-release-e2e.yml @@ -41,7 +41,7 @@ jobs: sha = os.environ["TARGET_SHA"] mode = os.environ["E2E_MODE"] with open(os.environ["PROOF_PATH"], "w") as out: - json.dump({"schema": 1, "status": "failed", "trials": [], + json.dump({"schema": 2, "kind": "qualification", "status": "failed", "trials": [], "targetSha": sha if re.fullmatch(r"[a-f0-9]{40}", sha) else None, "harnessSha": None, "mode": mode if mode in ("stock", "compare") else None, diff --git a/apps/ios/UITests/OpenClawSnapshotUITests.swift b/apps/ios/UITests/OpenClawSnapshotUITests.swift index 85e426e5cfd4..d2d9c683799f 100644 --- a/apps/ios/UITests/OpenClawSnapshotUITests.swift +++ b/apps/ios/UITests/OpenClawSnapshotUITests.swift @@ -591,6 +591,7 @@ final class OpenClawSnapshotUITests: XCTestCase { let latestSeededReply = app.staticTexts["OPENCLAW_LONG_CHAT_LATEST"] XCTAssertTrue(latestSeededReply.waitForExistence(timeout: 8)) + let transcript = try self.chatTranscript(in: app) let work = app.buttons.matching(NSPredicate(format: "label BEGINSWITH %@", "Worked")).firstMatch XCTAssertTrue(work.waitForExistence(timeout: 5)) work.tap() @@ -599,27 +600,33 @@ final class OpenClawSnapshotUITests: XCTestCase { } XCTAssertTrue(latestSeededReply.isHittable) + // Revealing the reply after expanding work does not establish live-edge following. + let initialJumpToLatest = app.buttons["Jump to latest reply"] + if initialJumpToLatest.exists { + initialJumpToLatest.tap() + } + XCTAssertTrue(initialJumpToLatest.waitForNonExistence(timeout: 3)) + let input = self.chatMessageInput(in: app) XCTAssertTrue(input.waitForExistence(timeout: 8)) + self.waitForEnabled(input) input.tap() let keyboard = app.keyboards.firstMatch XCTAssertTrue(keyboard.waitForExistence(timeout: 3)) func visibleAreaAboveKeyboard() -> CGRect { - CGRect( + transcript.frame.intersection(CGRect( x: app.frame.minX, y: app.frame.minY, width: app.frame.width, - height: keyboard.frame.minY - app.frame.minY) + height: keyboard.frame.minY - app.frame.minY)) } XCTAssertTrue(latestSeededReply.exists) XCTAssertTrue(latestSeededReply.frame.intersects(visibleAreaAboveKeyboard())) XCTAssertLessThanOrEqual(latestSeededReply.frame.maxY, keyboard.frame.minY + 1) self.assertElementHasRenderedContent(latestSeededReply, named: "seeded reply after keyboard opens") - let promptPrefix = - "Give me a long, detailed status update covering the release plan, review feedback, " + - "open follow-ups, " - let promptSuffix = "and the next steps for the team." + let promptPrefix = "Check the " + let promptSuffix = "release plan." let prompt = promptPrefix + promptSuffix input.typeText(promptPrefix) XCTAssertTrue(latestSeededReply.exists) @@ -634,6 +641,7 @@ final class OpenClawSnapshotUITests: XCTestCase { let send = app.buttons["chat-send-message"] XCTAssertTrue(send.waitForExistence(timeout: 5)) + XCTAssertTrue(send.isEnabled) send.tap() // Regression proof for #108692 and #135214: the transcript remains rendered while typing, @@ -647,15 +655,57 @@ final class OpenClawSnapshotUITests: XCTestCase { NSPredicate(format: "label CONTAINS %@", "keep the mobile workflow connected to the gateway")) .firstMatch XCTAssertTrue(reply.waitForExistence(timeout: 8)) - Thread.sleep(forTimeInterval: 1.0) + XCTAssertTrue(app.staticTexts["Writing"].waitForNonExistence(timeout: 1)) let visibleArea = visibleAreaAboveKeyboard() - XCTAssertTrue(sentPrompt.frame.intersects(visibleArea)) XCTAssertTrue(reply.frame.intersects(visibleArea)) XCTAssertLessThanOrEqual(reply.frame.maxY, keyboard.frame.minY + 1) - self.assertElementHasRenderedContent(sentPrompt, named: "sent prompt after send") self.assertElementHasRenderedContent(reply, named: "reply after send") XCTAssertFalse(app.buttons["Jump to latest reply"].exists) self.attachScreenshot(named: "keyboard-transcript-visible-after-send") + + // Keep repeated typing and reader positioning on the real editor with deterministic history. + // The final multiline turn extends below the viewport so the anchored-reply jump is observable. + let anchoredPrompt = String( + repeating: "Reader context keeps the question anchored as the reply arrives.\n", + count: 9) + "Keep this reply anchored." + self.waitForEnabled(input) + input.tap() + input.typeText(anchoredPrompt) + XCTAssertTrue(keyboard.waitForExistence(timeout: 3)) + try self.dismissChatKeyboardThroughTranscript(in: app) + XCTAssertTrue(sentPrompt.frame.intersects(transcript.frame)) + self.assertElementHasRenderedContent(sentPrompt, named: "sent prompt after keyboard dismissal") + XCTAssertEqual(input.value as? String, anchoredPrompt) + XCTAssertTrue(send.isEnabled) + send.tap() + let submitted = app.staticTexts.matching(NSPredicate(format: "label == %@", anchoredPrompt)).firstMatch + XCTAssertTrue(submitted.waitForExistence(timeout: 5)) + let response = app.staticTexts.matching(NSPredicate( + format: "label CONTAINS %@ AND label CONTAINS %@", + "I can help with", + "Keep this reply anchored.")).firstMatch + XCTAssertTrue(response.waitForExistence(timeout: 8)) + XCTAssertTrue(app.staticTexts["Writing"].waitForNonExistence(timeout: 5)) + + let jumpToLatest = app.buttons["Jump to latest reply"] + XCTAssertTrue(jumpToLatest.waitForExistence(timeout: 3)) + self.attachScreenshot(named: "reader-reply-anchored") + jumpToLatest.tap() + XCTAssertTrue(jumpToLatest.waitForNonExistence(timeout: 3)) + let finalReply = app.staticTexts.matching(NSPredicate( + format: "label CONTAINS %@ AND label CONTAINS %@", + "I can help with", + "Keep this reply anchored.")).firstMatch + XCTAssertTrue(finalReply.exists) + self.assertElementHasRenderedContent(finalReply, named: "reader reply after jumping to latest") + self.attachScreenshot(named: "reader-jumped-to-latest") + + transcript.swipeDown() + XCTAssertTrue(jumpToLatest.waitForExistence(timeout: 3)) + self.attachScreenshot(named: "reader-manual-departure") + jumpToLatest.tap() + XCTAssertTrue(jumpToLatest.waitForNonExistence(timeout: 3)) + XCTAssertTrue(finalReply.exists) } func testCompletedWorkDisclosureKeepsFinalReplyVisible() throws { @@ -1008,69 +1058,36 @@ final class OpenClawSnapshotUITests: XCTestCase { XCTAssertEqual(refreshedArtifacts.last?["sessionKey"] as? String, artifacts.first?["sessionKey"] as? String) } - func testLiveGatewayFreshInstallSetupAndRelaunch() throws { - try XCTSkipIf(UIDevice.current.userInterfaceIdiom != .phone, "Phone setup proof only") + func testLiveGatewayPairChatAndRelaunch() throws { + try XCTSkipIf(UIDevice.current.userInterfaceIdiom != .phone, "Phone lifecycle proof only") let app = try self.launchPairedLiveGatewayApp(initialTab: "chat", initialDestination: "chat") XCTAssertEqual(app.state, .runningForeground) - - let controlApp = self.relaunchConnectedLiveGatewayApp( - initialTab: "control", - initialDestination: "overview") - XCTAssertTrue(controlApp.staticTexts["Agent session"].waitForExistence(timeout: 8)) - XCTAssertTrue(controlApp.buttons["RootTabs.Sidebar.Show"].exists) - XCTAssertEqual(controlApp.state, .runningForeground) - } - - func testLiveGatewayChatRoundTripAndControlOverview() throws { - try XCTSkipIf(UIDevice.current.userInterfaceIdiom != .phone, "Phone chat proof only") - let app = try launchPairedLiveGatewayApp(initialTab: "chat", initialDestination: "chat") - - // Seed real history with the keyboard open before checking transcript-tap dismissal and reader behavior. - for index in 0..<3 { - let seedMarker = "OPENCLAW_E2E_SEED_\(index)_\(Int(Date().timeIntervalSince1970 * 1000))" - let seedContext = String(repeating: "Reader context \(index). ", count: 6) + print("IOS_RELEASE_CHECKPOINT paired") + for (stage, marker) in [("first", "OPENCLAW_E2E_FIRST"), ("second", "OPENCLAW_E2E_SECOND")] { try self.sendLiveGatewayMessage( - "\(seedContext)Reply exactly with \(seedMarker) and no other text.", - expecting: seedMarker, - stage: "seed-\(index)", - dismissKeyboard: false, + "Reply exactly with \(marker) and no other text.", + expecting: marker, + stage: stage, in: app) + print("IOS_RELEASE_CHECKPOINT \(stage)") } - let replyMarker = "OPENCLAW_E2E_OK_\(Int(Date().timeIntervalSince1970 * 1000))" + // A new request after process termination exercises the credentials persisted during pairing. + let relaunchedApp = self.relaunchConnectedLiveGatewayApp(initialTab: "chat", initialDestination: "chat") + print("IOS_RELEASE_CHECKPOINT relaunched") try self.sendLiveGatewayMessage( - "Reply exactly with \(replyMarker) and no other text.", - expecting: replyMarker, - stage: "final", - dismissKeyboard: true, - in: app) - let jumpToLatest = app.buttons["Jump to latest reply"] - XCTAssertTrue(jumpToLatest.waitForExistence(timeout: 3)) - self.attachScreenshot(named: "live-gateway-chat-reply-anchored") - - jumpToLatest.tap() - XCTAssertTrue(jumpToLatest.waitForNonExistence(timeout: 3)) - XCTAssertTrue(app.staticTexts[replyMarker].exists) - Thread.sleep(forTimeInterval: 0.5) - self.attachScreenshot(named: "live-gateway-chat-jumped-to-latest") - - let transcript = try self.chatTranscript(in: app) - XCTAssertTrue(transcript.exists) - transcript.swipeDown() - XCTAssertTrue(jumpToLatest.waitForExistence(timeout: 3)) - self.attachScreenshot(named: "live-gateway-chat-manual-departure") - jumpToLatest.tap() - XCTAssertTrue(jumpToLatest.waitForNonExistence(timeout: 3)) - - let controlApp = self.relaunchConnectedLiveGatewayApp( - initialTab: "control", - initialDestination: "overview") - XCTAssertTrue(controlApp.staticTexts["Agent session"].waitForExistence(timeout: 8)) - self.attachScreenshot(named: "live-gateway-control") + "Reply exactly with OPENCLAW_E2E_RELAUNCH and no other text.", + expecting: "OPENCLAW_E2E_RELAUNCH", + stage: "relaunch", + in: relaunchedApp) + print("IOS_RELEASE_CHECKPOINT relaunch") try self.selectSidebarDestination("Overview") - XCTAssertTrue(controlApp.buttons["Gateway settings"].waitForExistence(timeout: 5)) + XCTAssertTrue(relaunchedApp.staticTexts["Agent session"].waitForExistence(timeout: 8)) + XCTAssertTrue(relaunchedApp.buttons["RootTabs.Sidebar.Show"].exists) + XCTAssertTrue(relaunchedApp.buttons["Gateway settings"].waitForExistence(timeout: 5)) self.attachScreenshot(named: "live-gateway-overview") - XCTAssertEqual(controlApp.state, .runningForeground) + XCTAssertEqual(relaunchedApp.state, .runningForeground) + print("IOS_RELEASE_CHECKPOINT overview") } func testManualAuthRetryUsesEditedToken() throws { @@ -1843,7 +1860,7 @@ extension OpenClawSnapshotUITests { initialTab: String, initialDestination: String) -> XCUIApplication { - self.app?.terminate() + self.terminateCurrentApp() let app = XCUIApplication() app.launchArguments += [ "--openclaw-initial-tab", @@ -1868,7 +1885,6 @@ extension OpenClawSnapshotUITests { _ text: String, expecting replyMarker: String, stage: String, - dismissKeyboard: Bool, in app: XCUIApplication) throws { let input = self.chatMessageInput(in: app) @@ -1880,27 +1896,6 @@ extension OpenClawSnapshotUITests { let send = app.buttons["chat-send-message"] XCTAssertTrue(send.waitForExistence(timeout: 3)) XCTAssertTrue(send.isEnabled) - if dismissKeyboard { - // Typing can move historical replies off-screen; tap visible text without activating an action. - let transcript = try self.chatTranscript(in: app) - let actionQueries = [transcript.buttons, transcript.links] - let dismissalText = try XCTUnwrap( - transcript.staticTexts.allElementsBoundByIndex.first { candidate in - guard candidate.isHittable, - candidate.buttons.count == 0, - candidate.links.count == 0 - else { - return false - } - let label = NSPredicate(format: "label == %@", candidate.label) - return actionQueries.allSatisfy { - !$0.matching(label).firstMatch.exists && !$0.containing(label).firstMatch.exists - } - }, - "Expected visible noninteractive transcript text") - dismissalText.tap() - XCTAssertTrue(app.keyboards.firstMatch.waitForNonExistence(timeout: 3)) - } XCTAssertEqual(input.value as? String, text) send.tap() @@ -1919,6 +1914,28 @@ extension OpenClawSnapshotUITests { XCTAssertTrue(app.staticTexts["Writing"].waitForNonExistence(timeout: 5)) } + private func dismissChatKeyboardThroughTranscript(in app: XCUIApplication) throws { + // Typing can move historical replies off-screen; tap visible text without activating an action. + let transcript = try self.chatTranscript(in: app) + let actionQueries = [transcript.buttons, transcript.links] + let dismissalText = try XCTUnwrap( + transcript.staticTexts.allElementsBoundByIndex.first { candidate in + guard candidate.isHittable, + candidate.buttons.count == 0, + candidate.links.count == 0 + else { + return false + } + let label = NSPredicate(format: "label == %@", candidate.label) + return actionQueries.allSatisfy { + !$0.matching(label).firstMatch.exists && !$0.containing(label).firstMatch.exists + } + }, + "Expected visible noninteractive transcript text") + dismissalText.tap() + XCTAssertTrue(app.keyboards.firstMatch.waitForNonExistence(timeout: 3)) + } + private func openChatGatewaySettings( file: StaticString = #filePath, line: UInt = #line) throws diff --git a/apps/ios/fastlane/SETUP.md b/apps/ios/fastlane/SETUP.md index d12c77b96447..5f6ff0275e20 100644 --- a/apps/ios/fastlane/SETUP.md +++ b/apps/ios/fastlane/SETUP.md @@ -171,10 +171,47 @@ that supports the test device and architecture, and records that runtime in its It builds the Gateway runtime and ad-hoc-signed Debug `OpenClawUITests` simulator products once. Ad-hoc signing preserves Keychain entitlements without certificates or provisioning profiles; this is not a signed -Release build. Each of the two live Gateway UI tests gets a new simulator, isolated real -Gateway, and fresh setup code. Chat uses the deterministic local -`openai/ios-e2e` provider fixture. Native Overview runs with Control UI disabled; -this is not screenshot mode or a substitute for external-provider validation. +Release build. Each arm starts an isolated real Gateway, then prepares its setup +handler and state worker with `device.pair.setupStatus` before booting one new +simulator. This status preparation prunes expired completion records without issuing +a credential. After boot, the same Gateway issues the fresh setup code consumed by +the app. The live test pairs a fresh install and +verifies the `first` and `second` message round trips. It then terminates and +relaunches the app, verifies the `relaunch` message on the restored connection, +and opens native Overview. Each message must reach the deterministic local +`openai/ios-e2e` provider fixture as the latest user request. + +The harness then stops the Gateway and provider fixture and runs the deterministic +keyboard/transcript reader test on the same simulator. Its first fixture send checks +that the transcript remains rendered above the open keyboard and follows the new +reply. Its second, multiline send checks keyboard dismissal, retained draft text, +reply anchoring, jumping to the latest reply, and returning after manual scrolling. +Control UI is disabled; this does not replace external-provider validation. + +To reuse native products for another complete qualification at the same source and +toolchain, pass `--build-dir /absolute/path/to/ios-e2e-build` to both invocations. +The build owner verifies the source, toolchain, build arguments, and product integrity +before reuse. Each qualification still creates fresh simulator and Gateway resources. +The explicit build directory retains native products and their receipt; it does not +retain raw XCTest results. + +For a narrower local diagnostic, use either: + +```bash +# Prepare native products without starting a simulator or Gateway. +node --import ./scripts/tsx.mjs scripts/ios-release-e2e.ts \ + --mode stock --target-sha "$(git rev-parse HEAD)" \ + --build-dir /tmp/ios-e2e-build --build-only --output /tmp/ios-e2e-build.json + +# Exercise Gateway startup, setup-status preparation, and code issuance without native resources. +node --import ./scripts/tsx.mjs scripts/ios-release-e2e.ts \ + --mode stock --target-sha "$(git rev-parse HEAD)" \ + --gateway-only --output /tmp/ios-e2e-gateway.json +``` + +These diagnostics produce `native-build`/`built` or `gateway-probe`/`probe-passed` +proofs, respectively. Neither is release qualification. Gateway runtime preparation +continues to use the existing build owner's cache in every mode. The stock gate runs in **iOS Store Release** after native tool setup and before signing assets are accessed. It qualifies the checked-out `main` commit used for release @@ -191,10 +228,13 @@ it does not accept an alternate target SHA. CI callers must also use their own revision. Compare runs four serial matched pairs in stock/slim, slim/stock, stock/slim, -slim/stock order, with both tests fresh in every arm. SimSlim keeps the existing -conservative search/family-only profile. Neither failures nor skipped tests are -retried or dropped. JSON reports preparation, test, arm, build, and overall -durations; the workflow additionally records shared toolchain installation time. +slim/stock order, for eight independently prepared arms. SimSlim keeps the existing +conservative search/family-only profile. A preparation or live-test failure stops +that arm before the reader test; a reader failure also fails the arm. Neither +failures nor skipped tests are retried or dropped. Cleanup runs once per arm, +and unconfirmed cleanup stops the run. JSON retains each attempted test's outcome +and duration, plus preparation, arm, build, and overall durations; the workflow +also records shared toolchain installation time. Qualification tests disable Xcode's verbose failure diagnostics, such as sysdiagnose, while retaining ordinary XCTest output, result inspection, and sanitized proof. @@ -207,7 +247,9 @@ seconds fail measurement. A stock gate without the meter requires no measurement Raw XCTest bundles and fixture logs stay private and are cleaned with owned resources. If owned cleanup cannot be confirmed, the working root is retained. Only sanitized JSON proof is uploaded, including on failure, with fixed operation -labels and bounded exit/error diagnostics rather than raw logs or setup codes. +labels, phase durations, setup RPC progress, and bounded exit/error diagnostics. +Raw logs and setup codes are excluded. Setup-code timeouts are preparation failures +and prevent native test execution. ## GitHub Actions diff --git a/docs/ci/runners.md b/docs/ci/runners.md index 66eff3bfcd88..c478b33e9300 100644 --- a/docs/ci/runners.md +++ b/docs/ci/runners.md @@ -47,7 +47,7 @@ Healthy eligible main pushes can additionally offload five check rows under the Native Swift builds/tests, iOS build phases, screenshot shards, and Periphery scans use Xcode 27 on GitHub-hosted `xcode-27`, the preview macOS 27 image. This toolchain change preserves hosted placement, job counts, worker caps, coverage, and deadlines; it adds no Blacksmith registrations. The Swift source-language minimum remains 6.3. Native compatibility and complete job timings require proof on the new image. -The reusable iOS release qualification job uses `xcode-27-xlarge` on the same preview image to give its Simulator and isolated Gateway more CPU and memory. Stock qualification keeps the simulator unmodified. Its two fresh test fixtures, assertions, deadlines, and job count remain unchanged. +The reusable iOS release qualification job uses `xcode-27-xlarge` on the same preview image to give its Simulator and isolated Gateway more CPU and memory. Stock qualification keeps the simulator unmodified. After the native build, each arm starts one isolated Gateway and prepares its setup handler through `device.pair.setupStatus` before booting a fresh simulator. The status call prunes expired completion records without issuing a credential; the setup code consumed by the app is issued after boot. The same Gateway serves the live pairing, chat, and relaunch case, then stops before the independent transcript-reader case on the same simulator. Both cases must pass within their individual deadlines. The earlier hosted-routing validation used `macos-26`: repeated first attempts left the Blacksmith macOS jobs unassigned while other CI completed. In [run 33616182173](https://github.com/openclaw/openclaw/actions/runs/33616182173), the hosted retry assigned all three waiting Mac jobs within eight seconds; the Debug/simulator job passed in 15m31s. That historical result predates Xcode 27. Complete native evidence remains required for full manual qualification. diff --git a/scripts/ios-release-e2e.ts b/scripts/ios-release-e2e.ts index d65475bb560d..8ebd845b8211 100644 --- a/scripts/ios-release-e2e.ts +++ b/scripts/ios-release-e2e.ts @@ -1,4 +1,4 @@ -import { mkdir, writeFile } from "node:fs/promises"; +import { mkdir, rename, writeFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { parseArgs } from "node:util"; @@ -8,19 +8,19 @@ import { } from "@openclaw/normalization-core/error-coercion"; export const IOS_RELEASE_TESTS = [ - "OpenClawUITests/OpenClawSnapshotUITests/testLiveGatewayFreshInstallSetupAndRelaunch", - "OpenClawUITests/OpenClawSnapshotUITests/testLiveGatewayChatRoundTripAndControlOverview", + "OpenClawUITests/OpenClawSnapshotUITests/testLiveGatewayPairChatAndRelaunch", + "OpenClawUITests/OpenClawSnapshotUITests/testKeyboardOpenPreservesTranscriptAndFollowsLiveEdgeAfterSend", ] as const; export const MODEL_REF = "openai/ios-e2e"; export const IOS_RELEASE_CHAT_FAILURE = - /IOS_RELEASE_CHAT_FAILURE (seed-[0-2]|final) (submission|reply) draft=(true|false) keyboard=(true|false) reply=(true|false) writing=(true|false) jump=(true|false) foreground=(true|false) input=(true|false) transcript=(true|false) send=(true|false)/u; + /IOS_RELEASE_CHAT_FAILURE (first|second|relaunch) (submission|reply) draft=(true|false) keyboard=(true|false) reply=(true|false) writing=(true|false) jump=(true|false) foreground=(true|false) input=(true|false) transcript=(true|false) send=(true|false)/u; export const IOS_RELEASE_TEST_FAILURE_LOCATION = /(?:^|\/)OpenClawSnapshotUITests\.swift:([1-9][0-9]{0,4})(?::[0-9]+)?: error:/gmu; export const SAMPLE_INTERVAL_MS = 1_000; export const MAX_SAMPLE_GAP_MS = 3_000; export type Mode = "stock" | "compare"; export type Arm = "stock" | "simslim"; -type TestIdentity = (typeof IOS_RELEASE_TESTS)[number]; +export type TestIdentity = (typeof IOS_RELEASE_TESTS)[number]; type JsonObject = Record; export type Operation = @@ -32,6 +32,7 @@ export type Operation = | "gateway-build" | "native-generate" | "native-build" + | "native-build-reuse" | "simulator-create" | "simulator-boot" | "simulator-ready" @@ -39,6 +40,7 @@ export type Operation = | "simulator-delete" | "fixture-server" | "gateway-start" + | "setup-status" | "setup-code" | "native-test" | "app-diagnostics" @@ -292,14 +294,14 @@ export function testRunnerEnv(setupCode: string): NodeJS.ProcessEnv { export type TrialResources = { prepare: () => Promise; - test: () => Promise; + test: (test: TestIdentity) => Promise; measure: () => Promise; cleanup: () => Promise; }; export type Trial = { pair: number; arm: Arm; - test: TestIdentity; + tests: { test: TestIdentity; status: "failed" | "passed"; ms: number }[]; status: "failed" | "passed"; errors: string[]; diagnostics: Diagnostic[]; @@ -309,7 +311,7 @@ export type Trial = { measurement?: ReturnType; }; export type TrialDependencies = { - create: (test: TestIdentity, arm: Arm, index: number) => Promise; + create: (arm: Arm, index: number) => Promise; now: () => number; wait: (ms: number, signal: AbortSignal) => Promise; signal: AbortSignal; @@ -324,124 +326,133 @@ export async function runTrials(mode: Mode, deps: TrialDependencies) { const arms: { pair: number; arm: Arm; totalMs: number }[] = []; for (const planned of armPlan(mode)) { const armStarted = deps.now(); - for (const test of IOS_RELEASE_TESTS) { - if (deps.signal.aborted) { - return { trials, arms, complete: false }; - } - const started = deps.now(); - const trial: Trial = { - ...planned, - test, - status: "failed", - errors: [], - diagnostics: [], - preparationMs: 0, - testMs: 0, - totalMs: 0, - }; - trials.push(trial); - let resources: TrialResources | undefined; - let testStarted: number | undefined; - let collector: Promise | undefined; - const stopCollection = new AbortController(); - const samples: Sample[] = []; - let measurementErrors = 0; - let cleanupFailed = false; - let stage = "preparation"; - try { - resources = await deps.create(test, planned.arm, trials.length); - await resources.prepare(); - trial.preparationMs = deps.now() - started; - deps.signal.throwIfAborted(); - stage = "test"; - testStarted = deps.now(); - if (deps.measure) { - const measure = async () => { + if (deps.signal.aborted) { + return { trials, arms, complete: false }; + } + const started = deps.now(); + const trial: Trial = { + ...planned, + tests: [], + status: "failed", + errors: [], + diagnostics: [], + preparationMs: 0, + testMs: 0, + totalMs: 0, + }; + trials.push(trial); + let resources: TrialResources | undefined; + let testStarted: number | undefined; + let collector: Promise | undefined; + const stopCollection = new AbortController(); + const samples: Sample[] = []; + let measurementErrors = 0; + let cleanupFailed = false; + let stage = "preparation"; + try { + resources = await deps.create(planned.arm, trials.length); + await resources.prepare(); + trial.preparationMs = deps.now() - started; + deps.signal.throwIfAborted(); + stage = "test"; + testStarted = deps.now(); + if (deps.measure) { + const measure = async () => { + try { + const measurement = parseMeasurement(await resources!.measure()); + samples.push({ ...measurement, atMs: deps.now() - testStarted! }); + } catch (error) { + measurementErrors++; + if (error instanceof OperationError && trial.diagnostics.length < 8) { + trial.diagnostics.push(error.diagnostic); + } + } + }; + // Sample serially, only after preparation has completed. + await measure(); + collector = (async () => { + let next = testStarted! + SAMPLE_INTERVAL_MS; + while (!stopCollection.signal.aborted && !deps.signal.aborted) { try { - const measurement = parseMeasurement(await resources!.measure()); - samples.push({ ...measurement, atMs: deps.now() - testStarted! }); - } catch (error) { - measurementErrors++; - if (error instanceof OperationError && trial.diagnostics.length < 8) { - trial.diagnostics.push(error.diagnostic); + await deps.wait(Math.max(0, next - deps.now()), stopCollection.signal); + } catch { + if (!stopCollection.signal.aborted && !deps.signal.aborted) { + measurementErrors++; } + break; } - }; - // Sample serially, only after preparation has completed. - await measure(); - collector = (async () => { - let next = testStarted! + SAMPLE_INTERVAL_MS; - while (!stopCollection.signal.aborted && !deps.signal.aborted) { - try { - await deps.wait(Math.max(0, next - deps.now()), stopCollection.signal); - } catch { - if (!stopCollection.signal.aborted && !deps.signal.aborted) { - measurementErrors++; - } - break; - } - if (stopCollection.signal.aborted || deps.signal.aborted) { - break; - } - await measure(); - next += SAMPLE_INTERVAL_MS; + if (stopCollection.signal.aborted || deps.signal.aborted) { + break; } - })(); + await measure(); + next += SAMPLE_INTERVAL_MS; + } + })(); + } + for (const test of IOS_RELEASE_TESTS) { + const testResult: Trial["tests"][number] = { test, status: "failed", ms: 0 }; + trial.tests.push(testResult); + const caseStarted = deps.now(); + try { + deps.signal.throwIfAborted(); + requireExactTestResult(await resources.test(test), test); + testResult.status = "passed"; + } finally { + testResult.ms = deps.now() - caseStarted; } - requireExactTestResult(await resources.test(), test); + } + } catch (error) { + if (error instanceof OperationError) { + trial.diagnostics.push(error.diagnostic); + } + const resultErrors = [ + "test-skipped", + "test-identity-or-result", + "failed-test-child", + "repeated-test", + ]; + trial.errors.push( + deps.signal.aborted + ? "cancelled" + : (error as { code?: string })?.code === "ETIMEDOUT" || + (error instanceof OperationError && error.diagnostic.code === "timeout") + ? `${stage}-timeout` + : error instanceof Error && resultErrors.includes(error.message) + ? error.message + : `${stage}-failed`, + ); + } finally { + stopCollection.abort(); + await collector; + if (testStarted !== undefined) { + trial.testMs = deps.now() - testStarted; + if (deps.measure) { + trial.measurement = summarizeMeasurements(samples, measurementErrors, trial.testMs); + if (!trial.measurement.complete) { + trial.errors.push("incomplete-measurement"); + } + } + } else { + trial.preparationMs = deps.now() - started; + } + try { + await resources?.cleanup(); } catch (error) { if (error instanceof OperationError) { trial.diagnostics.push(error.diagnostic); } - const resultErrors = [ - "test-skipped", - "test-identity-or-result", - "failed-test-child", - "repeated-test", - ]; - trial.errors.push( - deps.signal.aborted - ? "cancelled" - : (error as { code?: string })?.code === "ETIMEDOUT" || - (error instanceof OperationError && error.diagnostic.code === "timeout") - ? `${stage}-timeout` - : error instanceof Error && resultErrors.includes(error.message) - ? error.message - : `${stage}-failed`, - ); - } finally { - stopCollection.abort(); - await collector; - if (testStarted !== undefined) { - trial.testMs = deps.now() - testStarted; - if (deps.measure) { - trial.measurement = summarizeMeasurements(samples, measurementErrors, trial.testMs); - if (!trial.measurement.complete) { - trial.errors.push("incomplete-measurement"); - } - } - } else { - trial.preparationMs = deps.now() - started; - } - try { - await resources?.cleanup(); - } catch (error) { - if (error instanceof OperationError) { - trial.diagnostics.push(error.diagnostic); - } - trial.errors.push("cleanup-failed"); - cleanupFailed = true; - } - trial.totalMs = deps.now() - started; - if (deps.signal.aborted && !trial.errors.includes("cancelled")) { - trial.errors.push("cancelled"); - } - trial.status = trial.errors.length === 0 ? "passed" : "failed"; + trial.errors.push("cleanup-failed"); + cleanupFailed = true; } - // Unconfirmed cleanup cannot safely share a host with another trial. - if (cleanupFailed) { - return { trials, arms, complete: false }; + trial.totalMs = deps.now() - started; + if (deps.signal.aborted && !trial.errors.includes("cancelled")) { + trial.errors.push("cancelled"); } + trial.status = trial.errors.length === 0 ? "passed" : "failed"; + } + // Unconfirmed cleanup cannot safely share a host with another trial. + if (cleanupFailed) { + return { trials, arms, complete: false }; } arms.push({ ...planned, totalMs: deps.now() - armStarted }); } @@ -454,12 +465,17 @@ async function main() { mode: { type: "string", default: "stock" }, "target-sha": { type: "string" }, output: { type: "string" }, + "build-dir": { type: "string" }, + "build-only": { type: "boolean", default: false }, + "gateway-only": { type: "boolean", default: false }, }, }); if ( (values.mode !== "stock" && values.mode !== "compare") || !/^[a-f0-9]{40}$/u.test(values["target-sha"] ?? "") || - !values.output + !values.output || + (values["build-only"] && !values["build-dir"]) || + (values["gateway-only"] && (values["build-only"] || values.mode !== "stock")) ) { throw new Error("usage: --mode stock|compare --target-sha --output "); } @@ -469,10 +485,15 @@ async function main() { process.on("SIGINT", cancel); process.on("SIGTERM", cancel); const proof: Record = { - schema: 1, + schema: 2, targetSha: values["target-sha"], harnessSha: null, mode: values.mode, + kind: values["gateway-only"] + ? "gateway-probe" + : values["build-only"] + ? "native-build" + : "qualification", model: MODEL_REF, status: "failed", trials: [], @@ -480,6 +501,17 @@ async function main() { diagnostics: [], }; let cleanup: (() => Promise) | undefined; + await mkdir(path.dirname(values.output), { recursive: true }); + let pendingWrite = Promise.resolve(); + const writeProof = () => { + const bytes = `${JSON.stringify(proof, null, 2)}\n`; + pendingWrite = pendingWrite.then(async () => { + const temporary = `${values.output}.tmp`; + await writeFile(temporary, bytes, { mode: 0o600 }); + await rename(temporary, values.output!); + }); + return pendingWrite; + }; try { const { createNativeDependencies } = await import("./lib/ios-release-e2e-native.js"); const native = await createNativeDependencies({ @@ -487,15 +519,34 @@ async function main() { targetSha: values["target-sha"]!, signal: abort.signal, proof, + buildDir: values["build-dir"], + gatewayOnly: values["gateway-only"], + onProgress: writeProof, }); cleanup = native.cleanup; - const result = await runTrials(values.mode, native.dependencies); - Object.assign(proof, result); - proof.status = - result.complete && result.trials.every((trial) => trial.status === "passed") - ? "passed" - : "failed"; + let status: string; + if (values["build-only"]) { + status = "built"; + } else if (values["gateway-only"]) { + const fixture = await native.dependencies.create("stock", 1); + try { + await fixture.prepare(); + status = "probe-passed"; + } finally { + await fixture.cleanup(); + } + } else { + const result = await runTrials(values.mode, native.dependencies); + Object.assign(proof, result); + status = + result.complete && result.trials.every((trial) => trial.status === "passed") + ? "passed" + : "failed"; + } + await native.assertCurrentSource(); + proof.status = status; } catch (error) { + proof.status = "failed"; if (error instanceof OperationError) { (proof.diagnostics as Diagnostic[]).push(error.diagnostic); } @@ -512,12 +563,11 @@ async function main() { } proof.harnessMs = performance.now() - started; proof.overallMs = proof.harnessMs; - await mkdir(path.dirname(values.output), { recursive: true }); - await writeFile(values.output, `${JSON.stringify(proof, null, 2)}\n`, { mode: 0o600 }); + await writeProof(); process.removeListener("SIGINT", cancel); process.removeListener("SIGTERM", cancel); } - process.exitCode = proof.status === "passed" ? 0 : 1; + process.exitCode = ["passed", "built", "probe-passed"].includes(String(proof.status)) ? 0 : 1; } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { diff --git a/scripts/lib/ios-release-e2e-build.ts b/scripts/lib/ios-release-e2e-build.ts new file mode 100644 index 000000000000..8dd256c05bbf --- /dev/null +++ b/scripts/lib/ios-release-e2e-build.ts @@ -0,0 +1,160 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { + ARTIFACT_CACHE_VERSION, + acquireBuildArtifactLockAsync, + listCacheFiles, + portableRelativePath, + readArtifactRecord, + writeArtifactRecord, +} from "./build-artifact-cache.mts"; + +export type IOSReleaseNativeBuildIdentity = { + sourceSha: string; + checkoutPath: string; + xcodeVersion: string; + sdkVersion: string; + developerDir: string; + nodeVersion: string; + platform: string; + arch: string; + buildArgs: string[]; + generatorArgs: string[]; +}; + +const REQUIRED_PRODUCTS = [ + "Debug-iphonesimulator/OpenClaw.app/Info.plist", + "Debug-iphonesimulator/OpenClaw.app/OpenClaw", + "Debug-iphonesimulator/OpenClawUITests-Runner.app/Info.plist", + "Debug-iphonesimulator/OpenClawUITests-Runner.app/OpenClawUITests-Runner", + "Debug-iphonesimulator/OpenClawUITests-Runner.app/PlugIns/OpenClawUITests.xctest/Info.plist", + "Debug-iphonesimulator/OpenClawUITests-Runner.app/PlugIns/OpenClawUITests.xctest/OpenClawUITests", +]; + +function productInventory(derivedDataPath: string) { + const products = path.join(derivedDataPath, "Build/Products"); + for (const directory of [derivedDataPath, path.join(derivedDataPath, "Build"), products]) { + if (!fs.lstatSync(directory).isDirectory()) { + throw new Error("native-build-products-unavailable"); + } + } + const topology: string[][] = []; + // The shared byte inventory excludes symlinks. Xcode bundles also need their + // link targets, directory membership and executable modes bound to the receipt. + const visit = (directory: string) => { + for (const name of fs.readdirSync(directory).toSorted()) { + const file = path.join(directory, name); + const relative = portableRelativePath(products, file); + const stat = fs.lstatSync(file); + if (stat.isSymbolicLink()) { + const target = fs.realpathSync(file); + const resolved = path.relative(products, target); + if ( + resolved === ".." || + resolved.startsWith(`..${path.sep}`) || + path.isAbsolute(resolved) + ) { + throw new Error("native-build-external-product-link"); + } + topology.push([relative, "link", fs.readlinkSync(file)]); + } else if (stat.isDirectory()) { + topology.push([relative, "directory", String(stat.mode & 0o777)]); + visit(file); + } else if (stat.isFile()) { + topology.push([relative, "file", String(stat.mode & 0o777)]); + } else { + throw new Error("native-build-unsupported-product"); + } + } + }; + visit(products); + const outputs = Object.fromEntries( + listCacheFiles(products, ["."], fs).map((file) => [ + portableRelativePath(products, file), + createHash("sha256").update(fs.readFileSync(file)).digest("hex"), + ]), + ); + const testRuns = Object.keys(outputs).filter( + (file) => !file.includes("/") && file.endsWith(".xctestrun"), + ); + if (testRuns.length !== 1 || REQUIRED_PRODUCTS.some((file) => !Object.hasOwn(outputs, file))) { + throw new Error("native-build-products-incomplete"); + } + return { outputs, topology, xctestrunPath: path.join(products, testRuns[0]!) }; +} + +/** Caller owns source/toolchain admission, build commands, and retained-directory cleanup. */ +export async function prepareIOSReleaseNativeBuild(options: { + buildDir: string; + identity: IOSReleaseNativeBuildIdentity; + assertCurrentSource: () => Promise; + build: (derivedDataPath: string) => Promise; +}): Promise<{ derivedDataPath: string; xctestrunPath: string; reused: boolean }> { + const { identity } = options; + if (!/^[a-f0-9]{40}$/u.test(identity.sourceSha)) { + throw new Error("native-build-invalid-source"); + } + fs.mkdirSync(options.buildDir, { recursive: true, mode: 0o700 }); + const buildDir = fs.realpathSync(options.buildDir); + const derivedDataPath = path.join(buildDir, "DerivedData"); + const receiptPath = path.join(buildDir, "native-build.json"); + // Keep the transient lock outside the admitted empty directory. The lock owner + // joins the entire build callback before another invocation can inspect it. + const lock = await acquireBuildArtifactLockAsync(`${buildDir}.native-build`); + try { + await options.assertCurrentSource(); + const entries = fs.readdirSync(buildDir); + const reused = entries.includes("native-build.json"); + if (!reused) { + if (entries.length !== 0) { + throw new Error("native-build-incomplete-directory"); + } + await options.build(derivedDataPath); + } else if (!fs.lstatSync(receiptPath).isFile()) { + throw new Error("native-build-invalid-receipt"); + } + await options.assertCurrentSource(); + const inventory = productInventory(derivedDataPath); + const signature = createHash("sha256") + .update( + JSON.stringify([ + "ios-release-native-build-v1", + buildDir, + fs.realpathSync(identity.checkoutPath), + identity.sourceSha, + identity.xcodeVersion, + identity.sdkVersion, + identity.developerDir, + identity.nodeVersion, + identity.platform, + identity.arch, + identity.buildArgs, + identity.generatorArgs, + inventory.topology, + ]), + ) + .digest("hex"); + if (reused) { + const record = readArtifactRecord(receiptPath); + if ( + !record || + record.signature !== signature || + Object.keys(record.outputs).length !== Object.keys(inventory.outputs).length || + Object.entries(inventory.outputs).some(([file, digest]) => record.outputs[file] !== digest) + ) { + throw new Error("native-build-receipt-mismatch"); + } + } else { + writeArtifactRecord(receiptPath, { + version: ARTIFACT_CACHE_VERSION, + signature, + outputs: inventory.outputs, + }); + fs.chmodSync(receiptPath, 0o600); + } + return { derivedDataPath, xctestrunPath: inventory.xctestrunPath, reused }; + } finally { + await lock.release(); + } +} diff --git a/scripts/lib/ios-release-e2e-native.ts b/scripts/lib/ios-release-e2e-native.ts index bc9a5dd81b46..ccbc00c591a6 100644 --- a/scripts/lib/ios-release-e2e-native.ts +++ b/scripts/lib/ios-release-e2e-native.ts @@ -1,9 +1,11 @@ -import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import { mkdtemp, readFile, realpath, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { isRecord } from "@openclaw/normalization-core/record-coerce"; import type { DevicePairSetupCodeResult } from "../../packages/gateway-protocol/src/schema/devices.js"; +import { stripInboundMetadata } from "../../src/auto-reply/reply/strip-inbound-meta.js"; import { isGatewayTransportError } from "../../src/gateway/transport-error.js"; import type { OpenClawTestInstance } from "../../test/helpers/openclaw-test-instance.js"; import { applyMockOpenAiModelConfig } from "../e2e/lib/fixtures/mock-openai-config.mjs"; @@ -25,10 +27,9 @@ import { hasUnjoinedWork, runManagedCommand } from "./managed-child-process.mjs" const DEVICE_TYPE = "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro"; const UUID = /^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/iu; const CHAT_MARKERS = [ - ["seed-0", "OPENCLAW_E2E_SEED_0_"], - ["seed-1", "OPENCLAW_E2E_SEED_1_"], - ["seed-2", "OPENCLAW_E2E_SEED_2_"], - ["final", "OPENCLAW_E2E_OK_"], + ["first", "OPENCLAW_E2E_FIRST"], + ["second", "OPENCLAW_E2E_SECOND"], + ["relaunch", "OPENCLAW_E2E_RELAUNCH"], ] as const; export async function createNativeDependencies(options: { @@ -36,7 +37,14 @@ export async function createNativeDependencies(options: { targetSha: string; signal: AbortSignal; proof: Record; -}): Promise<{ dependencies: TrialDependencies; cleanup: () => Promise }> { + buildDir?: string; + gatewayOnly?: boolean; + onProgress?: () => Promise; +}): Promise<{ + dependencies: TrialDependencies; + assertCurrentSource: () => Promise; + cleanup: () => Promise; +}> { if (process.platform !== "darwin" || process.arch !== "arm64") { throw new Error("macos-arm64-required"); } @@ -49,6 +57,42 @@ export async function createNativeDependencies(options: { retainRoot = true; options.proof.resourcesPreserved = true; }; + const phases: { + operation: Operation; + trial?: number; + status: "running" | "passed" | "failed"; + ms: number; + checkpoint?: string; + }[] = []; + options.proof.phases = phases; + let currentTrial: number | undefined; + const phase = async (operation: Operation, action: () => Promise): Promise => { + const entry: (typeof phases)[number] = { + operation, + ...(currentTrial ? { trial: currentTrial } : {}), + status: "running", + ms: 0, + }; + if (operation === "simulator-measure") { + return action(); + } + phases.push(entry); + await options.onProgress?.(); + const started = performance.now(); + console.error(`iOS qualification: ${operation} started`); + try { + const value = await action(); + entry.status = "passed"; + return value; + } catch (error) { + entry.status = "failed"; + throw error; + } finally { + entry.ms = Math.round(performance.now() - started); + console.error(`iOS qualification: ${operation} ${entry.status} (${entry.ms} ms)`); + await options.onProgress?.(); + } + }; const command = async ( operation: Operation, bin: string, @@ -58,77 +102,94 @@ export async function createNativeDependencies(options: { timeoutMs?: number; cleanup?: boolean; captureChatFailure?: () => Promise; + signal?: AbortSignal; } = {}, - ) => { - let stdout = ""; - let stderr = ""; - const started = performance.now(); - let failureContext: Promise | undefined; - const observeChatFailure = () => { - const capture = config.captureChatFailure; - if ( - !failureContext && - capture && - !`${stdout}\n${stderr}`.matchAll(IOS_RELEASE_TEST_FAILURE_LOCATION).next().done - ) { - const timing = `failure-evidence-at-ms:${Math.round(performance.now() - started)}`; - // Snapshot at the assertion, before XCTest's potentially lengthy teardown; always join below. - failureContext = Promise.resolve() - .then(capture) - .then( - (context) => [timing, ...context], - () => [timing, "chat-evidence-unavailable"], - ); + ) => + phase(operation, async () => { + let stdout = ""; + let stderr = ""; + const started = performance.now(); + let failureContext: Promise | undefined; + const observeChatFailure = () => { + const capture = config.captureChatFailure; + if ( + !failureContext && + capture && + !`${stdout}\n${stderr}`.matchAll(IOS_RELEASE_TEST_FAILURE_LOCATION).next().done + ) { + const timing = `failure-evidence-at-ms:${Math.round(performance.now() - started)}`; + // Snapshot at the assertion, before XCTest's potentially lengthy teardown; always join below. + failureContext = Promise.resolve() + .then(capture) + .then( + (context) => [timing, ...context], + () => [timing, "chat-evidence-unavailable"], + ); + } + }; + let code: number; + try { + code = await runManagedCommand({ + bin, + args, + cwd, + env: { ...process.env, ...config.env }, + stdio: ["ignore", "pipe", "pipe"], + timeoutMs: config.timeoutMs ?? 120_000, + requireProcessTreeExit: true, + signal: config.cleanup ? undefined : (config.signal ?? options.signal), + onReady(child) { + child.stdout?.on("data", (chunk: Buffer) => { + stdout = (stdout + chunk.toString()).slice(-16 * 1024 * 1024); + observeChatFailure(); + if (operation === "native-test") { + const checkpoint = [ + ...stdout.matchAll( + /IOS_RELEASE_CHECKPOINT (paired|first|second|relaunched|relaunch|overview)\b/gu, + ), + ].at(-1)?.[1]; + const activePhase = phases.findLast((entry) => entry.operation === "native-test"); + if (checkpoint && activePhase && activePhase.checkpoint !== checkpoint) { + activePhase.checkpoint = checkpoint; + void options.onProgress?.().catch(() => {}); + } + } + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr = (stderr + chunk.toString()).slice(-4096); + observeChatFailure(); + }); + }, + }); + } catch (error) { + if (hasUnjoinedWork(error)) { + preserveResources(); + } + const failure = operationError(operation, error, `${stderr}\n${stdout}`); + failure.diagnostic.context.push(...((await failureContext) ?? [])); + throw failure; } - }; - let code: number; - try { - code = await runManagedCommand({ - bin, - args, - cwd, - env: { ...process.env, ...config.env }, - stdio: ["ignore", "pipe", "pipe"], - timeoutMs: config.timeoutMs ?? 120_000, - requireProcessTreeExit: true, - signal: config.cleanup ? undefined : options.signal, - onReady(child) { - child.stdout?.on("data", (chunk: Buffer) => { - stdout = (stdout + chunk.toString()).slice(-16 * 1024 * 1024); - observeChatFailure(); - }); - child.stderr?.on("data", (chunk: Buffer) => { - stderr = (stderr + chunk.toString()).slice(-4096); - observeChatFailure(); - }); - }, - }); - } catch (error) { - if (hasUnjoinedWork(error)) { - preserveResources(); + const context = (await failureContext) ?? []; + if (code !== 0) { + const failure = new OperationError(operation, "exit", code, `${stderr}\n${stdout}`); + failure.diagnostic.context.push(...context); + throw failure; } - const failure = operationError(operation, error, `${stderr}\n${stdout}`); - failure.diagnostic.context.push(...((await failureContext) ?? [])); - throw failure; + return stdout.trim(); + }); + const assertCurrentSource = async () => { + const head = await command("source-head", "git", ["rev-parse", "HEAD"]); + if (head !== options.targetSha) { + throw new OperationError("source-head", "identity-mismatch"); } - const context = (await failureContext) ?? []; - if (code !== 0) { - const failure = new OperationError(operation, "exit", code, `${stderr}\n${stdout}`); - failure.diagnostic.context.push(...context); - throw failure; + if ( + await command("source-status", "git", ["status", "--porcelain=v1", "--untracked-files=all"]) + ) { + throw new OperationError("source-status", "dirty-source"); } - return stdout.trim(); }; - const head = await command("source-head", "git", ["rev-parse", "HEAD"]); - if (head !== options.targetSha) { - throw new OperationError("source-head", "identity-mismatch"); - } - if ( - await command("source-status", "git", ["status", "--porcelain=v1", "--untracked-files=all"]) - ) { - throw new OperationError("source-status", "dirty-source"); - } - options.proof.harnessSha = head; + await assertCurrentSource(); + options.proof.harnessSha = options.targetSha; const xcodeVersion = await command("xcode-version", "xcodebuild", ["-version"]); const xcode = /^Xcode ([0-9.]+)\r?\nBuild version ([A-Za-z0-9]+)$/u.exec(xcodeVersion); if (!xcode) { @@ -194,7 +255,6 @@ export async function createNativeDependencies(options: { ); options.proof.gatewayBuildMs = performance.now() - buildStarted; const nativeStarted = performance.now(); - await command("native-generate", "pnpm", ["ios:gen"]); const buildArgs = [ "-project", "apps/ios/OpenClaw.xcodeproj", @@ -202,8 +262,6 @@ export async function createNativeDependencies(options: { "OpenClawUITests", "-configuration", "Debug", - "-derivedDataPath", - path.join(root, "DerivedData"), // Simulator Keychain access needs entitlements, but no signing certificate. "CODE_SIGNING_ALLOWED=YES", "CODE_SIGN_IDENTITY=-", @@ -213,23 +271,63 @@ export async function createNativeDependencies(options: { "-parallel-testing-enabled", "NO", ]; - await command( - "native-build", - "xcodebuild", - [ - ...buildArgs, - "-destination", - "generic/platform=iOS Simulator", - ...IOS_RELEASE_TESTS.map((test) => `-only-testing:${test}`), - "build-for-testing", - ], - { timeoutMs: 1_800_000 }, - ); - options.proof.nativeBuildMs = performance.now() - nativeStarted; + let derivedDataPath = ""; + let xctestrunPath = ""; + if (!options.gatewayOnly) { + const { prepareIOSReleaseNativeBuild } = await import("./ios-release-e2e-build.js"); + const sdkVersion = await command("xcode-version", "xcrun", [ + "--sdk", + "iphonesimulator", + "--show-sdk-build-version", + ]); + const developerDir = await realpath( + await command("xcode-version", "xcode-select", ["--print-path"]), + ); + const build = await phase("native-build-reuse", () => + prepareIOSReleaseNativeBuild({ + buildDir: options.buildDir ?? path.join(root, "native-build"), + identity: { + sourceSha: options.targetSha, + checkoutPath: cwd, + xcodeVersion, + sdkVersion, + nodeVersion: process.version, + platform: process.platform, + arch: process.arch, + developerDir, + buildArgs, + generatorArgs: ["ios:gen"], + }, + assertCurrentSource, + build: async (destination) => { + await command("native-generate", "pnpm", ["ios:gen"]); + await command( + "native-build", + "xcodebuild", + [ + ...buildArgs, + "-derivedDataPath", + destination, + "-destination", + "generic/platform=iOS Simulator", + ...IOS_RELEASE_TESTS.map((test) => `-only-testing:${test}`), + "build-for-testing", + ], + { timeoutMs: 1_800_000 }, + ); + }, + }), + ); + derivedDataPath = build.derivedDataPath; + xctestrunPath = build.xctestrunPath; + options.proof.nativeBuildReused = build.reused; + options.proof.nativeBuildMs = performance.now() - nativeStarted; + } const { createOpenClawTestInstance } = await import("../../test/helpers/openclaw-test-instance.js"); const { callGateway } = await import("../../src/gateway/call.js"); return { + assertCurrentSource, cleanup, dependencies: { signal: options.signal, @@ -238,20 +336,68 @@ export async function createNativeDependencies(options: { await sleep(ms, undefined, { signal }); }, measure: Boolean(binary), - async create(test, arm, index) { + async create(arm, index) { + currentTrial = index; + const fixtureEvidence: Record = { trial: index }; + const fixtures = (options.proof.fixtures ??= []) as Record[]; + fixtures.push(fixtureEvidence); let udid: string | undefined; let instance: OpenClawTestInstance | undefined; + let setupCode = ""; const mockAbort = new AbortController(); let mockDone: Promise | undefined; let mockFailed = false; + let closingFixture = false; + let gatewayExited = false; + const fixtureAbort = new AbortController(); + const fixtureSignal = AbortSignal.any([options.signal, fixtureAbort.signal]); + const gatewayExit = () => { + if (!closingFixture) { + gatewayExited = true; + fixtureEvidence.gatewayExited = true; + fixtureAbort.abort(); + } + }; + const requireLiveFixture = () => { + if ( + gatewayExited || + (instance?.child && + (instance.child.exitCode !== null || instance.child.signalCode !== null)) + ) { + throw new OperationError( + "gateway-start", + "exit", + instance?.child?.exitCode ?? undefined, + ); + } + if (mockFailed) { + throw new OperationError("fixture-server", "failed"); + } + }; const requestLog = path.join(root, `requests-${index}.jsonl`); + let fixtureCleanup: Promise | undefined; + const releaseFixture = () => + (fixtureCleanup ??= (async () => { + closingFixture = true; + instance?.child?.off("exit", gatewayExit); + const results = await Promise.allSettled([ + instance?.cleanup(), + (async () => { + mockAbort.abort(); + await mockDone; + })(), + ]); + const failure = results.find((entry) => entry.status === "rejected"); + if (failure?.status === "rejected") { + preserveResources(); + fixtureEvidence.cleanupConfirmed = false; + throw new OperationError("cleanup", "cleanup-unconfirmed"); + } + fixtureEvidence.cleanupConfirmed = true; + })()); const release = async () => { const results = await Promise.allSettled([ - instance?.cleanup(), - (async () => { - mockAbort.abort(); - await mockDone; - })(), + releaseFixture(), udid ? command("simulator-delete", "xcrun", ["simctl", "delete", udid], { cleanup: true, @@ -271,18 +417,6 @@ export async function createNativeDependencies(options: { }; return { async prepare() { - udid = await command("simulator-create", "xcrun", [ - "simctl", - "create", - `openclaw-ios-e2e-${index}`, - DEVICE_TYPE, - runtime.identifier, - ]); - if (!UUID.test(udid)) { - udid = undefined; - preserveResources(); - throw new OperationError("simulator-create", "failed"); - } let resolvePort!: (port: number) => void; let rejectPort!: (error: Error) => void; const portReady = new Promise((resolve, reject) => { @@ -316,10 +450,16 @@ export async function createNativeDependencies(options: { }).then( () => { mockFailed = true; + if (!closingFixture) { + fixtureAbort.abort(); + } rejectPort(new Error("mock-exited")); }, (error: unknown) => { mockFailed = true; + if (!closingFixture) { + fixtureAbort.abort(); + } rejectPort(operationError("fixture-server", error)); if (hasUnjoinedWork(error)) { preserveResources(); @@ -356,249 +496,402 @@ export async function createNativeDependencies(options: { config, env: gatewayEnv, }); - await instance.startGateway(); + await phase("gateway-start", () => instance!.startGateway()); + instance.child?.once("exit", gatewayExit); + requireLiveFixture(); } catch (error) { if (hasUnjoinedWork(error)) { preserveResources(); } throw operationError("gateway-start", error); + } finally { + fixtureEvidence.readiness = instance?.readiness?.slice(-2).map((entry) => ({ + outcome: entry.outcome, + elapsedMs: entry.elapsedMs, + attempts: entry.attempts, + childExitCode: entry.child.exitCode, + probes: entry.probes + .slice(-4) + .map(({ phase: probePhase, elapsedMs, status, ready, error }) => ({ + phase: probePhase, + elapsedMs, + status, + ready, + error, + })), + })); } - if (arm === "simslim") { - await command( - "simulator-slim", - "/bin/bash", - ["scripts/ios-simulator-prepare.sh", udid], - { - env: { CI: "true", OPENCLAW_CI_SIMSLIM_BINARY: binary }, - timeoutMs: 900_000, - }, - ); - } else { - await command("simulator-boot", "xcrun", ["simctl", "boot", udid]); - await command("simulator-ready", "xcrun", ["simctl", "bootstatus", udid, "-b"], { - timeoutMs: 600_000, - }); - } - }, - async test() { - if (!instance || !udid) { - throw new Error("trial-not-prepared"); - } - let setupCode: string; + const readyInstance = instance; + const callSetupRpc = async ( + operation: "setup-status" | "setup-code", + params: Record, + ): Promise => { + const rpcEvidence = { + authenticated: false, + dispatchEntered: false, + responseReceived: false, + }; + fixtureEvidence[operation === "setup-status" ? "setupStatusRpc" : "setupRpc"] = + rpcEvidence; + try { + requireLiveFixture(); + const result = await phase(operation, () => + callGateway({ + config: {}, + configPath: readyInstance.configPath, + url: readyInstance.url, + token: readyInstance.gatewayToken, + ignoreEnvUrlOverride: true, + deviceIdentity: null, + sharedStateMode: "read-only", + method: + operation === "setup-status" + ? "device.pair.setupStatus" + : "device.pair.setupCode", + params, + timeoutMs: 30_000, + signal: fixtureSignal, + onHelloOk: () => { + rpcEvidence.authenticated = true; + }, + assertDispatchCurrent: () => { + requireLiveFixture(); + rpcEvidence.dispatchEntered = true; + }, + }), + ); + rpcEvidence.responseReceived = true; + requireLiveFixture(); + return result; + } catch (error) { + if (hasUnjoinedWork(error)) { + preserveResources(); + } + if (!options.signal.aborted) { + requireLiveFixture(); + } + if (isGatewayTransportError(error) && error.kind === "timeout") { + const failure = new OperationError(operation, "timeout"); + failure.diagnostic.context.push( + `rpc-authenticated:${rpcEvidence.authenticated}`, + `rpc-dispatch-entered:${rpcEvidence.dispatchEntered}`, + ); + if (typeof error.requestDispatched === "boolean") { + failure.diagnostic.context.push( + `rpc-request-dispatched:${error.requestDispatched}`, + ); + } + throw failure; + } + throw operationError(operation, error); + } + }; + // Prepare the real setup handler and worker before cold Simulator boot. + // Status prunes expired completion records but issues no credential. + await callSetupRpc("setup-status", { setupId: randomUUID() }); try { - // The ready Gateway owns credential issuance; avoid another CLI startup beside the simulator. - const setup = await callGateway({ - config: {}, - configPath: instance.configPath, - url: instance.url, - token: instance.gatewayToken, - ignoreEnvUrlOverride: true, - deviceIdentity: null, - sharedStateMode: "read-only", - method: "device.pair.setupCode", - params: { publicUrl: instance.url, includeQr: false }, - timeoutMs: 30_000, - signal: options.signal, - }); - setupCode = setup.setupCode; + if (!options.gatewayOnly) { + // Capture the created device before reacting to Gateway death; cleanup needs its identity. + udid = await command("simulator-create", "xcrun", [ + "simctl", + "create", + `openclaw-ios-e2e-${index}`, + DEVICE_TYPE, + runtime.identifier, + ]); + if (!UUID.test(udid)) { + udid = undefined; + preserveResources(); + throw new OperationError("simulator-create", "failed"); + } + requireLiveFixture(); + if (arm === "simslim") { + await command( + "simulator-slim", + "/bin/bash", + ["scripts/ios-simulator-prepare.sh", udid], + { + env: { CI: "true", OPENCLAW_CI_SIMSLIM_BINARY: binary }, + timeoutMs: 900_000, + signal: fixtureSignal, + }, + ); + } else { + await command("simulator-boot", "xcrun", ["simctl", "boot", udid], { + signal: fixtureSignal, + }); + await command( + "simulator-ready", + "xcrun", + ["simctl", "bootstatus", udid, "-b"], + { + timeoutMs: 600_000, + signal: fixtureSignal, + }, + ); + } + } } catch (error) { - if (isGatewayTransportError(error) && error.kind === "timeout") { - throw new OperationError("setup-code", "timeout"); + if (!options.signal.aborted) { + requireLiveFixture(); } - if (hasUnjoinedWork(error)) { - preserveResources(); - } - throw operationError("setup-code", error); + throw error; } + // Issue the consumed credential after boot so preparation cannot spend its TTL. + const setup = await callSetupRpc("setup-code", { + publicUrl: readyInstance.url, + includeQr: false, + }); + setupCode = setup.setupCode; if (!setupCode.trim()) { throw new OperationError("setup-code", "failed"); } - const resultBundle = path.join(root, `trial-${index}.xcresult`); - const fixture = instance; - await command( - "native-test", - "xcodebuild", - [ - ...buildArgs, - "-destination", - `platform=iOS Simulator,id=${udid}`, - "-resultBundlePath", - resultBundle, - "-collect-test-diagnostics", - "never", - `-only-testing:${test}`, - "test-without-building", - ], - { - env: testRunnerEnv(setupCode.trim()), - timeoutMs: 600_000, - captureChatFailure: - test === IOS_RELEASE_TESTS[1] - ? async () => { - const facts = new Set(); - const logs = fixture.logs(); - for (const stage of ["start", "first_event", "completed", "error"]) { - if (logs.includes(`[responses] ${stage} `)) { - facts.add(`model-any-request-stage:${stage}`); - } - } - const [requests, history, appLog] = await Promise.allSettled([ - readFile(requestLog, "utf8"), - callGateway({ - config: {}, - configPath: fixture.configPath, - url: fixture.url, - token: fixture.gatewayToken, - ignoreEnvUrlOverride: true, - deviceIdentity: null, - sharedStateMode: "read-only", - method: "chat.history", - params: { sessionKey: "main", limit: 20, maxBytes: 50_000 }, - timeoutMs: 5_000, - signal: options.signal, - }), - (async () => { - const bundleID = await command( - "app-diagnostics", - "/usr/bin/plutil", - [ - "-extract", - "CFBundleIdentifier", - "raw", - "-o", - "-", - path.join( - root, - "DerivedData/Build/Products/Debug-iphonesimulator/OpenClaw.app/Info.plist", - ), - ], - { timeoutMs: 5_000 }, - ); - const container = await command( - "app-diagnostics", - "xcrun", - ["simctl", "get_app_container", udid!, bundleID, "data"], - { timeoutMs: 5_000 }, - ); - return readFile( - path.join(container, "Library/Caches/openclaw-gateway.log"), - "utf8", - ); - })(), - ]); - if (appLog.status === "fulfilled") { - if ( - appLog.value.includes( - "] chat.send skipped before dispatch: route changed", - ) - ) { - facts.add("app-send-stage:dispatch-route-changed"); - } - for (const [event, stage] of [ - ["send invoked", "invoked"], - ["send ignored", "ignored"], - ["send queued offline", "offline-outbox"], - ["send routed behind outbox", "ordered-outbox"], - ["send queued sessionKey=", "optimistic-message"], - ["transport send start", "transport-start"], - ["transport send accepted", "transport-accepted"], - ["send delivery unconfirmed", "delivery-unconfirmed"], - ["send queued after route change", "route-changed"], - ["send failed", "failed"], - ]) { - if (appLog.value.includes(`] chat.ui ${event}`)) { - facts.add(`app-send-stage:${stage}`); - } - } - facts.add("app-evidence-read"); - } else { - facts.add("app-evidence-unavailable"); - } - try { - if (requests.status !== "fulfilled") { - throw new Error("request-log-unavailable"); - } - const lastMarker = (text: string) => - [...text.matchAll(/\bOPENCLAW_E2E_[A-Z0-9]+(?:_[A-Z0-9]+)*\b/gu)].at( - -1, - )?.[0]; - const markerStage = (marker: string | undefined) => - CHAT_MARKERS.find(([, prefix]) => marker?.startsWith(prefix))?.[0] ?? - "other"; - for (const line of requests.value.trim().split("\n").slice(-20)) { - const request: unknown = JSON.parse(line); - if ( - !isRecord(request) || - request.path !== "/v1/responses" || - typeof request.body !== "string" - ) { - continue; - } - const body: unknown = JSON.parse(request.body); - if (!isRecord(body) || body.model !== "ios-e2e") { - continue; - } - const input = Array.isArray(body.input) ? body.input : []; - const user = input - .map(readMockUserText) - .findLast((text) => text !== undefined); - const userMarker = lastMarker(user ?? ""); - const tailMarker = lastMarker(request.body); - facts.add(`provider-latest-user:${markerStage(userMarker)}`); - facts.add(`provider-body-tail:${markerStage(tailMarker)}`); - facts.add( - `provider-marker-match:${userMarker !== undefined && userMarker === tailMarker}`, - ); - } - facts.add("provider-evidence-read"); - } catch { - facts.add("provider-evidence-unavailable"); - } - if ( - history.status === "fulfilled" && - isRecord(history.value) && - Array.isArray(history.value.messages) - ) { - for (const message of history.value.messages) { - if ( - !isRecord(message) || - (message.role !== "user" && message.role !== "assistant") - ) { - continue; - } - const content = JSON.stringify(message.content) ?? ""; - for (const [stage, marker] of CHAT_MARKERS) { - if (content.includes(marker)) { - facts.add(`history-${message.role}:${stage}`); - } - } - } - facts.add("history-evidence-read"); - } else { - facts.add("history-evidence-unavailable"); - } - return [...facts]; - } - : undefined, - }, - ); - if (mockFailed) { - throw new OperationError("fixture-server", "failed"); + }, + async test(test) { + if (!instance || !udid || !setupCode) { + throw new Error("trial-not-prepared"); } if (test === IOS_RELEASE_TESTS[1]) { - const requests = (await readFile(requestLog, "utf8")) - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - if ( - !requests.some( - (request) => - request.path === "/v1/responses" && - JSON.parse(request.body).model === "ios-e2e", - ) - ) { - throw new OperationError("provider-rpc", "failed"); + requireLiveFixture(); + await phase("cleanup", releaseFixture); + } + const resultBundle = path.join( + root, + `trial-${index}-${IOS_RELEASE_TESTS.indexOf(test)}.xcresult`, + ); + const fixture = instance; + if (test === IOS_RELEASE_TESTS[0]) { + requireLiveFixture(); + } + try { + await command( + "native-test", + "xcodebuild", + [ + "-xctestrun", + xctestrunPath, + "-parallel-testing-enabled", + "NO", + "-destination", + `platform=iOS Simulator,id=${udid}`, + "-resultBundlePath", + resultBundle, + "-collect-test-diagnostics", + "never", + `-only-testing:${test}`, + "test-without-building", + ], + { + env: + test === IOS_RELEASE_TESTS[0] + ? testRunnerEnv(setupCode.trim()) + : { + TEST_RUNNER_OPENCLAW_IOS_LIVE_GATEWAY: "0", + TEST_RUNNER_OPENCLAW_IOS_LIVE_SETUP_CODE: "", + }, + timeoutMs: 600_000, + signal: test === IOS_RELEASE_TESTS[0] ? fixtureSignal : options.signal, + captureChatFailure: + test === IOS_RELEASE_TESTS[0] + ? async () => { + const facts = new Set(); + const logs = fixture.logs(); + for (const stage of ["start", "first_event", "completed", "error"]) { + if (logs.includes(`[responses] ${stage} `)) { + facts.add(`model-any-request-stage:${stage}`); + } + } + const [requests, history, appLog] = await Promise.allSettled([ + readFile(requestLog, "utf8"), + callGateway({ + config: {}, + configPath: fixture.configPath, + url: fixture.url, + token: fixture.gatewayToken, + ignoreEnvUrlOverride: true, + deviceIdentity: null, + sharedStateMode: "read-only", + method: "chat.history", + params: { sessionKey: "main", limit: 20, maxBytes: 50_000 }, + timeoutMs: 5_000, + signal: options.signal, + }), + (async () => { + const bundleID = await command( + "app-diagnostics", + "/usr/bin/plutil", + [ + "-extract", + "CFBundleIdentifier", + "raw", + "-o", + "-", + path.join( + derivedDataPath, + "Build/Products/Debug-iphonesimulator/OpenClaw.app/Info.plist", + ), + ], + { timeoutMs: 5_000 }, + ); + const container = await command( + "app-diagnostics", + "xcrun", + ["simctl", "get_app_container", udid!, bundleID, "data"], + { timeoutMs: 5_000 }, + ); + return readFile( + path.join(container, "Library/Caches/openclaw-gateway.log"), + "utf8", + ); + })(), + ]); + if (appLog.status === "fulfilled") { + if ( + appLog.value.includes( + "] chat.send skipped before dispatch: route changed", + ) + ) { + facts.add("app-send-stage:dispatch-route-changed"); + } + for (const [event, stage] of [ + ["send invoked", "invoked"], + ["send ignored", "ignored"], + ["send queued offline", "offline-outbox"], + ["send routed behind outbox", "ordered-outbox"], + ["send queued sessionKey=", "optimistic-message"], + ["transport send start", "transport-start"], + ["transport send accepted", "transport-accepted"], + ["send delivery unconfirmed", "delivery-unconfirmed"], + ["send queued after route change", "route-changed"], + ["send failed", "failed"], + ]) { + if (appLog.value.includes(`] chat.ui ${event}`)) { + facts.add(`app-send-stage:${stage}`); + } + } + facts.add("app-evidence-read"); + } else { + facts.add("app-evidence-unavailable"); + } + try { + if (requests.status !== "fulfilled") { + throw new Error("request-log-unavailable"); + } + const lastMarker = (text: string) => + [ + ...text.matchAll(/\bOPENCLAW_E2E_[A-Z0-9]+(?:_[A-Z0-9]+)*\b/gu), + ].at(-1)?.[0]; + const markerStage = (marker: string | undefined) => + CHAT_MARKERS.find(([, prefix]) => + marker?.startsWith(prefix), + )?.[0] ?? "other"; + for (const line of requests.value.trim().split("\n").slice(-20)) { + const request: unknown = JSON.parse(line); + if ( + !isRecord(request) || + request.path !== "/v1/responses" || + typeof request.body !== "string" + ) { + continue; + } + const body: unknown = JSON.parse(request.body); + if (!isRecord(body) || body.model !== "ios-e2e") { + continue; + } + const input = Array.isArray(body.input) ? body.input : []; + const user = input + .map(readMockUserText) + .findLast((text) => text !== undefined); + const userMarker = lastMarker(user ?? ""); + const tailMarker = lastMarker(request.body); + facts.add(`provider-latest-user:${markerStage(userMarker)}`); + facts.add(`provider-body-tail:${markerStage(tailMarker)}`); + facts.add( + `provider-marker-match:${userMarker !== undefined && userMarker === tailMarker}`, + ); + } + facts.add("provider-evidence-read"); + } catch { + facts.add("provider-evidence-unavailable"); + } + if ( + history.status === "fulfilled" && + isRecord(history.value) && + Array.isArray(history.value.messages) + ) { + for (const message of history.value.messages) { + if ( + !isRecord(message) || + (message.role !== "user" && message.role !== "assistant") + ) { + continue; + } + const content = JSON.stringify(message.content) ?? ""; + for (const [stage, marker] of CHAT_MARKERS) { + if (content.includes(marker)) { + facts.add(`history-${message.role}:${stage}`); + } + } + } + facts.add("history-evidence-read"); + } else { + facts.add("history-evidence-unavailable"); + } + return [...facts]; + } + : undefined, + }, + ); + } catch (error) { + if (test === IOS_RELEASE_TESTS[0]) { + requireLiveFixture(); } + throw error; + } + if (test === IOS_RELEASE_TESTS[0]) { + requireLiveFixture(); + } + if (test === IOS_RELEASE_TESTS[0]) { + await phase("provider-rpc", async () => { + const requests = (await readFile(requestLog, "utf8")).trim().split("\n"); + const received: (string | undefined)[] = []; + for (const line of requests) { + const request: unknown = JSON.parse(line); + if ( + !isRecord(request) || + request.path !== "/v1/responses" || + typeof request.body !== "string" + ) { + continue; + } + const body: unknown = JSON.parse(request.body); + if (!isRecord(body) || body.model !== "ios-e2e") { + continue; + } + const latestUser = Array.isArray(body.input) + ? body.input.map(readMockUserText).findLast((text) => text !== undefined) + : undefined; + const userText = + latestUser === undefined ? undefined : stripInboundMetadata(latestUser); + received.push( + CHAT_MARKERS.find( + ([, marker]) => + userText === `Reply exactly with ${marker} and no other text.`, + )?.[0], + ); + } + fixtureEvidence.providerRequestCount = received.length; + fixtureEvidence.providerMessages = CHAT_MARKERS.map(([stage]) => ({ + stage, + received: received.includes(stage), + })); + if ( + received.length !== CHAT_MARKERS.length || + CHAT_MARKERS.some(([stage], requestIndex) => received[requestIndex] !== stage) + ) { + throw new OperationError("provider-rpc", "failed"); + } + }); } return JSON.parse( await command("test-results", "xcrun", [ diff --git a/src/gateway/transport-error.test.ts b/src/gateway/transport-error.test.ts index 6b86f6f60c07..f99e7df9bf96 100644 --- a/src/gateway/transport-error.test.ts +++ b/src/gateway/transport-error.test.ts @@ -1,7 +1,12 @@ import { describe, expect, it } from "vitest"; import { GatewayProtocolRequestTimeoutError } from "../../packages/gateway-client/src/protocol-request.js"; import { GatewayClientRequestError } from "../../packages/gateway-client/src/request-error.js"; -import { GatewayTransportError, isGatewayRpcUnavailableError } from "./transport-error.js"; +import { + createGatewayCloseTransportError, + createGatewayTimeoutTransportError, + GatewayTransportError, + isGatewayRpcUnavailableError, +} from "./transport-error.js"; const connectionDetails = { url: "ws://127.0.0.1:18789", @@ -10,6 +15,28 @@ const connectionDetails = { }; describe("Gateway RPC transport availability", () => { + it.each([false, true])( + "retains requestDispatched=%s across transport failures", + (requestDispatched) => { + const errors = [ + createGatewayCloseTransportError({ + code: 1006, + reason: "connection interrupted", + connectionDetails, + requestDispatched, + }), + createGatewayTimeoutTransportError({ + timeoutMs: 1_500, + connectionDetails, + requestDispatched, + }), + ]; + for (const error of errors) { + expect(error).toMatchObject({ requestDispatched }); + } + }, + ); + it.each([ { label: "typed connection close", diff --git a/src/gateway/transport-error.ts b/src/gateway/transport-error.ts index 7133b7b60664..2c2de7d87e47 100644 --- a/src/gateway/transport-error.ts +++ b/src/gateway/transport-error.ts @@ -9,6 +9,7 @@ export class GatewayTransportError extends Error { readonly code?: number; readonly reason?: string; readonly timeoutMs?: number; + readonly requestDispatched?: boolean; constructor(params: { kind: GatewayTransportErrorKind; @@ -17,6 +18,7 @@ export class GatewayTransportError extends Error { code?: number; reason?: string; timeoutMs?: number; + requestDispatched?: boolean; }) { super(params.message); this.name = "GatewayTransportError"; @@ -31,6 +33,9 @@ export class GatewayTransportError extends Error { if (params.timeoutMs !== undefined) { this.timeoutMs = params.timeoutMs; } + if (params.requestDispatched !== undefined) { + this.requestDispatched = params.requestDispatched; + } } } @@ -85,6 +90,7 @@ export function createGatewayCloseTransportError(params: { code, reason, connectionDetails, + requestDispatched, message: requestDispatched ? `${message}\n\n${DISPATCHED_REQUEST_OUTCOME_GUIDANCE}` : message, }); } @@ -100,6 +106,7 @@ export function createGatewayTimeoutTransportError(params: { kind: "timeout", timeoutMs, connectionDetails, + requestDispatched, message: requestDispatched ? `${message}\n\n${DISPATCHED_REQUEST_OUTCOME_GUIDANCE}` : message, }); } diff --git a/test/scripts/ios-release-e2e-build.test.ts b/test/scripts/ios-release-e2e-build.test.ts new file mode 100644 index 000000000000..da277e744881 --- /dev/null +++ b/test/scripts/ios-release-e2e-build.test.ts @@ -0,0 +1,149 @@ +import fs from "node:fs"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + prepareIOSReleaseNativeBuild, + type IOSReleaseNativeBuildIdentity, +} from "../../scripts/lib/ios-release-e2e-build.js"; +import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js"; + +const roots = useAutoCleanupTempDirTracker(afterEach); + +function fixture() { + const root = roots.make("ios-native-build-"); + const identity: IOSReleaseNativeBuildIdentity = { + sourceSha: "a".repeat(40), + checkoutPath: root, + xcodeVersion: "Xcode 27.0\nBuild version 18A1", + sdkVersion: "24A1", + developerDir: path.join(root, "Xcode"), + nodeVersion: process.version, + platform: "darwin", + arch: "arm64", + buildArgs: ["-scheme", "OpenClawUITests", "build-for-testing"], + generatorArgs: ["ios:gen"], + }; + const write = (relative: string, text = "fixture bytes") => { + const file = path.join(root, "build/DerivedData/Build/Products", relative); + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, text); + return file; + }; + const build = vi.fn(async () => { + write("OpenClawUITests_iphonesimulator.xctestrun", "fixture test configuration"); + for (const bundle of [ + "OpenClaw.app", + "OpenClawUITests-Runner.app", + "OpenClawUITests-Runner.app/PlugIns/OpenClawUITests.xctest", + ]) { + write(`Debug-iphonesimulator/${bundle}/Info.plist`); + const executable = path.basename(bundle).replace(/\.(app|xctest)$/u, ""); + fs.chmodSync(write(`Debug-iphonesimulator/${bundle}/${executable}`), 0o755); + } + const framework = "Debug-iphonesimulator/OpenClaw.app/Frameworks/Fixture.framework"; + const library = write(`${framework}/Versions/A/Fixture`); + fs.symlinkSync("A", path.join(path.dirname(path.dirname(library)), "Current")); + }); + const options = { + buildDir: path.join(root, "build"), + identity, + assertCurrentSource: vi.fn(async () => {}), + build, + }; + return { root, options, write }; +} + +describe("retained iOS qualification native build", () => { + it("reuses complete exact-candidate products without invoking the builder again", async () => { + const { options } = fixture(); + const first = await prepareIOSReleaseNativeBuild(options); + const second = await prepareIOSReleaseNativeBuild(options); + expect(first.reused).toBe(false); + expect(second).toEqual({ ...first, reused: true }); + expect(fs.existsSync(second.xctestrunPath)).toBe(true); + expect(options.build).toHaveBeenCalledTimes(1); + }); + + it.each([ + ["sourceSha", "b".repeat(40)], + ["xcodeVersion", "Xcode 27.1\nBuild version 18B1"], + ["sdkVersion", "24B1"], + ["developerDir", "/different/Xcode"], + ["nodeVersion", "v99.0.0"], + ["platform", "linux"], + ["arch", "x64"], + ["buildArgs", ["-scheme", "AnotherTests", "build-for-testing"]], + ["generatorArgs", ["other:gen"]], + ] as const)("refuses a changed %s instead of rebuilding over the receipt", async (key, value) => { + const { options } = fixture(); + await prepareIOSReleaseNativeBuild(options); + const identity = { ...options.identity, [key]: typeof value === "string" ? value : [...value] }; + await expect(prepareIOSReleaseNativeBuild({ ...options, identity })).rejects.toThrow( + "native-build-receipt-mismatch", + ); + expect(options.build).toHaveBeenCalledTimes(1); + }); + + it.each(["missing", "changed", "extra", "mode", "link"] as const)( + "refuses %s runnable artifacts without repair or replacement", + async (change) => { + const { options, write } = fixture(); + await prepareIOSReleaseNativeBuild(options); + const executable = path.join( + options.buildDir, + "DerivedData/Build/Products/Debug-iphonesimulator/OpenClaw.app/OpenClaw", + ); + if (change === "missing") { + fs.rmSync(executable); + } else if (change === "changed") { + fs.writeFileSync(executable, "changed app"); + } else if (change === "extra") { + write("Debug-iphonesimulator/OpenClaw.app/unrecorded.dylib"); + } else if (change === "mode") { + fs.chmodSync(executable, 0o644); + } else { + fs.rmSync(executable); + fs.symlinkSync("Info.plist", executable); + } + await expect(prepareIOSReleaseNativeBuild(options)).rejects.toThrow(/native-build-/u); + expect(options.build).toHaveBeenCalledTimes(1); + }, + ); + + it.each(["checkout", "build"] as const)("refuses a relocated physical %s", async (owner) => { + const { root, options } = fixture(); + await prepareIOSReleaseNativeBuild(options); + const moved = path.join(root, "moved"); + if (owner === "checkout") { + fs.mkdirSync(moved); + options.identity.checkoutPath = moved; + } else { + fs.renameSync(options.buildDir, moved); + options.buildDir = moved; + } + await expect(prepareIOSReleaseNativeBuild(options)).rejects.toThrow( + "native-build-receipt-mismatch", + ); + expect(options.build).toHaveBeenCalledTimes(1); + }); + + it("does not admit a failed build or overwrite its incomplete directory", async () => { + const { options, write } = fixture(); + options.build.mockImplementation(async () => { + write("partial-output"); + throw new Error("build failed"); + }); + await expect(prepareIOSReleaseNativeBuild(options)).rejects.toThrow("build failed"); + await expect(prepareIOSReleaseNativeBuild(options)).rejects.toThrow( + "native-build-incomplete-directory", + ); + expect(options.build).toHaveBeenCalledTimes(1); + }); + + it("refuses a receipt when source admission changes during the build", async () => { + const { options } = fixture(); + options.assertCurrentSource.mockResolvedValueOnce().mockRejectedValueOnce(new Error("dirty")); + await expect(prepareIOSReleaseNativeBuild(options)).rejects.toThrow("dirty"); + expect(fs.existsSync(path.join(options.buildDir, "native-build.json"))).toBe(false); + }); +}); diff --git a/test/scripts/ios-release-e2e.test.ts b/test/scripts/ios-release-e2e.test.ts index 40703f3180da..3e853323a288 100644 --- a/test/scripts/ios-release-e2e.test.ts +++ b/test/scripts/ios-release-e2e.test.ts @@ -1,4 +1,5 @@ import { spawnSync, type ChildProcess } from "node:child_process"; +import { EventEmitter } from "node:events"; import { mkdirSync, readFileSync, readdirSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -15,6 +16,7 @@ import { runTrials, summarizeMeasurements, testRunnerEnv, + type TestIdentity, type TrialDependencies, } from "../../scripts/ios-release-e2e.js"; import { createNativeDependencies } from "../../scripts/lib/ios-release-e2e-native.js"; @@ -24,6 +26,7 @@ import { evaluateWorkflowExpression } from "./ci-workflow.test-support.js"; const nativeMocks = vi.hoisted(() => ({ command: vi.fn(), + build: vi.fn(), gateway: vi.fn(), rpc: vi.fn(), })); @@ -31,6 +34,9 @@ vi.mock("../../scripts/lib/managed-child-process.mjs", async (importOriginal) => ...(await importOriginal()), runManagedCommand: nativeMocks.command, })); +vi.mock("../../scripts/lib/ios-release-e2e-build.js", () => ({ + prepareIOSReleaseNativeBuild: nativeMocks.build, +})); vi.mock("../helpers/openclaw-test-instance.js", () => ({ createOpenClawTestInstance: nativeMocks.gateway, })); @@ -42,6 +48,7 @@ afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); nativeMocks.command.mockReset(); + nativeMocks.build.mockReset(); nativeMocks.gateway.mockReset(); nativeMocks.rpc.mockReset(); }); @@ -95,7 +102,7 @@ describe("iOS release test identity", () => { ["skipped", { result: "Skipped" }], ["failed", { result: "Failed" }], ["failed child", { children: [{ nodeType: "Test Case Run", result: "Failed" }] }], - ["wrong class", { nodeIdentifier: "OtherTests/testLiveGatewayFreshInstallSetupAndRelaunch()" }], + ["wrong class", { nodeIdentifier: "OtherTests/testLiveGatewayPairChatAndRelaunch()" }], [ "retry to green", { @@ -221,7 +228,7 @@ describe("sampled simulator-tree footprint", () => { function fixture( options: { - fail?: "prepare" | "test" | "cleanup"; + fail?: "prepare" | "test" | "reader" | "cleanup"; measure?: boolean; invalidMeasurement?: boolean; cancel?: boolean; @@ -243,8 +250,8 @@ function fixture( }; signal.addEventListener("abort", () => reject(new Error("stopped")), { once: true }); }), - create: vi.fn(async (test, arm, index) => { - trace.push(`create:${index}:${arm}:${test}`); + create: vi.fn(async (arm, index) => { + trace.push(`create:${index}:${arm}`); return { prepare: async () => { trace.push(`prepare:${index}`); @@ -253,8 +260,8 @@ function fixture( throw new Error("private preparation diagnostics"); } }, - test: async () => { - trace.push(`test:${index}`); + test: async (test: TestIdentity) => { + trace.push(`test:${index}:${test}`); if (options.measure && !options.invalidMeasurement) { wake?.(); await new Promise((resolve) => { @@ -265,7 +272,11 @@ function fixture( if (options.cancel) { abort.abort(); } - if (index === 1 && options.fail === "test") { + if ( + index === 1 && + (options.fail === "test" || + (options.fail === "reader" && test === IOS_RELEASE_TESTS[1])) + ) { throw Object.assign(new Error("private timeout diagnostics"), { code: "ETIMEDOUT" }); } return result(test); @@ -300,37 +311,75 @@ describe("fresh trial ownership", () => { TEST_RUNNER_OPENCLAW_IOS_LIVE_SETUP_CODE: "synthetic-setup-code", }); }); - it("runs a stock gate with fresh resources per test and no meter", async () => { + it("runs live pairing and the independent reader test on one prepared simulator", async () => { const { deps, trace } = fixture(); const report = await runTrials("stock", deps); expect(report.complete).toBe(true); - expect(report.trials.map((trial) => trial.status)).toEqual(["passed", "passed"]); - expect(deps.create).toHaveBeenNthCalledWith(1, IOS_RELEASE_TESTS[0], "stock", 1); - expect(deps.create).toHaveBeenNthCalledWith(2, IOS_RELEASE_TESTS[1], "stock", 2); - expect(trace.filter((entry) => entry.startsWith("measure:"))).toEqual([]); - expect(trace.indexOf("cleanup:1")).toBeLessThan(trace.indexOf("prepare:2")); + expect(report.trials).toEqual([ + expect.objectContaining({ + status: "passed", + tests: IOS_RELEASE_TESTS.map((test) => ({ test, status: "passed", ms: 10 })), + preparationMs: 100, + testMs: 20, + totalMs: 125, + }), + ]); + expect(deps.create).toHaveBeenCalledExactlyOnceWith("stock", 1); + expect(trace).toEqual([ + "create:1:stock", + "prepare:1", + ...IOS_RELEASE_TESTS.map((test) => `test:1:${test}`), + "cleanup:1", + ]); }); - it("retains all sixteen fresh trials in fixed AB BA AB BA order", async () => { + it("retains all eight fresh arms in fixed AB BA AB BA order after a failed arm", async () => { const { deps, trace } = fixture({ fail: "test", measure: true }); const report = await runTrials("compare", deps); - expect(armPlan("compare").map(({ arm }) => arm)).toEqual([ - "stock", - "simslim", - "simslim", - "stock", - "stock", - "simslim", - "simslim", - "stock", - ]); - expect(deps.create).toHaveBeenCalledTimes(16); - expect(report.trials).toHaveLength(16); - expect(report.trials[0]?.status).toBe("failed"); - expect(report.trials[0]?.errors).toEqual(["test-timeout"]); - expect(report.trials.slice(1).every((trial) => trial.status === "passed")).toBe(true); - expect(trace.filter((entry) => entry.startsWith("cleanup:"))).toHaveLength(16); + const order = ["stock", "simslim", "simslim", "stock", "stock", "simslim", "simslim", "stock"]; + expect(armPlan("compare").map(({ arm }) => arm)).toEqual(order); + expect(report.trials.map(({ arm }) => arm)).toEqual(order); + expect(deps.create).toHaveBeenCalledTimes(8); + expect(report.trials[0]).toMatchObject({ + status: "failed", + errors: ["test-timeout"], + tests: [{ test: IOS_RELEASE_TESTS[0], status: "failed" }], + }); + expect( + report.trials + .slice(1) + .every((trial) => trial.status === "passed" && trial.tests.length === 2), + ).toBe(true); + expect(trace.filter((entry) => entry.startsWith("cleanup:"))).toHaveLength(8); + for (let index = 1; index < 8; index++) { + expect(trace.indexOf(`cleanup:${index}`)).toBeLessThan(trace.indexOf(`prepare:${index + 1}`)); + } expect(JSON.stringify(report)).not.toContain("private"); }); + it.each(["test", "reader"] as const)( + "fails the arm after its %s failure without repeating either test", + async (fail) => { + const { deps, trace } = fixture({ fail }); + const report = await runTrials("stock", deps); + expect(report.trials).toHaveLength(1); + expect(report.trials[0]).toMatchObject({ + status: "failed", + errors: ["test-timeout"], + tests: + fail === "test" + ? [{ test: IOS_RELEASE_TESTS[0], status: "failed" }] + : [ + { test: IOS_RELEASE_TESTS[0], status: "passed" }, + { test: IOS_RELEASE_TESTS[1], status: "failed" }, + ], + }); + expect(trace.filter((entry) => entry.startsWith("test:"))).toEqual( + (fail === "test" ? [IOS_RELEASE_TESTS[0]] : IOS_RELEASE_TESTS).map( + (test) => `test:1:${test}`, + ), + ); + expect(trace.at(-1)).toBe("cleanup:1"); + }, + ); it("refuses comparison without a meter", async () => { const { deps } = fixture(); await expect(runTrials("compare", deps)).rejects.toThrow("comparison-meter-required"); @@ -341,7 +390,8 @@ describe("fresh trial ownership", () => { const report = await runTrials("stock", deps); expect(report.trials[0]?.errors).toEqual(["preparation-failed"]); expect(trace.filter((entry) => entry === "prepare:1")).toHaveLength(1); - expect(trace).not.toContain("test:1"); + expect(trace.some((entry) => entry.startsWith("test:"))).toBe(false); + expect(report.trials[0]?.tests).toEqual([]); expect(trace).toContain("cleanup:1"); }); it("joins the collector and fails incomplete measurement without discarding the trial", async () => { @@ -362,6 +412,11 @@ describe("fresh trial ownership", () => { expect(report.trials).toHaveLength(1); expect(report.trials[0]?.status).toBe("failed"); expect(trace).toContain("cleanup:1"); + if ("cancel" in options && options.cancel) { + expect(trace.filter((entry) => entry.startsWith("test:"))).toEqual([ + `test:1:${IOS_RELEASE_TESTS[0]}`, + ]); + } } }); }); @@ -513,16 +568,6 @@ describe("release qualification workflow authority", () => { ); }, ); - it("uses ad-hoc Debug products, exact destinations and the XCTest result CLI", () => { - const source = readFileSync("scripts/lib/ios-release-e2e-native.ts", "utf8"); - expect(source).not.toContain('"CODE_SIGNING_ALLOWED=NO"'); - expect(source).toContain('"Debug"'); - expect(source).toContain("`platform=iOS Simulator,id=${udid}`"); - expect(source).toContain('"test-without-building"'); - expect(source).toContain('"test-results"'); - expect(source).not.toContain('"--format"'); - expect(source).not.toContain("autoapprove"); - }); }); describe("native command adapter", () => { @@ -530,6 +575,8 @@ describe("native command adapter", () => { "success", "dirty-tracked", "dirty-untracked", + "source-late-dirty", + "source-late-head-change", "different-xcode", "different-xcode-build", "invalid-xcode-output", @@ -542,10 +589,28 @@ describe("native command adapter", () => { "cleanup-failure", "build-unjoined", "build-exit", + "boot-timeout", + "gateway-start-failure", + "setup-status-timeout", + "setup-status-failure", + "status-unjoined-gateway-exit", + "gateway-exit-during-create", + "gateway-exit-during-boot", + "cancel-during-boot", + "gateway-only", "setup-code-timeout", "setup-code-rpc-timeout", "test-unjoined", "test-exit", + "reader-failure", + "fixture-exit", + "gateway-exit", + "missing-first", + "missing-second", + "missing-relaunch", + "provider-duplicate", + "provider-out-of-order", + "provider-extra", "test-timeout-output", "reply-failure-evidence", "reply-failure-history-error", @@ -554,6 +619,7 @@ describe("native command adapter", () => { "reply-failure-app-log-error", ])("owns admission, build, test and cleanup for %s", async (scenario) => { const temp = tempDirs.make("ios-release-e2e-adapter-"); + const developerDir = tempDirs.make("ios-release-e2e-developer-"); vi.spyOn(os, "tmpdir").mockReturnValue(temp); vi.stubGlobal( "process", @@ -564,6 +630,22 @@ describe("native command adapter", () => { ); vi.stubEnv("OPENCLAW_CI_SIMSLIM_BINARY", ""); const instances: { cleanup: ReturnType }[] = []; + const lifecycle: string[] = []; + let simulatorReady = false; + let sourceChanged = false; + let exitMock: (() => void) | undefined; + let requestLog = ""; + let xctestrunPath = ""; + nativeMocks.build.mockImplementation(async (options) => { + const derivedDataPath = path.join(options.buildDir, "DerivedData"); + xctestrunPath = path.join(derivedDataPath, "Build/Products/OpenClawUITests.xctestrun"); + await options.build(derivedDataPath); + return { derivedDataPath, xctestrunPath, reused: false }; + }); + const gatewayChild: EventEmitter & { + exitCode: number | null; + signalCode: NodeJS.Signals | null; + } = Object.assign(new EventEmitter(), { exitCode: null, signalCode: null }); let nativeCommandActive = false; let historyReadBeforeCommandExit = false; nativeMocks.rpc.mockImplementation(async (options) => { @@ -574,11 +656,42 @@ describe("native command adapter", () => { } return { messages: [ - { role: "user", content: [{ type: "text", text: "OPENCLAW_E2E_OK_200" }] }, - { role: "assistant", content: [{ type: "text", text: "OPENCLAW_E2E_SEED_0_100" }] }, + { role: "user", content: [{ type: "text", text: "OPENCLAW_E2E_RELAUNCH" }] }, + { role: "assistant", content: [{ type: "text", text: "OPENCLAW_E2E_FIRST" }] }, ], }; } + options.onHelloOk?.(); + options.assertDispatchCurrent?.(); + if (options.method === "device.pair.setupStatus") { + expect(simulatorReady).toBe(false); + lifecycle.push("setup-status"); + if (scenario === "status-unjoined-gateway-exit") { + gatewayChild.exitCode = 17; + gatewayChild.emit("exit", 17, null); + throw Object.assign(new Error("private status cleanup failure"), { + code: "ETIMEDOUT", + processTreeState: "unknown", + }); + } + if (scenario === "setup-status-timeout") { + throw new GatewayTransportError({ + kind: "timeout", + message: "private status timeout", + connectionDetails: { url: "ws://private", urlSource: "private", message: "private" }, + timeoutMs: 30_000, + requestDispatched: true, + }); + } + if (scenario === "setup-status-failure") { + throw new Error("private status preparation failure"); + } + lifecycle.push("setup-status-ready"); + return {}; + } + expect(options.method).toBe("device.pair.setupCode"); + expect(simulatorReady).toBe(scenario !== "gateway-only"); + lifecycle.push("setup-code"); if (scenario === "setup-code-timeout") { throw new Error("private fixture command failed", { cause: Object.assign(new Error("private setup code and path"), { code: "ETIMEDOUT" }), @@ -595,14 +708,25 @@ describe("native command adapter", () => { return { setupCode: `synthetic-code-${instances.length}` }; }); nativeMocks.gateway.mockImplementation(async () => { + expect(simulatorReady).toBe(false); + lifecycle.push("gateway-create"); const index = instances.length + 1; const instance = { url: `ws://127.0.0.1:${20000 + index}`, gatewayToken: `synthetic-token-${index}`, configPath: `/private/fixture-${index}/config.json`, - startGateway: vi.fn(async () => {}), + child: gatewayChild, + startGateway: vi.fn(async () => { + lifecycle.push("gateway-start"); + if (scenario === "gateway-start-failure") { + throw new Error("private Gateway startup failure"); + } + }), logs: () => "private log\n[responses] start private\n[responses] completed private\n", cleanup: vi.fn(async () => { + lifecycle.push("gateway-cleanup"); + gatewayChild.exitCode = 0; + gatewayChild.emit("exit", 0, null); if (scenario === "cleanup-failure") { throw new Error("private cleanup failure"); } @@ -621,50 +745,38 @@ describe("native command adapter", () => { options.onReady?.({ stdout, stderr } as unknown as ChildProcess); const args = options.args as string[]; if (args.includes("scripts/e2e/mock-openai-server.mjs")) { - const body = scenario.startsWith("reply-failure-") - ? { - model: "ios-e2e", - input: [ - { - role: "user", - content: "Reply exactly with OPENCLAW_E2E_OK_200 and no other text.", - }, - ], - metadata: { title: "OPENCLAW_E2E_SEED_0_100" }, - } - : { model: "ios-e2e" }; - const requests = [body]; - if (scenario.startsWith("reply-failure-")) { - requests.push({ model: "ios-e2e" }); + expect(simulatorReady).toBe(false); + if (scenario !== "gateway-only") { + expect(lifecycle).toContain("native-build-complete"); } - writeFileSync( - options.env.MOCK_REQUEST_LOG, - requests - .map((requestBody) => - JSON.stringify({ path: "/v1/responses", body: JSON.stringify(requestBody) }), - ) - .join("\n") + "\n", - ); + lifecycle.push("mock-start"); + requestLog = options.env.MOCK_REQUEST_LOG; stdout.write("mock-openai listening on 20001\n"); - await new Promise((_resolve, reject) => { - options.signal.addEventListener( - "abort", - () => { - joinedMocks++; - reject(Object.assign(new Error("stopped"), { code: "ABORT_ERR" })); - }, - { once: true }, - ); + await new Promise((resolve, reject) => { + const onAbort = () => { + joinedMocks++; + lifecycle.push("mock-cleanup"); + reject(Object.assign(new Error("stopped"), { code: "ABORT_ERR" })); + }; + exitMock = () => { + options.signal.removeEventListener("abort", onAbort); + joinedMocks++; + lifecycle.push("mock-exit"); + resolve(); + }; + options.signal.addEventListener("abort", onAbort, { once: true }); }); } else if (args[0] === "status") { expect(args).toEqual(["status", "--porcelain=v1", "--untracked-files=all"]); - if (scenario === "dirty-tracked") { + if (scenario === "dirty-tracked" || (scenario === "source-late-dirty" && sourceChanged)) { stdout.write(" M scripts/ios-release-e2e.ts\n"); } else if (scenario === "dirty-untracked") { stdout.write("?? untracked-source.ts\n"); } } else if (options.bin === "git") { - stdout.write("1".repeat(40)); + stdout.write( + (scenario === "source-late-head-change" && sourceChanged ? "2" : "1").repeat(40), + ); } else if (args.includes("-version")) { stdout.write( scenario === "different-xcode" @@ -675,6 +787,10 @@ describe("native command adapter", () => { ? "unrecognized toolchain\n" : "Xcode 27.0\nBuild version 27A266a\n", ); + } else if (args.includes("--print-path")) { + stdout.write(`${developerDir}\n`); + } else if (args.includes("--show-sdk-build-version")) { + stdout.write("27.0\n"); } else if (args.includes("runtimes")) { const runtime = { isAvailable: scenario !== "unavailable-runtime", @@ -725,6 +841,8 @@ describe("native command adapter", () => { }), ); } else if (args.includes("create")) { + expect(lifecycle).toContain("setup-status-ready"); + lifecycle.push("simulator-create"); expect(args.at(-1)).toBe( scenario === "different-runtime" ? "com.apple.CoreSimulator.SimRuntime.iOS-27-0" @@ -732,7 +850,34 @@ describe("native command adapter", () => { ? "com.apple.CoreSimulator.SimRuntime.iOS-26-10" : "com.apple.CoreSimulator.SimRuntime.iOS-26-5", ); + if (scenario === "gateway-exit-during-create") { + gatewayChild.exitCode = 17; + gatewayChild.emit("exit", 17, null); + expect(options.signal.aborted).toBe(false); + } stdout.write(`11111111-2222-3333-4444-${String(++created).padStart(12, "0")}`); + } else if (args.includes("bootstatus")) { + lifecycle.push("boot-wait"); + await Promise.resolve(); + if (scenario === "boot-timeout") { + throw Object.assign(new Error("private simulator boot timeout"), { code: "ETIMEDOUT" }); + } + if (scenario === "gateway-exit-during-boot" || scenario === "cancel-during-boot") { + if (scenario === "gateway-exit-during-boot") { + gatewayChild.exitCode = 17; + gatewayChild.emit("exit", 17, null); + } else { + abort.abort(); + } + expect(options.signal.aborted).toBe(true); + throw Object.assign(new Error("private simulator boot interrupted"), { + code: "ABORT_ERR", + }); + } + simulatorReady = true; + lifecycle.push("boot-ready"); + } else if (args.includes("delete")) { + lifecycle.push("simulator-delete"); } else if (args.includes("build-for-testing")) { appContainer = path.join( path.dirname(args[args.indexOf("-derivedDataPath") + 1]!), @@ -748,20 +893,125 @@ describe("native command adapter", () => { stderr.write("BUILD FAILED: private setup code and private path\n"); return 65; } + lifecycle.push("native-build-complete"); } else if (args.includes("test-without-building")) { + const testArgument = args.find((arg) => arg.startsWith("-only-testing:")); + if (!testArgument) { + throw new Error("native test selection missing"); + } + selectedTest = testArgument.slice("-only-testing:".length); + const reader = selectedTest === IOS_RELEASE_TESTS[1]; + lifecycle.push(reader ? "reader-test" : "live-test"); + if (!reader) { + stdout.write( + "IOS_RELEASE_CHECKPOINT paired\nIOS_RELEASE_CHECKPOINT private-credential\n", + ); + } + if (reader) { + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + expect(options.env.TEST_RUNNER_OPENCLAW_IOS_LIVE_SETUP_CODE).toBe(""); + expect(options.env.TEST_RUNNER_OPENCLAW_IOS_LIVE_GATEWAY).not.toBe("1"); + if (scenario === "reader-failure") { + return 65; + } + } else { + const stages = + scenario === "provider-duplicate" + ? ["first", "first", "second", "relaunch"] + : scenario === "provider-out-of-order" + ? ["second", "first", "relaunch"] + : [ + "first", + "second", + "relaunch", + ...(scenario === "provider-extra" ? ["unexpected"] : []), + ]; + const requests = scenario.startsWith("reply-failure-") + ? [ + { + model: "ios-e2e", + input: [ + { + role: "user", + content: "Reply exactly with OPENCLAW_E2E_RELAUNCH and no other text.", + }, + ], + metadata: { title: "OPENCLAW_E2E_FIRST" }, + }, + { model: "ios-e2e" }, + ] + : stages.map((stage) => { + const marker = `OPENCLAW_E2E_${stage.toUpperCase()}`; + const missing = scenario === `missing-${stage}`; + return { + model: "ios-e2e", + input: [ + { role: "user", content: `Reply exactly with ${marker} and no other text.` }, + { role: "assistant", content: marker }, + { + role: "user", + content: [ + { + type: "input_text", + text: + "[Sun 2026-09-27 21:02 CDT] " + + "Conversation info: ⟦openclaw:ctx⟧\n```json\n" + + JSON.stringify({ sender: { id: "fixture-owner", name: marker } }) + + "\n```\n\n" + + (missing + ? "No requested marker here." + : `Reply exactly with ${marker} and no other text.`), + }, + ], + }, + { + role: "user", + content: [ + { + type: "input_text", + text: + "<<>>\n" + + "OPENCLAW_E2E_FIRST\n<<>>", + }, + ], + }, + ], + metadata: { title: marker }, + }; + }); + writeFileSync( + requestLog, + requests + .map((body) => JSON.stringify({ path: "/v1/responses", body: JSON.stringify(body) })) + .join("\n") + "\n", + ); + } + if (scenario === "gateway-exit") { + gatewayChild.exitCode = 17; + gatewayChild.emit("exit", 17, null); + expect(options.signal.aborted).toBe(true); + throw Object.assign(new Error("private native command interrupted"), { + code: "ABORT_ERR", + }); + } + if (scenario === "fixture-exit") { + exitMock?.(); + await Promise.resolve(); + } if ( scenario.startsWith("reply-failure-") && - args.includes(`-only-testing:${IOS_RELEASE_TESTS[1]}`) + args.includes(`-only-testing:${IOS_RELEASE_TESTS[0]}`) ) { nativeCommandActive = true; const failureMessage = scenario === "reply-failure-source-only" ? "" - : `IOS_RELEASE_CHAT_FAILURE final ${scenario === "reply-failure-submission" ? "submission" : "reply"} draft=false keyboard=true reply=false writing=false jump=true foreground=true input=true transcript=true send=false`; + : `IOS_RELEASE_CHAT_FAILURE relaunch ${scenario === "reply-failure-submission" ? "submission" : "reply"} draft=false keyboard=true reply=false writing=false jump=true foreground=true input=true transcript=true send=false`; stdout.write( - "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayChatRoundTripAndControlOverview]' started.\n" + + "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayPairChatAndRelaunch]' started.\n" + `/private/checkout/OpenClawSnapshotUITests.swift:1913: error: private ${failureMessage}\n` + - "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayChatRoundTripAndControlOverview]' failed (99 seconds).\n", + "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayPairChatAndRelaunch]' failed (99 seconds).\n", ); await Promise.resolve(); nativeCommandActive = false; @@ -769,11 +1019,11 @@ describe("native command adapter", () => { } if (scenario === "test-timeout-output") { stdout.write( - "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayChatRoundTripAndControlOverview]' started.\n" + + "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayPairChatAndRelaunch]' started.\n" + "/private/checkout/OpenClawSnapshotUITests.swift:1904: error: private assertion details\n" + "/private/checkout/OpenClawSnapshotUITests.swift:1904: error: repeated private details\n" + "/private/Other.swift:42: error: private details\n" + - "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayChatRoundTripAndControlOverview]' failed (39.615 seconds).\n", + "Test Case '-[OpenClawUITests.OpenClawSnapshotUITests testLiveGatewayPairChatAndRelaunch]' failed (39.615 seconds).\n", ); throw Object.assign(new Error("private timeout diagnostics"), { code: "ETIMEDOUT" }); } @@ -791,16 +1041,16 @@ describe("native command adapter", () => { stderr.write("TEST FAILED: private setup code and private path\n"); return 65; } - selectedTest = args - .find((arg) => arg.startsWith("-only-testing:"))! - .slice("-only-testing:".length); + if (!reader) { + stdout.write("IOS_RELEASE_CHECKPOINT overview\n"); + } } else if (options.bin === "/usr/bin/plutil") { stdout.write("ai.synthetic.private\n"); } else if (args.includes("get_app_container")) { expect(args).toEqual([ "simctl", "get_app_container", - "11111111-2222-3333-4444-000000000002", + "11111111-2222-3333-4444-000000000001", "ai.synthetic.private", "data", ]); @@ -825,11 +1075,17 @@ describe("native command adapter", () => { return 0; }); const proof: Record = {}; + const progressSnapshots: string[] = []; + const abort = new AbortController(); const admission = createNativeDependencies({ mode: "stock", targetSha: "1".repeat(40), - signal: new AbortController().signal, + signal: abort.signal, + gatewayOnly: scenario === "gateway-only", proof, + onProgress: async () => { + progressSnapshots.push(JSON.stringify(proof)); + }, }); if (scenario.startsWith("dirty-")) { await expect(admission).rejects.toMatchObject({ @@ -894,21 +1150,51 @@ describe("native command adapter", () => { : "com.apple.CoreSimulator.SimRuntime.iOS-26-5", }); try { + if (scenario === "gateway-only") { + const gatewayProbe = await native.dependencies.create("stock", 1); + try { + await gatewayProbe.prepare(); + } finally { + await gatewayProbe.cleanup(); + } + expect(lifecycle).toEqual([ + "mock-start", + "gateway-create", + "gateway-start", + "setup-status", + "setup-status-ready", + "setup-code", + "gateway-cleanup", + "mock-cleanup", + ]); + expect(nativeMocks.build).not.toHaveBeenCalled(); + expect(created).toBe(0); + expect(joinedMocks).toBe(1); + expect(nativeMocks.rpc.mock.calls.map(([options]) => options.method)).toEqual([ + "device.pair.setupStatus", + "device.pair.setupCode", + ]); + expect(proof.fixtures).toEqual([expect.objectContaining({ cleanupConfirmed: true })]); + return; + } const report = await runTrials("stock", native.dependencies); + expect(JSON.stringify(proof)).not.toMatch(/private|synthetic|OPENCLAW_E2E_|metadata/); + expect(progressSnapshots.length).toBeGreaterThan(0); + expect(progressSnapshots.join("\n")).not.toMatch(/private|synthetic|OPENCLAW_E2E_|metadata/); if (scenario.startsWith("reply-failure-")) { expect(report.complete).toBe(true); - expect(report.trials.map((trial) => trial.status)).toEqual(["passed", "failed"]); - expect(report.trials[1]).toMatchObject({ + expect(report.trials.map((trial) => trial.status)).toEqual(["failed"]); + expect(report.trials[0]).toMatchObject({ errors: ["test-timeout"], diagnostics: [{ operation: "native-test", code: "timeout", errorCode: "ETIMEDOUT" }], }); - const context = report.trials[1]?.diagnostics[0]?.context; + const context = report.trials[0]?.diagnostics[0]?.context; expect(context).toEqual( expect.arrayContaining([ ...(scenario === "reply-failure-source-only" ? [] : [ - "chat-stage:final", + "chat-stage:relaunch", `chat-checkpoint:${scenario === "reply-failure-submission" ? "submission" : "reply"}`, "chat-draft-retained:false", "chat-keyboard:true", @@ -920,8 +1206,8 @@ describe("native command adapter", () => { "chat-transcript-present:true", "chat-send-present:false", ]), - "provider-latest-user:final", - "provider-body-tail:seed-0", + "provider-latest-user:relaunch", + "provider-body-tail:first", "provider-marker-match:false", "model-any-request-stage:start", "model-any-request-stage:completed", @@ -938,7 +1224,7 @@ describe("native command adapter", () => { ]), ...(scenario === "reply-failure-history-error" ? ["history-evidence-unavailable"] - : ["history-user:final", "history-assistant:seed-0"]), + : ["history-user:relaunch", "history-assistant:first"]), ]), ); expect(context).not.toContain("provider-marker-match:true"); @@ -972,22 +1258,194 @@ describe("native command adapter", () => { expect(JSON.stringify(report)).not.toContain("private"); return; } + if (scenario === "boot-timeout" || scenario === "gateway-start-failure") { + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [], + errors: [scenario === "boot-timeout" ? "preparation-timeout" : "preparation-failed"], + diagnostics: [ + { + operation: scenario === "boot-timeout" ? "simulator-ready" : "gateway-start", + code: scenario === "boot-timeout" ? "timeout" : "failed", + }, + ], + }, + ]); + expect(lifecycle).not.toContain("live-test"); + expect(lifecycle).not.toContain("setup-code"); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + if (scenario === "boot-timeout") { + expect(lifecycle).toContain("setup-status-ready"); + expect(lifecycle.at(-1)).toBe("simulator-delete"); + } else { + expect(created).toBe(0); + expect(nativeMocks.rpc).not.toHaveBeenCalled(); + } + return; + } + if (scenario.startsWith("setup-status-")) { + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [], + errors: [ + scenario === "setup-status-timeout" ? "preparation-timeout" : "preparation-failed", + ], + diagnostics: [ + { + operation: "setup-status", + code: scenario === "setup-status-timeout" ? "timeout" : "failed", + context: + scenario === "setup-status-timeout" + ? [ + "rpc-authenticated:true", + "rpc-dispatch-entered:true", + "rpc-request-dispatched:true", + ] + : [], + }, + ], + }, + ]); + expect(nativeMocks.rpc).toHaveBeenCalledOnce(); + expect(created).toBe(0); + expect(lifecycle).not.toContain("setup-code"); + expect(lifecycle).not.toContain("live-test"); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + expect(proof.fixtures).toEqual([ + expect.objectContaining({ + cleanupConfirmed: true, + setupStatusRpc: { authenticated: true, dispatchEntered: true, responseReceived: false }, + }), + ]); + return; + } + if (scenario === "status-unjoined-gateway-exit") { + expect(report.complete).toBe(false); + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [], + errors: ["preparation-failed", "cleanup-failed"], + diagnostics: [ + { operation: "gateway-start", code: "exit", exitCode: 17 }, + { operation: "cleanup", code: "cleanup-unconfirmed" }, + ], + }, + ]); + expect(proof.resourcesPreserved).toBe(true); + expect(nativeMocks.rpc).toHaveBeenCalledOnce(); + expect(created).toBe(0); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + return; + } + if ( + scenario === "gateway-exit-during-create" || + scenario === "gateway-exit-during-boot" || + scenario === "cancel-during-boot" + ) { + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [], + errors: [scenario === "cancel-during-boot" ? "cancelled" : "preparation-failed"], + diagnostics: [ + scenario === "cancel-during-boot" + ? { operation: "simulator-ready", code: "cancelled" } + : { operation: "gateway-start", code: "exit", exitCode: 17 }, + ], + }, + ]); + expect(lifecycle).not.toContain("setup-code"); + expect(lifecycle).not.toContain("live-test"); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + expect(lifecycle.at(-1)).toBe("simulator-delete"); + expect(gatewayChild.listenerCount("exit")).toBe(0); + return; + } + if ( + scenario.startsWith("missing-") || + scenario.startsWith("provider-") || + scenario === "fixture-exit" || + scenario === "gateway-exit" + ) { + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [{ test: IOS_RELEASE_TESTS[0], status: "failed" }], + errors: ["test-failed"], + diagnostics: [ + { + operation: + scenario === "fixture-exit" + ? "fixture-server" + : scenario === "gateway-exit" + ? "gateway-start" + : "provider-rpc", + code: scenario === "gateway-exit" ? "exit" : "failed", + ...(scenario === "gateway-exit" ? { exitCode: 17 } : {}), + }, + ], + }, + ]); + expect(lifecycle.filter((event) => event === "live-test")).toHaveLength(1); + expect(lifecycle).not.toContain("reader-test"); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(joinedMocks).toBe(1); + return; + } + if (scenario === "reader-failure") { + expect(report.trials).toMatchObject([ + { + status: "failed", + tests: [ + { test: IOS_RELEASE_TESTS[0], status: "passed" }, + { test: IOS_RELEASE_TESTS[1], status: "failed" }, + ], + errors: ["test-failed"], + diagnostics: [{ operation: "native-test", code: "exit", exitCode: 65 }], + }, + ]); + expect(lifecycle.filter((event) => event.endsWith("-test"))).toEqual([ + "live-test", + "reader-test", + ]); + expect(lifecycle.indexOf("gateway-cleanup")).toBeLessThan(lifecycle.indexOf("reader-test")); + expect(lifecycle.indexOf("mock-cleanup")).toBeLessThan(lifecycle.indexOf("reader-test")); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + return; + } if (scenario === "setup-code-timeout" || scenario === "setup-code-rpc-timeout") { expect(report.complete).toBe(true); for (const trial of report.trials) { expect(trial).toMatchObject({ status: "failed", - errors: ["test-timeout"], + errors: ["preparation-timeout"], diagnostics: [ { operation: "setup-code", code: "timeout", ...(scenario === "setup-code-timeout" ? { errorCode: "ETIMEDOUT" } : {}), - context: [], + context: + scenario === "setup-code-rpc-timeout" + ? ["rpc-authenticated:true", "rpc-dispatch-entered:true"] + : [], }, ], }); } + expect(report.trials[0]?.tests).toEqual([]); + expect(proof.fixtures).toEqual([ + expect.objectContaining({ + trial: 1, + setupRpc: { authenticated: true, dispatchEntered: true, responseReceived: false }, + }), + ]); expect( nativeMocks.command.mock.calls.some(([{ args }]) => args.includes("test-without-building"), @@ -1005,23 +1463,68 @@ describe("native command adapter", () => { return; } if (scenario === "test-exit") { - expect(report.trials.map((trial) => trial.status)).toEqual(["failed", "failed"]); + expect(report.trials.map((trial) => trial.status)).toEqual(["failed"]); expect(report.trials[0]?.diagnostics).toEqual([ { operation: "native-test", code: "exit", exitCode: 65, - context: ["test-failed", "xctest-line:1904"], + context: expect.arrayContaining(["test-failed", "xctest-line:1904"]), }, ]); expect(JSON.stringify(report)).not.toContain("private"); return; } - expect(report.trials.map((trial) => trial.status)).toEqual(["passed", "passed"]); - expect(created).toBe(2); - expect(joinedMocks).toBe(2); + if (scenario === "source-late-dirty" || scenario === "source-late-head-change") { + expect(report.trials.map((trial) => trial.status)).toEqual(["passed"]); + expect(report.trials[0]?.tests).toHaveLength(2); + sourceChanged = true; + await expect(native.assertCurrentSource()).rejects.toMatchObject({ + diagnostic: + scenario === "source-late-dirty" + ? { operation: "source-status", code: "dirty-source" } + : { operation: "source-head", code: "identity-mismatch" }, + }); + expect(instances[0]?.cleanup).toHaveBeenCalledOnce(); + expect(lifecycle.filter((entry) => entry.endsWith("-test"))).toEqual([ + "live-test", + "reader-test", + ]); + return; + } + expect(report.trials.map((trial) => trial.status)).toEqual(["passed"]); + expect(report.trials[0]?.tests.map(({ test, status }) => ({ test, status }))).toEqual( + IOS_RELEASE_TESTS.map((test) => ({ test, status: "passed" })), + ); + expect(lifecycle.indexOf("mock-start")).toBeLessThan(lifecycle.indexOf("boot-ready")); + expect(lifecycle.indexOf("gateway-start")).toBeLessThan(lifecycle.indexOf("boot-ready")); + expect(lifecycle.indexOf("gateway-start")).toBeLessThan(lifecycle.indexOf("setup-status")); + expect(lifecycle.indexOf("setup-status-ready")).toBeLessThan( + lifecycle.indexOf("simulator-create"), + ); + expect(lifecycle.indexOf("boot-ready")).toBeLessThan(lifecycle.indexOf("setup-code")); + expect(lifecycle.indexOf("setup-code")).toBeLessThan(lifecycle.indexOf("live-test")); + expect(lifecycle.indexOf("gateway-cleanup")).toBeLessThan(lifecycle.indexOf("reader-test")); + expect(lifecycle.indexOf("mock-cleanup")).toBeLessThan(lifecycle.indexOf("reader-test")); + expect(lifecycle.at(-1)).toBe("simulator-delete"); + expect(created).toBe(1); + expect(joinedMocks).toBe(1); for (const [index, instance] of instances.entries()) { - expect(nativeMocks.rpc).toHaveBeenNthCalledWith(index + 1, { + expect(nativeMocks.rpc).toHaveBeenCalledTimes(2); + expect(nativeMocks.rpc).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + method: "device.pair.setupStatus", + params: { + setupId: expect.stringMatching(/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/u), + }, + timeoutMs: 30_000, + sharedStateMode: "read-only", + token: `synthetic-token-${index + 1}`, + url: `ws://127.0.0.1:${20001 + index}`, + }), + ); + expect(nativeMocks.rpc).toHaveBeenNthCalledWith(2, { config: {}, configPath: `/private/fixture-${index + 1}/config.json`, url: `ws://127.0.0.1:${20001 + index}`, @@ -1032,34 +1535,78 @@ describe("native command adapter", () => { method: "device.pair.setupCode", params: { publicUrl: `ws://127.0.0.1:${20001 + index}`, includeQr: false }, timeoutMs: 30_000, - signal: native.dependencies.signal, + signal: expect.any(AbortSignal), + onHelloOk: expect.any(Function), + assertDispatchCurrent: expect.any(Function), }); expect(instance.cleanup).toHaveBeenCalledOnce(); } + expect(proof.fixtures).toEqual([ + expect.objectContaining({ + trial: 1, + cleanupConfirmed: true, + setupStatusRpc: { authenticated: true, dispatchEntered: true, responseReceived: true }, + setupRpc: { authenticated: true, dispatchEntered: true, responseReceived: true }, + providerMessages: [ + { stage: "first", received: true }, + { stage: "second", received: true }, + { stage: "relaunch", received: true }, + ], + }), + ]); + expect(proof.fixtures).not.toEqual( + expect.arrayContaining([expect.objectContaining({ gatewayExited: true })]), + ); + expect(gatewayChild.listenerCount("exit")).toBe(0); + expect(proof.phases).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + operation: "native-test", + trial: 1, + status: "passed", + checkpoint: "overview", + }), + ]), + ); const commands = nativeMocks.command.mock.calls.map(([options]) => options); expect(commands.filter(({ args }) => args.includes("build-for-testing"))).toHaveLength(1); - for (const { args: nativeArgs } of commands.filter( - ({ args }) => args.includes("build-for-testing") || args.includes("test-without-building"), - )) { - expect(nativeArgs).toEqual( - expect.arrayContaining([ - "CODE_SIGNING_ALLOWED=YES", - "CODE_SIGN_IDENTITY=-", - "CODE_SIGN_STYLE=Manual", - "PROVISIONING_PROFILE=", - "PROVISIONING_PROFILE_SPECIFIER=", - ]), - ); - expect(nativeArgs).not.toContain("-allowProvisioningUpdates"); - } + const nativeBuild = commands.find(({ args }) => args.includes("build-for-testing")); + expect(nativeBuild.args).toEqual( + expect.arrayContaining([ + "-configuration", + "Debug", + "CODE_SIGNING_ALLOWED=YES", + "CODE_SIGN_IDENTITY=-", + "CODE_SIGN_STYLE=Manual", + "PROVISIONING_PROFILE=", + "PROVISIONING_PROFILE_SPECIFIER=", + ]), + ); + expect(nativeBuild.args).not.toContain("-allowProvisioningUpdates"); + expect(nativeMocks.build).toHaveBeenCalledOnce(); + expect(nativeMocks.build.mock.calls[0]?.[0].identity).toMatchObject({ + sourceSha: "1".repeat(40), + sdkVersion: "27.0", + platform: "darwin", + arch: "arm64", + }); expect( commands .filter(({ args }) => args.includes("test-without-building")) .map(({ env }) => env.TEST_RUNNER_OPENCLAW_IOS_LIVE_SETUP_CODE), - ).toEqual(["synthetic-code-1", "synthetic-code-2"]); + ).toEqual(["synthetic-code-1", ""]); for (const { args: testArgs } of commands.filter(({ args }) => args.includes("test-without-building"), )) { + expect(testArgs).toContain( + "platform=iOS Simulator,id=11111111-2222-3333-4444-000000000001", + ); + expect( + testArgs.slice(testArgs.indexOf("-xctestrun"), testArgs.indexOf("-xctestrun") + 2), + ).toEqual(["-xctestrun", xctestrunPath]); + expect(testArgs).not.toContain("-project"); + expect(testArgs).not.toContain("-derivedDataPath"); + expect(testArgs.some((argument: string) => argument.startsWith("CODE_SIGN"))).toBe(false); const diagnosticsIndex = testArgs.indexOf("-collect-test-diagnostics"); expect(testArgs.slice(diagnosticsIndex, diagnosticsIndex + 2)).toEqual([ "-collect-test-diagnostics", @@ -1069,9 +1616,17 @@ describe("native command adapter", () => { expect(testArgs).not.toContain("-retry-tests-on-failure"); expect(testArgs).not.toContain("-run-tests-until-failure"); } + expect( + commands + .filter(({ args }) => args.includes("xcresulttool")) + .map(({ args }) => args.slice(0, 4)), + ).toEqual([ + ["xcresulttool", "get", "test-results", "tests"], + ["xcresulttool", "get", "test-results", "tests"], + ]); expect( commands.filter(({ args }) => args.includes("delete")).map(({ args }) => args.at(-1)), - ).toEqual(["11111111-2222-3333-4444-000000000001", "11111111-2222-3333-4444-000000000002"]); + ).toEqual(["11111111-2222-3333-4444-000000000001"]); expect(nativeMocks.gateway.mock.calls[0]?.[0]).toMatchObject({ config: { gateway: { controlUi: { enabled: false } }, @@ -1080,7 +1635,11 @@ describe("native command adapter", () => { env: gatewayEnv, }); } finally { - if (scenario === "cleanup-failure" || scenario === "test-unjoined") { + if ( + scenario === "cleanup-failure" || + scenario === "test-unjoined" || + scenario === "status-unjoined-gateway-exit" + ) { // This is the outer owner's cleanup call after the trial loop has stopped. await expect(native.cleanup()).rejects.toMatchObject({ diagnostic: { operation: "cleanup", code: "cleanup-unconfirmed" },