diff --git a/extensions/qa-lab/src/qa-gateway-config.test.ts b/extensions/qa-lab/src/qa-gateway-config.test.ts index 6b8e909f96f1..5e8e46e15dc5 100644 --- a/extensions/qa-lab/src/qa-gateway-config.test.ts +++ b/extensions/qa-lab/src/qa-gateway-config.test.ts @@ -7,6 +7,11 @@ import { } from "./providers/shared/session-observer-registry.js"; import { buildQaGatewayConfig } from "./qa-gateway-config.js"; import type { QaTransportGatewayConfig } from "./qa-transport.js"; +import { readQaScenarioById } from "./scenario-catalog.js"; +import { + applyQaSuiteGatewayConfigPatches, + collectQaSuiteGatewayConfigPatches, +} from "./suite-planning.js"; function buildConfig(params: Partial[0]>) { return buildQaGatewayConfig({ @@ -70,6 +75,35 @@ function expectQaLabPluginEnabled(cfg: ReturnType) } describe("buildQaGatewayConfig", () => { + it.each([ + { + scenarioId: "anthropic-thinking-error-recovery-replay-safe-read", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + allowedRefs: ["anthropic/claude-opus-4-8"], + }, + { + scenarioId: "thinking-slash-model-remap", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.5", + allowedRefs: ["openai/gpt-5.5", "anthropic/claude-sonnet-4-6"], + }, + ] as const)( + "composes an exact override policy for $scenarioId", + ({ scenarioId, providerMode, primaryModel, allowedRefs }) => { + const base = buildConfig({ providerMode, primaryModel }); + const config = applyQaSuiteGatewayConfigPatches( + base, + collectQaSuiteGatewayConfigPatches([readQaScenarioById(scenarioId)]), + ); + + expect(config).toMatchObject({ + agents: { defaults: { modelPolicy: { allow: [...allowedRefs] } } }, + }); + expect(base.agents?.defaults?.modelPolicy?.allow).not.toContain(allowedRefs.at(-1)); + }, + ); + it("uses only active full-mock observer registrations across endpoint aliases", () => { const baseUrl = "http://127.0.0.1:44082"; const observerUrl = `${baseUrl}/debug/session`; diff --git a/extensions/qa-lab/src/scenario-flow-runner-cron-proof.test.ts b/extensions/qa-lab/src/scenario-flow-runner-cron-proof.test.ts index 03d57ac1a362..974a98542792 100644 --- a/extensions/qa-lab/src/scenario-flow-runner-cron-proof.test.ts +++ b/extensions/qa-lab/src/scenario-flow-runner-cron-proof.test.ts @@ -160,7 +160,7 @@ async function runSchedulingFixture( if (restarted) { options.mutateRestartedJobs?.(visible); } - return { jobs: visible }; + return { jobs: [...visible, job("background-maintenance", 99)] }; } if (method === "cron.runs") { if (params.id === "job-0") { @@ -279,7 +279,7 @@ describe("scheduling YAML canonical tool proof", () => { ); it.each(["direct", "nested"] as const)( - "accepts one-shot/recurring evidence in %s history", + "accepts one-shot/recurring evidence alongside other owners in %s history", async (shape) => { const { result, restarted, removed } = await runSchedulingFixture("recurring", shape); expect(result.status).toBe("pass"); @@ -418,6 +418,16 @@ describe("scheduling YAML canonical tool proof", () => { ).rejects.toThrow(/recurring job did not remain scheduled/); }); + it.each(["at", "every"])("rejects a replayed %s job with a new identity", async (schedule) => { + await expect( + runSchedulingFixture("recurring", "nested", { + mutateRestartedJobs: (jobs) => { + jobs.push(job(`qa-model-${schedule}-${suffix}`, 98)); + }, + }), + ).rejects.toThrow(/unexpected scenario cron jobs/); + }); + it("rejects a recurring job whose next execution does not advance", async () => { await expect( runSchedulingFixture("recurring", "nested", { diff --git a/qa/scenarios/models/thinking-slash-model-remap.yaml b/qa/scenarios/models/thinking-slash-model-remap.yaml index 7b744f2d08ea..fb72ef47df49 100644 --- a/qa/scenarios/models/thinking-slash-model-remap.yaml +++ b/qa/scenarios/models/thinking-slash-model-remap.yaml @@ -15,6 +15,10 @@ scenario: gatewayConfigPatch: agents: defaults: + modelPolicy: + allow: + - openai/gpt-5.5 + - anthropic/claude-sonnet-4-6 models: anthropic/claude-sonnet-4-6: params: {} diff --git a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml index b144f23bbbc8..f59abb63136c 100644 --- a/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml +++ b/qa/scenarios/runtime/anthropic-thinking-error-recovery-replay-safe-read.yaml @@ -14,6 +14,9 @@ scenario: enabled: false agents: defaults: + modelPolicy: + allow: + - anthropic/claude-opus-4-8 models: anthropic/claude-opus-4-8: params: {} diff --git a/qa/scenarios/scheduling/cron-model-created-one-shot-recurring.yaml b/qa/scenarios/scheduling/cron-model-created-one-shot-recurring.yaml index 7eeb0c20a242..5d9edafd1b67 100644 --- a/qa/scenarios/scheduling/cron-model-created-one-shot-recurring.yaml +++ b/qa/scenarios/scheduling/cron-model-created-one-shot-recurring.yaml @@ -270,10 +270,13 @@ flow: expr: "JSON.stringify([...(finalRecurring.payload.toolsAllow ?? [])].sort()) === JSON.stringify([...(recurringJob.payload.toolsAllow ?? [])].sort()) && finalRecurring.payload.toolsAllowIsDefault === true" message: expr: "`recurring authority changed across restart: before=${JSON.stringify(recurringJob.payload)} after=${JSON.stringify(finalRecurring?.payload)}`" + - set: finalOwnedJobs + value: + expr: "finalList.jobs.filter((job) => job.name === oneShotName || job.name === recurringName)" - assert: - expr: "finalList.jobs.length === 1 && finalList.jobs[0].id === recurringJob.id" + expr: "finalOwnedJobs.length === 1 && finalOwnedJobs[0].id === recurringJob.id" message: - expr: "`gateway restart replayed or created unexpected cron jobs: ${JSON.stringify(finalList.jobs.map((job) => ({ id: job.id, name: job.name, schedule: job.schedule })))}`" + expr: "`gateway restart replayed or created unexpected scenario cron jobs: ${JSON.stringify(finalOwnedJobs.map((job) => ({ id: job.id, name: job.name, schedule: job.schedule })))}`" - call: env.gateway.call saveAs: oneShotRunsPage args: diff --git a/test/helpers/openclaw-test-instance.test.ts b/test/helpers/openclaw-test-instance.test.ts index 2053621b4d5c..fb94db1dd148 100644 --- a/test/helpers/openclaw-test-instance.test.ts +++ b/test/helpers/openclaw-test-instance.test.ts @@ -44,6 +44,7 @@ const RESTART_MARKER = "[openclaw-test-instance] restarting gateway after migration convergence refusal"; const LEGACY_STORE_PATH = "/fixture/sessions/sessions.json"; const LEGACY_MIGRATION_REFUSAL = `Legacy session store requires migration: ${LEGACY_STORE_PATH}. Run "openclaw doctor --fix" against the same state/config before starting OpenClaw.`; +const PROFILED_LEGACY_MIGRATION_REFUSAL = `Legacy session store requires migration: ${LEGACY_STORE_PATH}. Run "openclaw --profile qa-fixture doctor --fix" against the same state/config before starting OpenClaw.`; const LEGACY_STARTUP_FAILURE = [ "OpenClaw startup migrations did not complete cleanly; refusing to report the gateway ready.", `- Legacy sessions store unreadable; left in place at ${LEGACY_STORE_PATH}`, @@ -326,8 +327,8 @@ if (kind === "cli" || kind === "cli-drain") { process.exit(Number(argv[0])); } const refusal = ${JSON.stringify(MIGRATION_CONVERGENCE_REFUSAL)}; -const legacyRefusal = kind.startsWith("startup-") ? ${JSON.stringify(LEGACY_STARTUP_FAILURE)} : ${JSON.stringify(LEGACY_MIGRATION_REFUSAL)}; -if (kind === "legacy-refuse" || kind === "startup-legacy-refuse") { process.stderr.write(legacyRefusal + "\\n"); process.exit(78); } +const legacyRefusal = kind.startsWith("startup-") ? ${JSON.stringify(LEGACY_STARTUP_FAILURE)} : kind.startsWith("profile-") ? ${JSON.stringify(PROFILED_LEGACY_MIGRATION_REFUSAL)} : ${JSON.stringify(LEGACY_MIGRATION_REFUSAL)}; +if (kind === "legacy-refuse" || kind === "startup-legacy-refuse" || kind === "profile-legacy-refuse") { process.stderr.write(legacyRefusal + "\\n"); process.exit(78); } if (kind === "late-legacy-refuse" || kind === "startup-late-legacy-refuse") { spawnInheritedWriter("stderr", legacyRefusal + "\\n"); process.exit(78); @@ -1359,13 +1360,16 @@ describe("openclaw test instance", () => { it.for([ "legacy-refuse", + "profile-legacy-refuse", "late-legacy-refuse", "startup-legacy-refuse", "startup-late-legacy-refuse", ])("reports a typed %s only after the child's stderr closes", async (action) => { const message = action.startsWith("startup-") ? LEGACY_STARTUP_FAILURE - : LEGACY_MIGRATION_REFUSAL; + : action === "profile-legacy-refuse" + ? PROFILED_LEGACY_MIGRATION_REFUSAL + : LEGACY_MIGRATION_REFUSAL; const control = action.includes("late-") ? await createGatewayControl() : undefined; const { instance, readAttempts } = await createFakeGateway( `${action},config-refuse`, @@ -1373,6 +1377,9 @@ describe("openclaw test instance", () => { 1_500, control, ); + if (action === "profile-legacy-refuse" || action.startsWith("startup-")) { + instance.env.OPENCLAW_PROFILE = "qa-fixture"; + } const exited = createDeferred(); if (control) { control.observers.onLaunch = () => { @@ -1411,10 +1418,14 @@ describe("openclaw test instance", () => { "legacy-stdout", "legacy-status1", "legacy-no-advice", + "profile-legacy-refuse", "startup-no-advice", "startup-warning", ])("does not classify %s as an explained legacy migration refusal", async (action) => { const { instance, readAttempts } = await createFakeGateway(action); + if (action === "profile-legacy-refuse") { + instance.env.OPENCLAW_PROFILE = "different-fixture"; + } const error = await instance.startGateway().catch((failure: unknown) => failure); expect(error).toBeInstanceOf(Error); expect(error).not.toBeInstanceOf(GatewayStartupRefusedError); diff --git a/test/helpers/openclaw-test-instance.ts b/test/helpers/openclaw-test-instance.ts index 6c2154b1109b..44790fc61140 100644 --- a/test/helpers/openclaw-test-instance.ts +++ b/test/helpers/openclaw-test-instance.ts @@ -19,6 +19,7 @@ import { loadManagedChildSpawner, terminateManagedChild, } from "../../scripts/lib/managed-child-process.mts"; +import { formatCliCommand } from "../../src/cli/command-format.js"; import { hasErrnoCode } from "../../src/infra/errno.js"; import { appendCapturedOutput, @@ -1093,13 +1094,15 @@ export async function createOpenClawTestInstance( if (closed && !signal?.aborted && exitCode === 78 && signalCode === null) { // Admission after a checkpoint and a refused migration step have // different reports; both must name the source and its repair. + const admissionRefusal = completedStderr.match( + /^(?:Gateway failed to start: )?Legacy session store requires migration: (.+)\. Run "([^"\r\n]+)" against the same state\/config before starting OpenClaw\.\r?$/mu, + ); const legacyStorePath = - completedStderr.match( - /^(?:Gateway failed to start: )?Legacy session store requires migration: (.+)\. Run "openclaw doctor --fix" against the same state\/config before starting OpenClaw\.\r?$/mu, - )?.[1] ?? - completedStderr.match( - /^OpenClaw startup migrations did not complete cleanly; refusing to report the gateway ready\.\r?\n- Legacy sessions store unreadable; left in place at ([^\r\n]+)\r?\n(?:- [^\r\n]+\r?\n)*Run "openclaw doctor --fix" against the same state\/config, then restart the gateway\.\r?$/mu, - )?.[1]; + admissionRefusal?.[2] === formatCliCommand("openclaw doctor --fix", env) + ? admissionRefusal[1] + : completedStderr.match( + /^OpenClaw startup migrations did not complete cleanly; refusing to report the gateway ready\.\r?\n- Legacy sessions store unreadable; left in place at ([^\r\n]+)\r?\n(?:- [^\r\n]+\r?\n)*Run "openclaw doctor --fix" against the same state\/config, then restart the gateway\.\r?$/mu, + )?.[1]; if (legacyStorePath) { throw new GatewayStartupRefusedError( legacyStorePath, diff --git a/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.test.ts b/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.test.ts new file mode 100644 index 000000000000..6cc8cd22aafc --- /dev/null +++ b/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { withEnv } from "../../src/test-utils/env.js"; +import { assertSqliteFlipStartupRefusal } from "./sqlite-sessions-transcripts-flip-proof-assertions.js"; + +function startupRefusal(command: string) { + return { + message: `gateway refused startup: legacy migration required (code=78 signal=null) +Legacy session store requires migration: /qa/state/sessions/sessions.json. Run "${command}" against the same state/config before starting OpenClaw.`, + preservedSourceFiles: [ + "agents/main/sessions/sessions.json", + "agents/main/sessions/archive-fixture/cold-archive.jsonl", + "sessions/sessions.json", + ], + }; +} + +describe("SQLite flip proof startup refusal assertions", () => { + it.each([ + { label: "unprofiled", profile: undefined, command: "openclaw doctor --fix" }, + { + label: "profile-qualified", + profile: "qa-sqlite-proof", + command: "openclaw --profile qa-sqlite-proof doctor --fix", + }, + ])("accepts $label guidance with preserved legacy sources", ({ profile, command }) => { + withEnv({ OPENCLAW_PROFILE: profile, OPENCLAW_CONTAINER_HINT: undefined }, () => { + expect(() => assertSqliteFlipStartupRefusal(startupRefusal(command))).not.toThrow(); + }); + }); + + it.each(["openclaw doctor --fix", "openclaw --profile unrelated doctor --fix"])( + "rejects guidance outside the active profile: %s", + (command) => { + withEnv({ OPENCLAW_PROFILE: "qa-sqlite-proof", OPENCLAW_CONTAINER_HINT: undefined }, () => { + const refusal = startupRefusal(command); + expect(() => assertSqliteFlipStartupRefusal(refusal)).toThrow( + expect.objectContaining({ + actual: refusal.message, + expected: 'Run "openclaw --profile qa-sqlite-proof doctor --fix"', + }), + ); + }); + }, + ); + + it("rejects valid guidance when a legacy source was not preserved", () => { + withEnv({ OPENCLAW_PROFILE: undefined, OPENCLAW_CONTAINER_HINT: undefined }, () => { + const refusal = startupRefusal("openclaw doctor --fix"); + refusal.preservedSourceFiles.pop(); + expect(() => assertSqliteFlipStartupRefusal(refusal)).toThrow( + expect.objectContaining({ actual: refusal.preservedSourceFiles }), + ); + }); + }); +}); diff --git a/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.ts b/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.ts index 99ce13bbf3f2..1fd54ae58f70 100644 --- a/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.ts +++ b/test/helpers/sqlite-sessions-transcripts-flip-proof-assertions.ts @@ -1,9 +1,23 @@ import { asOptionalRecord as asRecord } from "@openclaw/normalization-core/record-coerce"; import { expect } from "vitest"; +import { formatCliCommand } from "../../src/cli/command-format.js"; import type { runSqliteSessionsTranscriptsFlipProof } from "./sqlite-sessions-transcripts-flip-proof.ts"; type SqliteFlipProofReport = Awaited>; +export function assertSqliteFlipStartupRefusal( + refusal: SqliteFlipProofReport["startupRefusal"], +): void { + expect(refusal?.message).toContain(`Run "${formatCliCommand("openclaw doctor --fix")}"`); + expect(refusal?.preservedSourceFiles.map((filePath) => filePath.replaceAll("\\", "/"))).toEqual( + expect.arrayContaining([ + "agents/main/sessions/sessions.json", + "agents/main/sessions/archive-fixture/cold-archive.jsonl", + "sessions/sessions.json", + ]), + ); +} + export function assertSqliteFlipProofCore(report: SqliteFlipProofReport): void { expect(report.failures).toEqual([]); expect(report.ok).toBe(true); @@ -13,16 +27,7 @@ export function assertSqliteFlipProofCore(report: SqliteFlipProofReport): void { const refusalCheckpoint = report.checkpoints.find( (checkpoint) => checkpoint.label === "after-startup-refusal", ); - expect(report.startupRefusal?.message).toContain('Run "openclaw doctor --fix"'); - expect( - report.startupRefusal?.preservedSourceFiles.map((filePath) => filePath.replaceAll("\\", "/")), - ).toEqual( - expect.arrayContaining([ - "agents/main/sessions/sessions.json", - "agents/main/sessions/archive-fixture/cold-archive.jsonl", - "sessions/sessions.json", - ]), - ); + assertSqliteFlipStartupRefusal(report.startupRefusal); expect(refusalCheckpoint?.activeJsonl).toEqual(seededCheckpoint?.activeJsonl); expect(refusalCheckpoint?.legacyStateJsonl).toEqual(seededCheckpoint?.legacyStateJsonl); expect(refusalCheckpoint?.sqlite.sessionEntries).toBe(seededCheckpoint?.sqlite.sessionEntries);