From 9666db607eb5a52b4db7eaf7343c9db89ef14477 Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 24 Jun 2026 00:43:00 -0700 Subject: [PATCH] test(qa): clean up smoke taxonomy profile (#96320) --- docs/concepts/qa-e2e-automation.md | 2 +- extensions/qa-lab/src/cli.runtime.test.ts | 10 +++++----- extensions/qa-lab/src/cli.test.ts | 10 +++++----- .../runtime/docker-prometheus-smoke.yaml | 2 +- qa/scenarios/runtime/first-hour-20-turn.yaml | 2 +- qa/scenarios/runtime/soak-100-turn.yaml | 2 +- taxonomy.yaml | 16 ++++++---------- 7 files changed, 20 insertions(+), 24 deletions(-) diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 68aede00fda..ee36ad8915c 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -68,7 +68,7 @@ Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`; ```bash pnpm openclaw qa run \ --qa-profile smoke-ci \ - --category agent-runtime-and-provider-execution.agent-turn-execution \ + --category channel-framework.conversation-routing-and-delivery \ --provider-mode mock-openai \ --output-dir .artifacts/qa-e2e/smoke-ci-profile-dispatch ``` diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index a25276c0dbe..9d31914843d 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -431,8 +431,8 @@ describe("qa cli runtime", () => { repoRoot: "/tmp/openclaw-repo", outputDir: ".artifacts/qa-e2e/smoke-ci", profile: "smoke-ci", - surface: "agent-runtime-and-provider-execution", - category: "agent-runtime-and-provider-execution.agent-turn-execution", + surface: "channel-framework", + category: "channel-framework.conversation-routing-and-delivery", scenarioIds: ["dm-chat-baseline"], transportId: "qa-channel", fastMode: true, @@ -482,7 +482,7 @@ describe("qa cli runtime", () => { expect(evidence.scorecard).not.toHaveProperty("profile"); expect(evidence.scorecard?.features?.fulfilled).toBe(0); expect(evidence.scorecard?.categoryReports?.[0]).toMatchObject({ - id: "agent-runtime-and-provider-execution.agent-turn-execution", + id: "channel-framework.conversation-routing-and-delivery", features: { fulfilled: 0, }, @@ -595,11 +595,11 @@ describe("qa cli runtime", () => { runQaProfileCommand({ repoRoot: "/tmp/openclaw-repo", profile: "smoke-ci", - category: "agent-runtime-and-provider-execution.agent-turn-execution", + category: "channel-framework.conversation-routing-and-delivery", scenarioIds: ["not-a-real-scenario"], }), ).rejects.toThrow( - "qa run did not find taxonomy scenarios for --qa-profile smoke-ci --category agent-runtime-and-provider-execution.agent-turn-execution --scenario not-a-real-scenario.", + "qa run did not find taxonomy scenarios for --qa-profile smoke-ci --category channel-framework.conversation-routing-and-delivery --scenario not-a-real-scenario.", ); expect(runQaSuite).not.toHaveBeenCalled(); }); diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index e9ed4e43fff..380c9f984ae 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -214,9 +214,9 @@ describe("qa cli registration", () => { "--qa-profile", "smoke-ci", "--surface", - "agent-runtime-and-provider-execution", + "channel-framework", "--category", - "agent-runtime-and-provider-execution.agent-turn-execution", + "channel-framework.conversation-routing-and-delivery", "--scenario", "dm-chat-baseline", "--evidence-mode", @@ -239,8 +239,8 @@ describe("qa cli registration", () => { repoRoot: "/tmp/openclaw-repo", outputDir: ".artifacts/qa-e2e/smoke-ci", profile: "smoke-ci", - surface: "agent-runtime-and-provider-execution", - category: "agent-runtime-and-provider-execution.agent-turn-execution", + surface: "channel-framework", + category: "channel-framework.conversation-routing-and-delivery", scenarioIds: ["dm-chat-baseline"], evidenceMode: "slim", transportId: "qa-channel", @@ -257,7 +257,7 @@ describe("qa cli registration", () => { it.each([ ["--output-dir", [".artifacts/qa-e2e/smoke-ci"]], ["--surface", ["agent-runtime-and-provider-execution"]], - ["--category", ["agent-runtime-and-provider-execution.agent-turn-execution"]], + ["--category", ["channel-framework.conversation-routing-and-delivery"]], ["--scenario", ["dm-chat-baseline"]], ["--evidence-mode", ["slim"]], ["--exclude-test-execution-evidence", []], diff --git a/qa/scenarios/runtime/docker-prometheus-smoke.yaml b/qa/scenarios/runtime/docker-prometheus-smoke.yaml index 221039a62df..a5c080480c2 100644 --- a/qa/scenarios/runtime/docker-prometheus-smoke.yaml +++ b/qa/scenarios/runtime/docker-prometheus-smoke.yaml @@ -9,7 +9,7 @@ scenario: - telemetry.prometheus-authenticated-gateway-export secondary: - harness.qa-lab - - docker.e2e + - docker.runtime-validation objective: Verify a QA-lab gateway run emits protected, bounded Prometheus diagnostics metrics through the diagnostics-prometheus plugin. successCriteria: - The diagnostics-prometheus plugin exposes the protected scrape route. diff --git a/qa/scenarios/runtime/first-hour-20-turn.yaml b/qa/scenarios/runtime/first-hour-20-turn.yaml index fa502b547bb..96130baf824 100644 --- a/qa/scenarios/runtime/first-hour-20-turn.yaml +++ b/qa/scenarios/runtime/first-hour-20-turn.yaml @@ -6,7 +6,7 @@ scenario: runtimeParityTier: standard coverage: primary: - - runtime.first-hour-20 + - runtime.multi-turn-continuity secondary: - runtime.long-context objective: Verify both runtimes preserve a same-session conversation across the required 20-turn maintainer gate. diff --git a/qa/scenarios/runtime/soak-100-turn.yaml b/qa/scenarios/runtime/soak-100-turn.yaml index f2988c8c17e..298810cbb6e 100644 --- a/qa/scenarios/runtime/soak-100-turn.yaml +++ b/qa/scenarios/runtime/soak-100-turn.yaml @@ -6,7 +6,7 @@ scenario: runtimeParityTier: soak coverage: primary: - - runtime.soak-100 + - runtime.long-run-stability secondary: - runtime.long-context objective: Provide an optional long-run soak that can be scheduled or run in Testbox without entering the maintainer default gate. diff --git a/taxonomy.yaml b/taxonomy.yaml index d9701f1e12f..0287e1a078e 100644 --- a/taxonomy.yaml +++ b/taxonomy.yaml @@ -12,13 +12,11 @@ profiles: evidenceMode: slim channelDriver: crabline categoryIds: - - agent-runtime-and-provider-execution.agent-turn-execution - agent-runtime-and-provider-execution.model-and-runtime-selection - agent-runtime-and-provider-execution.provider-auth - agent-runtime-and-provider-execution.streaming-and-progress - agent-runtime-and-provider-execution.tool-calls-and-response-handling - agent-runtime-and-provider-execution.tool-execution-controls - - session-memory-and-context-engine.token-management - session-memory-and-context-engine.context-engine - session-memory-and-context-engine.cross-client-history-and-session-parity - session-memory-and-context-engine.core-prompts-and-context @@ -36,7 +34,6 @@ profiles: - channel-framework.outbound-delivery-and-reply-pipeline - channel-framework.group-thread-and-ambient-room-behavior - channel-framework.status-health-and-operator-controls - - session-memory-and-context-engine.memory - session-memory-and-context-engine.diagnostics-maintenance-and-recovery - automation-cron-hooks-tasks-polling.cron-jobs - plugin-sdk-and-bundled-plugin-architecture.installing-and-running-plugins @@ -45,7 +42,6 @@ profiles: - media-understanding-and-media-generation.media-understanding - media-understanding-and-media-generation.media-generation - browser-control-ui-and-webchat.browser-ui - - security-auth-pairing-and-secrets.credential-and-secret-hygiene - id: release description: Stable/LTS proof selector for live providers, live channels, package artifacts, upgrade paths, and platform proof where the claim depends on real upstreams or release @@ -1305,13 +1301,13 @@ surfaces: id: agent-turn-execution features: - name: Turn startup and runtime choice - coverageIds: [agents.create, agents.instructions, channels.discord-config, config.crestodian-setup, runtime.first-action, runtime.first-hour-20, runtime.long-context] + coverageIds: [agents.create, agents.instructions, channels.discord-config, config.crestodian-setup, runtime.first-action, runtime.multi-turn-continuity, runtime.long-context] description: Starting an agent turn and choosing gateway versus embedded runtime execution. - name: Session and run coordination coverageIds: [agents.subagents, channels.dedup, channels.dm, channels.qa-channel, channels.reconnect, channels.streaming, channels.threads, commitments.heartbeat-target-none, commitments.scope, personal.channel-replies, runtime.codex-plugin.lifecycle, runtime.delivery, runtime.fallback-delivery, runtime.gateway-restart, runtime.restart-recovery, runtime.turn-ordering] description: Establishing session and run ids, queue locks, and related execution coordination. - name: Abort and terminal outcomes - coverageIds: [channels.streaming, runtime.delivery, runtime.fallback-delivery, runtime.long-context, runtime.soak-100] + coverageIds: [channels.streaming, runtime.delivery, runtime.fallback-delivery, runtime.long-context, runtime.long-run-stability] description: Honoring aborts, timing provider/model work, and emitting terminal outcomes. docs: - docs/concepts/agent-loop.md @@ -1593,7 +1589,7 @@ surfaces: coverageIds: [session.pruning] description: Covers Pruning across manual and automatic compaction, preemptive overflow checks, context-window estimation, session pruning, tool-result trimming, compaction providers, retry/timeout behavior, and compacted transcript checkpoints. - name: Token Pressure - coverageIds: [runtime.codex-app-server, runtime.first-hour-20, runtime.gateway-log-sentinel.codex-progress, runtime.long-context, runtime.soak-100] + coverageIds: [runtime.codex-app-server, runtime.multi-turn-continuity, runtime.gateway-log-sentinel.codex-progress, runtime.long-context, runtime.long-run-stability] description: Covers Token Pressure across manual and automatic compaction, preemptive overflow checks, context-window estimation, session pruning, tool-result trimming, compaction providers, retry/timeout behavior, and compacted transcript checkpoints. docs: - docs/concepts/compaction.md @@ -2418,7 +2414,7 @@ surfaces: coverageIds: [telemetry.trusted-trace-context] description: Trusted trace context, W3C traceparent propagation to model calls, file-log correlation, content-capture controls, and redacted/bounded attributes - name: Model and runtime telemetry - coverageIds: [docker.e2e, harness.qa-lab, harness.tool-trace-visibility, personal.failure-recovery, personal.no-fake-progress, personal.task-followthrough, runtime.qa-bus, telemetry.otel, telemetry.prometheus, tools.evidence, tools.trace] + coverageIds: [docker.runtime-validation, harness.qa-lab, harness.tool-trace-visibility, personal.failure-recovery, personal.no-fake-progress, personal.task-followthrough, runtime.qa-bus, telemetry.otel, telemetry.prometheus, tools.evidence, tools.trace] description: Model, tool, message, session, queue, Talk, exec, webhook, context assembly, harness, and exporter-health signals - name: diagnostics-prometheus plugin install coverageIds: [telemetry.diagnostics-prometheus-plugin-install] @@ -2427,7 +2423,7 @@ surfaces: coverageIds: [telemetry.prometheus-authenticated-gateway-export] description: Gateway-authenticated GET /api/diagnostics/prometheus behavior, status, and operator-visible verification. - name: Prometheus text exposition - coverageIds: [docker.e2e, harness.qa-lab, telemetry.prometheus] + coverageIds: [docker.runtime-validation, harness.qa-lab, telemetry.prometheus] description: Prometheus text exposition, counters, gauges, histograms, label policy, series cap, and overflow metric - name: Trusted diagnostic event subscription coverageIds: [telemetry.trusted-diagnostic-event-subscription] @@ -6445,7 +6441,7 @@ surfaces: coverageIds: [docker.package-artifact-generation] description: Docker E2E package artifact generation and shared build helpers - name: Docker E2E plan/scheduler scripts - coverageIds: [docker.e2e, harness.qa-lab, telemetry.prometheus] + coverageIds: [docker.runtime-validation, harness.qa-lab, telemetry.prometheus] description: Docker E2E plan/scheduler scripts, lane metadata, targeted grouping, package artifact generation, and GitHub hydration action - name: Release-path install coverageIds: [docker.release-path-install]