mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-10 00:11:19 +00:00
test(qa): clean up smoke taxonomy profile (#96320)
This commit is contained in:
Dallin Romney
GitHub
parent
9773cbafdb
commit
9666db607e
7 changed files with 20 additions and 24 deletions
|
|
@ -68,7 +68,7 @@ Slim evidence omits per-entry `execution` and sets `evidenceMode: "slim"`;
|
|||
```bash
|
||||
pnpm openclaw qa run \
|
||||
--qa-profile smoke-ci \
|
||||
--category agent-runtime-and-provider-execution.agent-turn-execution \
|
||||
--category channel-framework.conversation-routing-and-delivery \
|
||||
--provider-mode mock-openai \
|
||||
--output-dir .artifacts/qa-e2e/smoke-ci-profile-dispatch
|
||||
```
|
||||
|
|
|
|||
|
|
@ -431,8 +431,8 @@ describe("qa cli runtime", () => {
|
|||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa-e2e/smoke-ci",
|
||||
profile: "smoke-ci",
|
||||
surface: "agent-runtime-and-provider-execution",
|
||||
category: "agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
surface: "channel-framework",
|
||||
category: "channel-framework.conversation-routing-and-delivery",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
transportId: "qa-channel",
|
||||
fastMode: true,
|
||||
|
|
@ -482,7 +482,7 @@ describe("qa cli runtime", () => {
|
|||
expect(evidence.scorecard).not.toHaveProperty("profile");
|
||||
expect(evidence.scorecard?.features?.fulfilled).toBe(0);
|
||||
expect(evidence.scorecard?.categoryReports?.[0]).toMatchObject({
|
||||
id: "agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
id: "channel-framework.conversation-routing-and-delivery",
|
||||
features: {
|
||||
fulfilled: 0,
|
||||
},
|
||||
|
|
@ -595,11 +595,11 @@ describe("qa cli runtime", () => {
|
|||
runQaProfileCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
profile: "smoke-ci",
|
||||
category: "agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
category: "channel-framework.conversation-routing-and-delivery",
|
||||
scenarioIds: ["not-a-real-scenario"],
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
"qa run did not find taxonomy scenarios for --qa-profile smoke-ci --category agent-runtime-and-provider-execution.agent-turn-execution --scenario not-a-real-scenario.",
|
||||
"qa run did not find taxonomy scenarios for --qa-profile smoke-ci --category channel-framework.conversation-routing-and-delivery --scenario not-a-real-scenario.",
|
||||
);
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -214,9 +214,9 @@ describe("qa cli registration", () => {
|
|||
"--qa-profile",
|
||||
"smoke-ci",
|
||||
"--surface",
|
||||
"agent-runtime-and-provider-execution",
|
||||
"channel-framework",
|
||||
"--category",
|
||||
"agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
"channel-framework.conversation-routing-and-delivery",
|
||||
"--scenario",
|
||||
"dm-chat-baseline",
|
||||
"--evidence-mode",
|
||||
|
|
@ -239,8 +239,8 @@ describe("qa cli registration", () => {
|
|||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa-e2e/smoke-ci",
|
||||
profile: "smoke-ci",
|
||||
surface: "agent-runtime-and-provider-execution",
|
||||
category: "agent-runtime-and-provider-execution.agent-turn-execution",
|
||||
surface: "channel-framework",
|
||||
category: "channel-framework.conversation-routing-and-delivery",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
evidenceMode: "slim",
|
||||
transportId: "qa-channel",
|
||||
|
|
@ -257,7 +257,7 @@ describe("qa cli registration", () => {
|
|||
it.each([
|
||||
["--output-dir", [".artifacts/qa-e2e/smoke-ci"]],
|
||||
["--surface", ["agent-runtime-and-provider-execution"]],
|
||||
["--category", ["agent-runtime-and-provider-execution.agent-turn-execution"]],
|
||||
["--category", ["channel-framework.conversation-routing-and-delivery"]],
|
||||
["--scenario", ["dm-chat-baseline"]],
|
||||
["--evidence-mode", ["slim"]],
|
||||
["--exclude-test-execution-evidence", []],
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ scenario:
|
|||
- telemetry.prometheus-authenticated-gateway-export
|
||||
secondary:
|
||||
- harness.qa-lab
|
||||
- docker.e2e
|
||||
- docker.runtime-validation
|
||||
objective: Verify a QA-lab gateway run emits protected, bounded Prometheus diagnostics metrics through the diagnostics-prometheus plugin.
|
||||
successCriteria:
|
||||
- The diagnostics-prometheus plugin exposes the protected scrape route.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ scenario:
|
|||
runtimeParityTier: standard
|
||||
coverage:
|
||||
primary:
|
||||
- runtime.first-hour-20
|
||||
- runtime.multi-turn-continuity
|
||||
secondary:
|
||||
- runtime.long-context
|
||||
objective: Verify both runtimes preserve a same-session conversation across the required 20-turn maintainer gate.
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ scenario:
|
|||
runtimeParityTier: soak
|
||||
coverage:
|
||||
primary:
|
||||
- runtime.soak-100
|
||||
- runtime.long-run-stability
|
||||
secondary:
|
||||
- runtime.long-context
|
||||
objective: Provide an optional long-run soak that can be scheduled or run in Testbox without entering the maintainer default gate.
|
||||
|
|
|
|||
|
|
@ -12,13 +12,11 @@ profiles:
|
|||
evidenceMode: slim
|
||||
channelDriver: crabline
|
||||
categoryIds:
|
||||
- agent-runtime-and-provider-execution.agent-turn-execution
|
||||
- agent-runtime-and-provider-execution.model-and-runtime-selection
|
||||
- agent-runtime-and-provider-execution.provider-auth
|
||||
- agent-runtime-and-provider-execution.streaming-and-progress
|
||||
- agent-runtime-and-provider-execution.tool-calls-and-response-handling
|
||||
- agent-runtime-and-provider-execution.tool-execution-controls
|
||||
- session-memory-and-context-engine.token-management
|
||||
- session-memory-and-context-engine.context-engine
|
||||
- session-memory-and-context-engine.cross-client-history-and-session-parity
|
||||
- session-memory-and-context-engine.core-prompts-and-context
|
||||
|
|
@ -36,7 +34,6 @@ profiles:
|
|||
- channel-framework.outbound-delivery-and-reply-pipeline
|
||||
- channel-framework.group-thread-and-ambient-room-behavior
|
||||
- channel-framework.status-health-and-operator-controls
|
||||
- session-memory-and-context-engine.memory
|
||||
- session-memory-and-context-engine.diagnostics-maintenance-and-recovery
|
||||
- automation-cron-hooks-tasks-polling.cron-jobs
|
||||
- plugin-sdk-and-bundled-plugin-architecture.installing-and-running-plugins
|
||||
|
|
@ -45,7 +42,6 @@ profiles:
|
|||
- media-understanding-and-media-generation.media-understanding
|
||||
- media-understanding-and-media-generation.media-generation
|
||||
- browser-control-ui-and-webchat.browser-ui
|
||||
- security-auth-pairing-and-secrets.credential-and-secret-hygiene
|
||||
- id: release
|
||||
description: Stable/LTS proof selector for live providers, live channels, package artifacts,
|
||||
upgrade paths, and platform proof where the claim depends on real upstreams or release
|
||||
|
|
@ -1305,13 +1301,13 @@ surfaces:
|
|||
id: agent-turn-execution
|
||||
features:
|
||||
- name: Turn startup and runtime choice
|
||||
coverageIds: [agents.create, agents.instructions, channels.discord-config, config.crestodian-setup, runtime.first-action, runtime.first-hour-20, runtime.long-context]
|
||||
coverageIds: [agents.create, agents.instructions, channels.discord-config, config.crestodian-setup, runtime.first-action, runtime.multi-turn-continuity, runtime.long-context]
|
||||
description: Starting an agent turn and choosing gateway versus embedded runtime execution.
|
||||
- name: Session and run coordination
|
||||
coverageIds: [agents.subagents, channels.dedup, channels.dm, channels.qa-channel, channels.reconnect, channels.streaming, channels.threads, commitments.heartbeat-target-none, commitments.scope, personal.channel-replies, runtime.codex-plugin.lifecycle, runtime.delivery, runtime.fallback-delivery, runtime.gateway-restart, runtime.restart-recovery, runtime.turn-ordering]
|
||||
description: Establishing session and run ids, queue locks, and related execution coordination.
|
||||
- name: Abort and terminal outcomes
|
||||
coverageIds: [channels.streaming, runtime.delivery, runtime.fallback-delivery, runtime.long-context, runtime.soak-100]
|
||||
coverageIds: [channels.streaming, runtime.delivery, runtime.fallback-delivery, runtime.long-context, runtime.long-run-stability]
|
||||
description: Honoring aborts, timing provider/model work, and emitting terminal outcomes.
|
||||
docs:
|
||||
- docs/concepts/agent-loop.md
|
||||
|
|
@ -1593,7 +1589,7 @@ surfaces:
|
|||
coverageIds: [session.pruning]
|
||||
description: Covers Pruning across manual and automatic compaction, preemptive overflow checks, context-window estimation, session pruning, tool-result trimming, compaction providers, retry/timeout behavior, and compacted transcript checkpoints.
|
||||
- name: Token Pressure
|
||||
coverageIds: [runtime.codex-app-server, runtime.first-hour-20, runtime.gateway-log-sentinel.codex-progress, runtime.long-context, runtime.soak-100]
|
||||
coverageIds: [runtime.codex-app-server, runtime.multi-turn-continuity, runtime.gateway-log-sentinel.codex-progress, runtime.long-context, runtime.long-run-stability]
|
||||
description: Covers Token Pressure across manual and automatic compaction, preemptive overflow checks, context-window estimation, session pruning, tool-result trimming, compaction providers, retry/timeout behavior, and compacted transcript checkpoints.
|
||||
docs:
|
||||
- docs/concepts/compaction.md
|
||||
|
|
@ -2418,7 +2414,7 @@ surfaces:
|
|||
coverageIds: [telemetry.trusted-trace-context]
|
||||
description: Trusted trace context, W3C traceparent propagation to model calls, file-log correlation, content-capture controls, and redacted/bounded attributes
|
||||
- name: Model and runtime telemetry
|
||||
coverageIds: [docker.e2e, harness.qa-lab, harness.tool-trace-visibility, personal.failure-recovery, personal.no-fake-progress, personal.task-followthrough, runtime.qa-bus, telemetry.otel, telemetry.prometheus, tools.evidence, tools.trace]
|
||||
coverageIds: [docker.runtime-validation, harness.qa-lab, harness.tool-trace-visibility, personal.failure-recovery, personal.no-fake-progress, personal.task-followthrough, runtime.qa-bus, telemetry.otel, telemetry.prometheus, tools.evidence, tools.trace]
|
||||
description: Model, tool, message, session, queue, Talk, exec, webhook, context assembly, harness, and exporter-health signals
|
||||
- name: diagnostics-prometheus plugin install
|
||||
coverageIds: [telemetry.diagnostics-prometheus-plugin-install]
|
||||
|
|
@ -2427,7 +2423,7 @@ surfaces:
|
|||
coverageIds: [telemetry.prometheus-authenticated-gateway-export]
|
||||
description: Gateway-authenticated GET /api/diagnostics/prometheus behavior, status, and operator-visible verification.
|
||||
- name: Prometheus text exposition
|
||||
coverageIds: [docker.e2e, harness.qa-lab, telemetry.prometheus]
|
||||
coverageIds: [docker.runtime-validation, harness.qa-lab, telemetry.prometheus]
|
||||
description: Prometheus text exposition, counters, gauges, histograms, label policy, series cap, and overflow metric
|
||||
- name: Trusted diagnostic event subscription
|
||||
coverageIds: [telemetry.trusted-diagnostic-event-subscription]
|
||||
|
|
@ -6445,7 +6441,7 @@ surfaces:
|
|||
coverageIds: [docker.package-artifact-generation]
|
||||
description: Docker E2E package artifact generation and shared build helpers
|
||||
- name: Docker E2E plan/scheduler scripts
|
||||
coverageIds: [docker.e2e, harness.qa-lab, telemetry.prometheus]
|
||||
coverageIds: [docker.runtime-validation, harness.qa-lab, telemetry.prometheus]
|
||||
description: Docker E2E plan/scheduler scripts, lane metadata, targeted grouping, package artifact generation, and GitHub hydration action
|
||||
- name: Release-path install
|
||||
coverageIds: [docker.release-path-install]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue