From 5db2f6c1fccb2bc36acd5c55f92fb835d040c315 Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Fri, 19 Jun 2026 16:06:37 +1000 Subject: [PATCH] Add stdout diagnostics OTEL log exporter Adds stdout and both-mode diagnostics OTEL log export, with focused QA Lab smoke coverage and docs/config updates. Prepared head SHA: efa2ef07ab68c5b10b68ffb9d75bad68ce60a68a Verification: CI 27808480969 passed for the prepared head. Reviewed-by: @jesse-merhi --- docs/.generated/config-baseline.sha256 | 6 +- docs/gateway/configuration-reference.md | 2 + docs/gateway/opentelemetry.md | 24 +- docs/logging.md | 6 +- docs/plugins/plugin-inventory.md | 2 +- docs/plugins/reference/diagnostics-otel.md | 4 +- extensions/diagnostics-otel/README.md | 2 +- .../diagnostics-otel/openclaw.plugin.json | 2 +- extensions/diagnostics-otel/package.json | 2 +- .../diagnostics-otel/src/service.test.ts | 255 +++++++++++++- extensions/diagnostics-otel/src/service.ts | 311 +++++++++++------- .../qa-lab/src/scenario-catalog.test.ts | 2 + extensions/qa-lab/src/scenario-catalog.ts | 1 + extensions/qa-lab/src/suite-planning.test.ts | 5 + extensions/qa-lab/src/suite-planning.ts | 11 +- extensions/qa-lab/src/suite-test-helpers.ts | 2 +- extensions/qa-lab/src/suite.ts | 5 +- qa/scenarios/runtime/otel-both-log-smoke.yaml | 103 ++++++ .../runtime/otel-stdout-log-smoke.yaml | 103 ++++++ src/config/config-misc.test.ts | 24 ++ src/config/schema.help.quality.test.ts | 2 + src/config/schema.help.ts | 2 + src/config/schema.labels.ts | 1 + src/config/types.base.ts | 2 + src/config/zod-schema.ts | 3 + .../qa-lab/runtime/qa-otel-smoke-runtime.ts | 265 ++++++++++++++- .../qa-lab/runtime/qa-otel-smoke.e2e.test.ts | 141 +++++++- 27 files changed, 1133 insertions(+), 155 deletions(-) create mode 100644 qa/scenarios/runtime/otel-both-log-smoke.yaml create mode 100644 qa/scenarios/runtime/otel-stdout-log-smoke.yaml diff --git a/docs/.generated/config-baseline.sha256 b/docs/.generated/config-baseline.sha256 index ec1d9e27e6e..cb7cf440161 100644 --- a/docs/.generated/config-baseline.sha256 +++ b/docs/.generated/config-baseline.sha256 @@ -1,4 +1,4 @@ -e78623d6eace69e46950cd5d9a5cf14aa910dac1ecdf9d054a0bd9999e936061 config-baseline.json -5ecafa3c9a59fc0675f964f6e3238b2f20625376ebad1835278c5dd7323770d3 config-baseline.core.json +ac06b6c20a93a8543ec1bd3748ef4f7bdae5006839dd93b3fff874d0da4244aa config-baseline.json +e7965566fdaedef445bcd562141f4f3ea1a499cf8ea5956418af7c98049bf242 config-baseline.core.json 2d735389858305509528e74329b6f8c65d311e1471c3b4e91dc17aaab8e63a80 config-baseline.channel.json -7c2c51b795d32e4c4c325080d59fec8fd11317c41db7db642f70e436779738bc config-baseline.plugin.json +0039da0cf2ba2845b37db52c4cf3a0f25e367cf3d2d507c5d6f8a5e5bdfdc4d4 config-baseline.plugin.json diff --git a/docs/gateway/configuration-reference.md b/docs/gateway/configuration-reference.md index adf0e55e7ef..47908f37f92 100644 --- a/docs/gateway/configuration-reference.md +++ b/docs/gateway/configuration-reference.md @@ -1096,6 +1096,7 @@ Notes: traces: true, metrics: true, logs: false, + logsExporter: "otlp", sampleRate: 1.0, flushIntervalMs: 5000, captureContent: { @@ -1132,6 +1133,7 @@ Notes: - `otel.headers`: extra HTTP/gRPC metadata headers sent with OTel export requests. - `otel.serviceName`: service name for resource attributes. - `otel.traces` / `otel.metrics` / `otel.logs`: enable trace, metrics, or log export. +- `otel.logsExporter`: log export sink: `"otlp"` (default), `"stdout"` for one JSON object per stdout line, or `"both"`. - `otel.sampleRate`: trace sampling rate `0`-`1`. - `otel.flushIntervalMs`: periodic telemetry flush interval in ms. - `otel.captureContent`: opt-in raw content capture for OTEL span attributes. Defaults to off. Boolean `true` captures non-system message/tool content; the object form lets you enable `inputMessages`, `outputMessages`, `toolInputs`, `toolOutputs`, `systemPrompt`, and `toolDefinitions` explicitly. diff --git a/docs/gateway/opentelemetry.md b/docs/gateway/opentelemetry.md index 777cf4e084c..f6797687d76 100644 --- a/docs/gateway/opentelemetry.md +++ b/docs/gateway/opentelemetry.md @@ -1,5 +1,5 @@ --- -summary: "Export OpenClaw diagnostics to any OpenTelemetry collector via the diagnostics-otel plugin (OTLP/HTTP)" +summary: "Export OpenClaw diagnostics to OpenTelemetry collectors or stdout JSONL via the diagnostics-otel plugin" title: "OpenTelemetry export" read_when: - You want to send OpenClaw model usage, message flow, or session metrics to an OpenTelemetry collector @@ -8,9 +8,10 @@ read_when: --- OpenClaw exports diagnostics through the official `diagnostics-otel` plugin -using **OTLP/HTTP (protobuf)**. Any collector or backend that accepts OTLP/HTTP -works without code changes. For local file logs and how to read them, see -[Logging](/logging). +using **OTLP/HTTP (protobuf)**. Logs can also be written as stdout JSONL for +container and sandbox log pipelines. Any collector or backend that accepts +OTLP/HTTP works without code changes. For local file logs and how to read them, +see [Logging](/logging). ## How it fits together @@ -18,7 +19,8 @@ works without code changes. For local file logs and how to read them, see Gateway and bundled plugins for model runs, message flow, sessions, queues, and exec. - **`diagnostics-otel` plugin** subscribes to those events and exports them as - OpenTelemetry **metrics**, **traces**, and **logs** over OTLP/HTTP. + OpenTelemetry **metrics**, **traces**, and **logs** over OTLP/HTTP. It can + also mirror diagnostic log records to stdout JSONL. - **Provider calls** receive a W3C `traceparent` header from OpenClaw's trusted model-call span context when the provider transport accepts custom headers. Plugin-emitted trace context is not propagated. @@ -74,11 +76,13 @@ openclaw plugins enable diagnostics-otel | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | **Metrics** | Counters and histograms for token usage, cost, run duration, failover, skill usage, message flow, Talk events, queue lanes, session state/recovery, tool execution, oversized payloads, exec, and memory pressure. | | **Traces** | Spans for model usage, model calls, harness lifecycle, skill usage, tool execution, exec, webhook/message processing, context assembly, and tool loops. | -| **Logs** | Structured `logging.file` records exported over OTLP when `diagnostics.otel.logs` is enabled; log bodies are withheld unless content capture is explicitly enabled. | +| **Logs** | Structured `logging.file` records exported over OTLP or stdout JSONL when `diagnostics.otel.logs` is enabled; log bodies are withheld unless content capture is explicitly enabled. | Toggle `traces`, `metrics`, and `logs` independently. Traces and metrics default to on when `diagnostics.otel.enabled` is true. Logs default to off and -are exported only when `diagnostics.otel.logs` is explicitly `true`. +are exported only when `diagnostics.otel.logs` is explicitly `true`. Log export +defaults to OTLP; set `diagnostics.otel.logsExporter` to `stdout` for JSONL on +stdout, or `both` to send each diagnostic log record to OTLP and stdout. ## Configuration reference @@ -98,6 +102,7 @@ are exported only when `diagnostics.otel.logs` is explicitly `true`. traces: true, metrics: true, logs: true, + logsExporter: "otlp", // otlp | stdout | both sampleRate: 0.2, // root-span sampler, 0.0..1.0 flushIntervalMs: 60000, // metric export interval (min 1000ms) captureContent: { @@ -176,6 +181,11 @@ on the public diagnostic event bus. - **Logs:** OTLP logs respect `logging.level` (file log level). They use the diagnostic log-record redaction path, not console formatting. High-volume installs should prefer OTLP collector sampling/filtering over local sampling. + Set `diagnostics.otel.logsExporter: "stdout"` when your platform already + ships stdout/stderr to a log processor and you do not have an OTLP logs + collector. Stdout records are one JSON object per line with `ts`, `signal`, + `service.name`, severity, body, redacted attributes, and trusted trace fields + when available. - **File-log correlation:** JSONL file logs include top-level `traceId`, `spanId`, `parentSpanId`, and `traceFlags` when the log call carries a valid diagnostic trace context, which lets log processors join local log lines with diff --git a/docs/logging.md b/docs/logging.md index 07a1dfb05d8..43ad7d12810 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -224,8 +224,10 @@ model-call traces become children of the active request trace, so local logs, diagnostic snapshots, OTEL spans, and trusted provider `traceparent` headers can be joined by `traceId` without logging raw request or model content. -Talk lifecycle log records also flow to OTLP logs when OpenTelemetry log export -is enabled, using the same bounded attributes as file logs. +Talk lifecycle log records also flow to diagnostics-otel log export when +OpenTelemetry log export is enabled, using the same bounded attributes as file +logs. Configure `diagnostics.otel.logsExporter` to choose OTLP, stdout JSONL, or +both sinks. ### Model call size and timing diff --git a/docs/plugins/plugin-inventory.md b/docs/plugins/plugin-inventory.md index 20251cdd5cc..f82daec5eeb 100644 --- a/docs/plugins/plugin-inventory.md +++ b/docs/plugins/plugin-inventory.md @@ -227,7 +227,7 @@ Each entry lists the package, distribution route, and description. - **[deepseek](/plugins/reference/deepseek)** (`@openclaw/deepseek-provider`) - npm; ClawHub: `clawhub:@openclaw/deepseek-provider`. Adds DeepSeek model provider support to OpenClaw. -- **[diagnostics-otel](/plugins/reference/diagnostics-otel)** (`@openclaw/diagnostics-otel`) - npm; ClawHub: `clawhub:@openclaw/diagnostics-otel`. OpenClaw diagnostics OpenTelemetry exporter for metrics and traces. +- **[diagnostics-otel](/plugins/reference/diagnostics-otel)** (`@openclaw/diagnostics-otel`) - npm; ClawHub: `clawhub:@openclaw/diagnostics-otel`. OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs. - **[diagnostics-prometheus](/plugins/reference/diagnostics-prometheus)** (`@openclaw/diagnostics-prometheus`) - npm; ClawHub: `clawhub:@openclaw/diagnostics-prometheus`. OpenClaw diagnostics Prometheus exporter for runtime metrics. diff --git a/docs/plugins/reference/diagnostics-otel.md b/docs/plugins/reference/diagnostics-otel.md index 8f062f2592e..cb63ca08348 100644 --- a/docs/plugins/reference/diagnostics-otel.md +++ b/docs/plugins/reference/diagnostics-otel.md @@ -1,5 +1,5 @@ --- -summary: "OpenClaw diagnostics OpenTelemetry exporter for metrics and traces." +summary: "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs." read_when: - You are installing, configuring, or auditing the diagnostics-otel plugin title: "Diagnostics OpenTelemetry plugin" @@ -7,7 +7,7 @@ title: "Diagnostics OpenTelemetry plugin" # Diagnostics OpenTelemetry plugin -OpenClaw diagnostics OpenTelemetry exporter for metrics and traces. +OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs. ## Distribution diff --git a/extensions/diagnostics-otel/README.md b/extensions/diagnostics-otel/README.md index c34bb5d2d49..a788a4e291c 100644 --- a/extensions/diagnostics-otel/README.md +++ b/extensions/diagnostics-otel/README.md @@ -2,7 +2,7 @@ Official OpenTelemetry diagnostics exporter for OpenClaw. -This plugin exports OpenClaw Gateway traces, metrics, and logs to an OTLP collector for observability stacks such as Grafana, Datadog, Honeycomb, New Relic, Tempo, and compatible collectors. +This plugin exports OpenClaw Gateway traces, metrics, and logs to an OTLP collector for observability stacks such as Grafana, Datadog, Honeycomb, New Relic, Tempo, and compatible collectors. It can also write diagnostic log records as stdout JSONL for container log pipelines. ## Install diff --git a/extensions/diagnostics-otel/openclaw.plugin.json b/extensions/diagnostics-otel/openclaw.plugin.json index 92e06df8b96..0832d62bb2d 100644 --- a/extensions/diagnostics-otel/openclaw.plugin.json +++ b/extensions/diagnostics-otel/openclaw.plugin.json @@ -1,7 +1,7 @@ { "id": "diagnostics-otel", "name": "Diagnostics OpenTelemetry", - "description": "OpenClaw diagnostics OpenTelemetry exporter for metrics and traces.", + "description": "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs.", "activation": { "onStartup": true }, diff --git a/extensions/diagnostics-otel/package.json b/extensions/diagnostics-otel/package.json index ed850c57af1..ca3f3df7008 100644 --- a/extensions/diagnostics-otel/package.json +++ b/extensions/diagnostics-otel/package.json @@ -1,7 +1,7 @@ { "name": "@openclaw/diagnostics-otel", "version": "2026.6.8", - "description": "OpenClaw diagnostics OpenTelemetry exporter for metrics and traces.", + "description": "OpenClaw diagnostics OpenTelemetry exporter for metrics, traces, and logs.", "repository": { "type": "git", "url": "https://github.com/openclaw/openclaw" diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index bf6eaf037af..dce7fb60fb9 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -220,13 +220,26 @@ type OtelContextFlags = { traces?: boolean; metrics?: boolean; logs?: boolean; + protocol?: NonNullable< + NonNullable["otel"] + >["protocol"]; + logsExporter?: NonNullable< + NonNullable["otel"] + >["logsExporter"]; captureContent?: NonNullable< NonNullable["otel"] >["captureContent"]; }; function createOtelContext( endpoint: string, - { traces = false, metrics = false, logs = false, captureContent }: OtelContextFlags = {}, + { + traces = false, + metrics = false, + logs = false, + protocol = OTEL_TEST_PROTOCOL, + logsExporter, + captureContent, + }: OtelContextFlags = {}, ): OpenClawPluginServiceContext { return { config: { @@ -235,10 +248,11 @@ function createOtelContext( otel: { enabled: true, endpoint, - protocol: OTEL_TEST_PROTOCOL, + protocol, traces, metrics, logs, + ...(logsExporter !== undefined ? { logsExporter } : {}), ...(captureContent !== undefined ? { captureContent } : {}), }, }, @@ -362,6 +376,38 @@ function histogramCreateOptions(name: string) { | undefined; } +type StdoutDiagnosticLogLine = { + ts?: string; + signal?: string; + "service.name"?: string; + severityText?: string; + severityNumber?: number; + body?: unknown; + attributes?: Record; + trace_id?: string; + span_id?: string; + trace_flags?: string; +}; + +function captureStdoutWrites() { + const writes: string[] = []; + const spy = vi.spyOn(process.stdout, "write").mockImplementation((( + chunk: string | Uint8Array, + ) => { + writes.push(typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8")); + return true; + }) as typeof process.stdout.write); + return { writes, spy }; +} + +function parseSingleStdoutDiagnosticLogLine(writes: string[]): StdoutDiagnosticLogLine { + expect(writes).toHaveLength(1); + expect(writes[0]?.endsWith("\n")).toBe(true); + const line = writes[0]?.slice(0, -1) ?? ""; + expect(line).not.toContain("\n"); + return JSON.parse(line) as StdoutDiagnosticLogLine; +} + async function emitAndCaptureLog( event: Omit[0], { type: "log.record" }>, "type">, options: { @@ -1067,6 +1113,94 @@ describe("diagnostics-otel service", () => { await service.stop?.(ctx); }); + test("starts stdout-only logs when OTLP protocol is unsupported", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { + traces: false, + metrics: false, + logs: true, + protocol: "grpc", + logsExporter: "stdout", + }); + const capture = captureStdoutWrites(); + try { + await service.start(ctx); + emitDiagnosticEvent({ + type: "log.record", + level: "INFO", + message: "stdout only log", + }); + await flushDiagnosticEvents(); + + const line = parseSingleStdoutDiagnosticLogLine(capture.writes); + expect(line.body).toBe("log"); + expect(logExporterCtor).not.toHaveBeenCalled(); + expect(traceExporterCtor).not.toHaveBeenCalled(); + expect(metricExporterCtor).not.toHaveBeenCalled(); + expect(ctx.logger.warn).not.toHaveBeenCalledWith( + "diagnostics-otel: unsupported protocol grpc", + ); + } finally { + capture.spy.mockRestore(); + await service.stop?.(ctx); + } + }); + + test("exports trusted security events as stdout JSONL logs", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("", { logs: true, logsExporter: "stdout" }); + const trace = createDiagnosticTraceContext({ + traceId: TRACE_ID, + spanId: SPAN_ID, + traceFlags: "01", + }); + const stdout = captureStdoutWrites(); + + try { + await service.start(ctx); + emitTrustedSecurityEvent({ + eventId: "security-event-stdout", + category: "tool", + action: "tool.execution.blocked", + outcome: "denied", + severity: "medium", + reason: "tools.deny", + attributes: { + secretish: "token sk-test-secret", + [PROTO_KEY]: "blocked", + }, + trace, + }); + await flushDiagnosticEvents(); + + expect(logExporterCtor).not.toHaveBeenCalled(); + expect(logEmit).not.toHaveBeenCalled(); + const record = parseSingleStdoutDiagnosticLogLine(stdout.writes); + expect(record.body).toBe("openclaw.security.event"); + expect(record.severityText).toBe("WARN"); + expect(record.severityNumber).toBe(13); + expect(record.attributes).toMatchObject({ + "openclaw.security.event_id": "security-event-stdout", + "openclaw.security.category": "tool", + "openclaw.security.action": "tool.execution.blocked", + "openclaw.security.outcome": "denied", + "openclaw.security.severity": "medium", + "openclaw.security.reason": "tools.deny", + "openclaw.security.attribute.secretish": "unknown", + }); + expect(Object.hasOwn(record.attributes ?? {}, "openclaw.security.attribute.__proto__")).toBe( + false, + ); + expect(record.trace_id).toBe(TRACE_ID); + expect(record.span_id).toBe(SPAN_ID); + expect(record.trace_flags).toBe("01"); + expect(JSON.stringify(record)).not.toContain("sk-test-secret"); + } finally { + stdout.spy.mockRestore(); + await service.stop?.(ctx); + } + }); + test("records liveness warning diagnostics", async () => { const service = createDiagnosticsOtelService(); const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { traces: true, metrics: true }); @@ -1442,6 +1576,123 @@ describe("diagnostics-otel service", () => { await service.stop?.(ctx); }); + test("exports diagnostic logs as stdout JSONL without constructing the OTLP log exporter", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("", { + logs: true, + logsExporter: "stdout", + captureContent: true, + }); + ctx.config.diagnostics!.otel!.serviceName = "rovoclaw-openclaw"; + const stdout = captureStdoutWrites(); + + try { + await service.start(ctx); + + expect(logExporterCtor).not.toHaveBeenCalled(); + emitDiagnosticEventWithTrustedTraceContext({ + type: "log.record", + level: "WARN", + message: "Using API key sk-1234567890abcdef1234567890abcdef", + attributes: { + token: "ghp_abcdefghijklmnopqrstuvwxyz123456", // pragma: allowlist secret + subsystem: "diagnostic", + }, + trace: { + traceId: TRACE_ID, + spanId: SPAN_ID, + traceFlags: "01", + }, + }); + await flushDiagnosticEvents(); + + expect(logEmit).not.toHaveBeenCalled(); + const record = parseSingleStdoutDiagnosticLogLine(stdout.writes); + expect(record.ts).toMatch(/^\d{4}-\d{2}-\d{2}T/); + expect(record.signal).toBe("openclaw.diagnostic.log"); + expect(record["service.name"]).toBe("rovoclaw-openclaw"); + expect(record.severityText).toBe("WARN"); + expect(record.severityNumber).toBe(13); + expect(String(record.body)).not.toContain("sk-1234567890abcdef1234567890abcdef"); + expect(String(record.body)).toContain("sk-123"); + expect(record.attributes).toMatchObject({ + "openclaw.log.level": "WARN", + "openclaw.subsystem": "diagnostic", + }); + const tokenAttr = record.attributes?.["openclaw.token"]; + expect(tokenAttr).not.toBe("ghp_abcdefghijklmnopqrstuvwxyz123456"); // pragma: allowlist secret + expect(record.trace_id).toBe(TRACE_ID); + expect(record.span_id).toBe(SPAN_ID); + expect(JSON.stringify(record)).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz123456"); // pragma: allowlist secret + } finally { + stdout.spy.mockRestore(); + await service.stop?.(ctx); + } + }); + + test("keeps explicit OTLP log export off stdout", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { + logs: true, + logsExporter: "otlp", + }); + const stdout = captureStdoutWrites(); + + try { + await service.start(ctx); + emitDiagnosticEvent({ + type: "log.record", + level: "INFO", + message: "otlp only", + }); + await flushDiagnosticEvents(); + + expect(logExporterCtor).toHaveBeenCalledTimes(1); + expect(logEmit).toHaveBeenCalledTimes(1); + expect(stdout.writes).toEqual([]); + } finally { + stdout.spy.mockRestore(); + await service.stop?.(ctx); + } + }); + + test("exports diagnostic logs to OTLP and stdout when logsExporter is both", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext(OTEL_TEST_ENDPOINT, { + logs: true, + logsExporter: "both", + }); + const stdout = captureStdoutWrites(); + + try { + await service.start(ctx); + emitDiagnosticEvent({ + type: "log.record", + level: "ERROR", + message: "both sinks", + attributes: { + subsystem: "diagnostic", + }, + }); + await flushDiagnosticEvents(); + + expect(logExporterCtor).toHaveBeenCalledTimes(1); + const emitCall = mockCallArg(logEmit, 0) as { + attributes?: Record; + body?: string; + severityText?: string; + }; + const record = parseSingleStdoutDiagnosticLogLine(stdout.writes); + expect(emitCall.body).toBe("log"); + expect(record.body).toBe(emitCall.body); + expect(record.severityText).toBe(emitCall.severityText); + expect(record.attributes).toEqual(emitCall.attributes); + } finally { + stdout.spy.mockRestore(); + await service.stop?.(ctx); + } + }); + test("omits log message bodies from OTLP logs unless broad content capture is enabled", async () => { const emitCall = await emitAndCaptureLog({ level: "INFO", diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index 1867a10c864..af889209058 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -103,6 +103,13 @@ type OtelContentCapturePolicy = { logBodies: boolean; }; +type OtelLogsExporter = "otlp" | "stdout" | "both"; + +type BuiltOtelLogRecord = { + logRecord: LogRecord; + traceContext?: DiagnosticTraceContext; +}; + type OtelModelCallContent = { inputMessages?: unknown; outputMessages?: unknown; @@ -350,6 +357,43 @@ function shouldCaptureOtelLogBody(policy: OtelContentCapturePolicy): boolean { return policy.logBodies; } +function otelLogTimestampIso(timestamp: LogRecord["timestamp"]): string { + if (timestamp instanceof Date) { + return timestamp.toISOString(); + } + if (typeof timestamp === "number" && Number.isFinite(timestamp)) { + return new Date(timestamp).toISOString(); + } + if (Array.isArray(timestamp)) { + const [seconds, nanoseconds] = timestamp; + if (Number.isFinite(seconds) && Number.isFinite(nanoseconds)) { + return new Date(seconds * 1000 + Math.trunc(nanoseconds / 1_000_000)).toISOString(); + } + } + return new Date().toISOString(); +} + +function writeStdoutDiagnosticLogRecord(params: { + logRecord: LogRecord; + serviceName: string; + traceContext?: DiagnosticTraceContext; +}): void { + const { logRecord, serviceName, traceContext } = params; + const line = { + ts: otelLogTimestampIso(logRecord.timestamp), + signal: "openclaw.diagnostic.log", + "service.name": serviceName, + severityText: logRecord.severityText, + severityNumber: logRecord.severityNumber, + body: logRecord.body, + attributes: logRecord.attributes ?? {}, + ...(traceContext?.traceId ? { trace_id: traceContext.traceId } : {}), + ...(traceContext?.spanId ? { span_id: traceContext.spanId } : {}), + ...(traceContext?.traceFlags ? { trace_flags: traceContext.traceFlags } : {}), + }; + process.stdout.write(`${JSON.stringify(line)}\n`); +} + function hasOtelSemconvOptIn(value: string | undefined, optIn: string): boolean { return ( value @@ -1157,11 +1201,7 @@ function assignOtelSecurityAttributes( ); } if (evt.policy.decision) { - assignOtelLogAttribute( - attributes, - "openclaw.security.policy.decision", - evt.policy.decision, - ); + assignOtelLogAttribute(attributes, "openclaw.security.policy.decision", evt.policy.decision); } if (evt.policy.reason) { assignOtelLogAttribute( @@ -1180,11 +1220,7 @@ function assignOtelSecurityAttributes( ); } if (evt.control.family) { - assignOtelLogAttribute( - attributes, - "openclaw.security.control.family", - evt.control.family, - ); + assignOtelLogAttribute(attributes, "openclaw.security.control.family", evt.control.family); } } assignOtelSecurityEventAttributes(attributes, evt.attributes); @@ -1217,6 +1253,15 @@ function contextForTrustedTraceContext( : undefined; } +function normalizedTrustedTraceContext( + evt: DiagnosticEventPayload, + metadata: DiagnosticEventMetadata, +): DiagnosticTraceContext | undefined { + return metadata.trusted || metadata.trustedTraceContext === true + ? normalizeTraceContext(evt.trace) + : undefined; +} + function addTraceAttributes( attributes: Record, traceContext: DiagnosticTraceContext | undefined, @@ -1302,6 +1347,14 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { const tracesEnabled = otel.traces !== false; const metricsEnabled = otel.metrics !== false; const logsEnabled = otel.logs === true; + const logsExporter: OtelLogsExporter = otel.logsExporter ?? "otlp"; + const logsToOtlp = logsEnabled && (logsExporter === "otlp" || logsExporter === "both"); + const logsToStdout = logsEnabled && (logsExporter === "stdout" || logsExporter === "both"); + const otlpSignals: TelemetryExporterDiagnosticEvent["signal"][] = [ + ...(tracesEnabled ? (["traces"] as const) : []), + ...(metricsEnabled ? (["metrics"] as const) : []), + ...(logsToOtlp ? (["logs"] as const) : []), + ]; const enabledSignals: TelemetryExporterDiagnosticEvent["signal"][] = [ ...(tracesEnabled ? (["traces"] as const) : []), ...(metricsEnabled ? (["metrics"] as const) : []), @@ -1312,8 +1365,8 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { } const protocol = otel.protocol ?? process.env.OTEL_EXPORTER_OTLP_PROTOCOL ?? "http/protobuf"; - if (protocol !== "http/protobuf") { - emitForSignals(enabledSignals, { + if (otlpSignals.length > 0 && protocol !== "http/protobuf") { + emitForSignals(otlpSignals, { exporter: "diagnostics-otel", status: "failure", reason: "unsupported_protocol", @@ -1774,79 +1827,130 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { | undefined; if (logsEnabled) { let logRecordExportFailureLastReportedAt = Number.NEGATIVE_INFINITY; - const logExporter = new OTLPLogExporter({ - ...(logUrl ? { url: logUrl } : {}), - ...(headers ? { headers } : {}), - }); - const logProcessor = new BatchLogRecordProcessor( - logExporter, - typeof otel.flushIntervalMs === "number" - ? { scheduledDelayMillis: Math.max(1000, otel.flushIntervalMs) } - : {}, - ); - logProvider = new LoggerProvider({ - resource, - processors: [logProcessor], - }); - const otelLogger = logProvider.getLogger("openclaw"); + + let otelLogger: { emit: (logRecord: LogRecord) => void } | undefined; + if (logsToOtlp) { + const logExporter = new OTLPLogExporter({ + ...(logUrl ? { url: logUrl } : {}), + ...(headers ? { headers } : {}), + }); + const logProcessor = new BatchLogRecordProcessor( + logExporter, + typeof otel.flushIntervalMs === "number" + ? { scheduledDelayMillis: Math.max(1000, otel.flushIntervalMs) } + : {}, + ); + logProvider = new LoggerProvider({ + resource, + processors: [logProcessor], + }); + otelLogger = logProvider.getLogger("openclaw"); + } + + const reportLogExportFailure = (err: unknown, label: "log record" | "security event") => { + emitExporterEvent({ + exporter: "diagnostics-otel", + signal: "logs", + status: "failure", + reason: "emit_failed", + errorCategory: errorCategory(err), + }); + const now = Date.now(); + if ( + now - logRecordExportFailureLastReportedAt >= + LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS + ) { + logRecordExportFailureLastReportedAt = now; + ctx.logger.error(`diagnostics-otel: ${label} export failed: ${formatError(err)}`); + } + }; + + const emitLogRecord = ({ logRecord, traceContext }: BuiltOtelLogRecord) => { + if (logsToOtlp) { + otelLogger?.emit(logRecord); + } + if (logsToStdout) { + writeStdoutDiagnosticLogRecord({ + logRecord, + serviceName, + ...(traceContext ? { traceContext } : {}), + }); + } + }; + + const buildDiagnosticLogRecord = ( + evt: Extract, + metadata: DiagnosticEventMetadata, + ): BuiltOtelLogRecord => { + const logLevelName = evt.level || "INFO"; + const severityNumber = logSeverityMap[logLevelName] ?? (9 as SeverityNumber); + const body = shouldCaptureOtelLogBody(contentCapturePolicy) + ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) + : "log"; + const attributes = Object.create(null) as Record; + assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName); + if (evt.loggerName) { + assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName); + } + if (evt.loggerParents?.length) { + assignOtelLogAttribute( + attributes, + "openclaw.logger.parents", + evt.loggerParents.join("."), + ); + } + assignOtelLogEventAttributes(attributes, evt.attributes); + if (evt.code?.line) { + assignOtelLogAttribute(attributes, "code.lineno", evt.code.line); + } + if (evt.code?.functionName) { + assignOtelLogAttribute(attributes, "code.function", evt.code.functionName); + } + const traceContext = normalizedTrustedTraceContext(evt, metadata); + addTraceAttributes(attributes, traceContext); + + const logRecord: LogRecord = { + body, + severityText: logLevelName, + severityNumber, + attributes: redactOtelAttributes(attributes), + timestamp: evt.ts, + }; + const logContext = contextForTrustedTraceContext(evt, metadata); + if (logContext) { + logRecord.context = logContext; + } + return { logRecord, ...(traceContext ? { traceContext } : {}) }; + }; + + const buildSecurityLogRecord = ( + evt: Extract, + metadata: DiagnosticEventMetadata, + ): BuiltOtelLogRecord => { + const severityText = securitySeverityText(evt.severity); + const attributes = Object.create(null) as Record; + assignOtelSecurityAttributes(attributes, evt); + + const traceContext = normalizedTrustedTraceContext(evt, metadata); + const logRecord: LogRecord = { + body: "openclaw.security.event", + severityText, + severityNumber: logSeverityMap[severityText] ?? (9 as SeverityNumber), + attributes: redactOtelAttributes(attributes), + timestamp: evt.ts, + }; + const logContext = contextForTrustedTraceContext(evt, metadata); + if (logContext) { + logRecord.context = logContext; + } + return { logRecord, ...(traceContext ? { traceContext } : {}) }; + }; + recordLogRecord = (evt, metadata) => { try { - const logLevelName = evt.level || "INFO"; - const severityNumber = logSeverityMap[logLevelName] ?? (9 as SeverityNumber); - const body = shouldCaptureOtelLogBody(contentCapturePolicy) - ? normalizeOtelLogString(evt.message || "log", MAX_OTEL_LOG_BODY_CHARS) - : "log"; - const attributes = Object.create(null) as Record; - assignOtelLogAttribute(attributes, "openclaw.log.level", logLevelName); - if (evt.loggerName) { - assignOtelLogAttribute(attributes, "openclaw.logger", evt.loggerName); - } - if (evt.loggerParents?.length) { - assignOtelLogAttribute( - attributes, - "openclaw.logger.parents", - evt.loggerParents.join("."), - ); - } - assignOtelLogEventAttributes(attributes, evt.attributes); - if (evt.code?.line) { - assignOtelLogAttribute(attributes, "code.lineno", evt.code.line); - } - if (evt.code?.functionName) { - assignOtelLogAttribute(attributes, "code.function", evt.code.functionName); - } - if (metadata.trusted || metadata.trustedTraceContext === true) { - addTraceAttributes(attributes, evt.trace); - } - - const logRecord: LogRecord = { - body, - severityText: logLevelName, - severityNumber, - attributes: redactOtelAttributes(attributes), - timestamp: evt.ts, - }; - const logContext = contextForTrustedTraceContext(evt, metadata); - if (logContext) { - logRecord.context = logContext; - } - otelLogger.emit(logRecord); + emitLogRecord(buildDiagnosticLogRecord(evt, metadata)); } catch (err) { - emitExporterEvent({ - exporter: "diagnostics-otel", - signal: "logs", - status: "failure", - reason: "emit_failed", - errorCategory: errorCategory(err), - }); - const now = Date.now(); - if ( - now - logRecordExportFailureLastReportedAt >= - LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS - ) { - logRecordExportFailureLastReportedAt = now; - ctx.logger.error(`diagnostics-otel: log record export failed: ${formatError(err)}`); - } + reportLogExportFailure(err, "log record"); } }; recordSecurityEvent = (evt, metadata) => { @@ -1854,40 +1958,9 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { return; } try { - const severityText = securitySeverityText(evt.severity); - const attributes = Object.create(null) as Record; - assignOtelSecurityAttributes(attributes, evt); - - const logRecord: LogRecord = { - body: "openclaw.security.event", - severityText, - severityNumber: logSeverityMap[severityText] ?? (9 as SeverityNumber), - attributes: redactOtelAttributes(attributes), - timestamp: evt.ts, - }; - const logContext = contextForTrustedTraceContext(evt, metadata); - if (logContext) { - logRecord.context = logContext; - } - otelLogger.emit(logRecord); + emitLogRecord(buildSecurityLogRecord(evt, metadata)); } catch (err) { - emitExporterEvent({ - exporter: "diagnostics-otel", - signal: "logs", - status: "failure", - reason: "emit_failed", - errorCategory: errorCategory(err), - }); - const now = Date.now(); - if ( - now - logRecordExportFailureLastReportedAt >= - LOG_RECORD_EXPORT_FAILURE_REPORT_INTERVAL_MS - ) { - logRecordExportFailureLastReportedAt = now; - ctx.logger.error( - `diagnostics-otel: security event export failed: ${formatError(err)}`, - ); - } + reportLogExportFailure(err, "security event"); } }; } @@ -3721,7 +3794,13 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { }); if (logsEnabled) { - ctx.logger.info("diagnostics-otel: logs exporter enabled (OTLP/Protobuf)"); + const label = + logsExporter === "both" + ? "OTLP/Protobuf + stdout JSONL" + : logsExporter === "stdout" + ? "stdout JSONL" + : "OTLP/Protobuf"; + ctx.logger.info(`diagnostics-otel: logs exporter enabled (${label})`); } }, async stop() { diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index ec9f9fe49c1..21d6c34fa0e 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -149,8 +149,10 @@ describe("qa scenario catalog", () => { it("loads scenario-declared gateway runtime options from YAML", () => { const scenario = readQaScenarioById("control-ui-qa-channel-image-roundtrip"); + const otelStdout = readQaScenarioById("otel-stdout-log-smoke"); expect(scenario.gatewayRuntime?.forwardHostHome).toBe(true); + expect(otelStdout.gatewayRuntime?.preserveDebugArtifacts).toBe(true); }); it("loads native test execution scenarios from YAML", () => { diff --git a/extensions/qa-lab/src/scenario-catalog.ts b/extensions/qa-lab/src/scenario-catalog.ts index 1c9d1ef3a34..a6b7400f24c 100644 --- a/extensions/qa-lab/src/scenario-catalog.ts +++ b/extensions/qa-lab/src/scenario-catalog.ts @@ -124,6 +124,7 @@ const qaScenarioCoverageSchema = z const qaScenarioGatewayRuntimeSchema = z.object({ forwardHostHome: z.boolean().optional(), + preserveDebugArtifacts: z.boolean().optional(), }); export const QA_RUNTIME_PARITY_TIERS = ["standard", "optional", "live-only", "soak"] as const; diff --git a/extensions/qa-lab/src/suite-planning.test.ts b/extensions/qa-lab/src/suite-planning.test.ts index 1c20aae3237..99f25a0bffd 100644 --- a/extensions/qa-lab/src/suite-planning.test.ts +++ b/extensions/qa-lab/src/suite-planning.test.ts @@ -330,10 +330,15 @@ describe("qa suite planning helpers", () => { plugins: ["browser"], gatewayRuntime: { forwardHostHome: true }, }), + makeQaSuiteTestScenario("otel-stdout", { + plugins: ["diagnostics-otel"], + gatewayRuntime: { preserveDebugArtifacts: true }, + }), ]; expect(collectQaSuiteGatewayRuntimeOptions(scenarios)).toEqual({ forwardHostHome: true, + preserveDebugArtifacts: true, }); }); diff --git a/extensions/qa-lab/src/suite-planning.ts b/extensions/qa-lab/src/suite-planning.ts index 9631ae96428..efa37bc4b3e 100644 --- a/extensions/qa-lab/src/suite-planning.ts +++ b/extensions/qa-lab/src/suite-planning.ts @@ -144,12 +144,21 @@ function collectQaSuiteGatewayRuntimeOptions( scenarios: ReturnType["scenarios"], ) { let forwardHostHome = false; + let preserveDebugArtifacts = false; for (const scenario of scenarios) { if (scenario.gatewayRuntime?.forwardHostHome === true) { forwardHostHome = true; } + if (scenario.gatewayRuntime?.preserveDebugArtifacts === true) { + preserveDebugArtifacts = true; + } } - return forwardHostHome ? { forwardHostHome: true } : undefined; + return forwardHostHome || preserveDebugArtifacts + ? { + ...(forwardHostHome ? { forwardHostHome: true } : {}), + ...(preserveDebugArtifacts ? { preserveDebugArtifacts: true } : {}), + } + : undefined; } function shouldUseIsolatedQaSuiteScenarioWorkers(params: { diff --git a/extensions/qa-lab/src/suite-test-helpers.ts b/extensions/qa-lab/src/suite-test-helpers.ts index 2eeb945fd2f..02a845b7d28 100644 --- a/extensions/qa-lab/src/suite-test-helpers.ts +++ b/extensions/qa-lab/src/suite-test-helpers.ts @@ -9,7 +9,7 @@ export function makeQaSuiteTestScenario( config?: Record; plugins?: string[]; gatewayConfigPatch?: Record; - gatewayRuntime?: { forwardHostHome?: boolean }; + gatewayRuntime?: { forwardHostHome?: boolean; preserveDebugArtifacts?: boolean }; runtimeParityTier?: QaSuiteTestScenario["runtimeParityTier"]; surface?: string; } = {}, diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 22c9e4583d9..055697cebf6 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -1573,7 +1573,10 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise scenario.status === "fail").length; - if (scenarios.some((scenario) => scenario.status === "fail")) { + if ( + scenarios.some((scenario) => scenario.status === "fail") || + gatewayRuntimeOptions?.preserveDebugArtifacts === true + ) { preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime"); } lab.setScenarioRun({ diff --git a/qa/scenarios/runtime/otel-both-log-smoke.yaml b/qa/scenarios/runtime/otel-both-log-smoke.yaml new file mode 100644 index 00000000000..31330039176 --- /dev/null +++ b/qa/scenarios/runtime/otel-both-log-smoke.yaml @@ -0,0 +1,103 @@ +title: OTEL dual log exporter smoke + +scenario: + id: otel-both-log-smoke + surface: telemetry + coverage: + primary: + - telemetry.otel + secondary: + - harness.qa-lab + objective: Verify a QA-lab gateway run emits bounded OpenTelemetry traces and metrics while routing diagnostics-otel logs to OTLP and stdout JSONL. + successCriteria: + - The diagnostics-otel plugin starts with trace, metric, and log export enabled. + - Diagnostic log records are written to OTLP and stdout JSONL. + - A minimal QA-channel agent turn completes. + - The run emits low-cardinality telemetry without content or raw diagnostic identifiers. + plugins: + - diagnostics-otel + gatewayRuntime: + preserveDebugArtifacts: true + gatewayConfigPatch: + logging: + file: .artifacts/qa-e2e/otel-smoke-gateway.jsonl + level: info + diagnostics: + enabled: true + otel: + enabled: true + protocol: http/protobuf + traces: true + metrics: true + logs: true + logsExporter: both + sampleRate: 1 + flushIntervalMs: 1000 + captureContent: + enabled: false + docsRefs: + - docs/gateway/opentelemetry.md + - docs/concepts/qa-e2e-automation.md + codeRefs: + - extensions/diagnostics-otel/src/service.ts + - test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts + execution: + kind: flow + summary: Emit minimal QA-lab telemetry with diagnostics-otel OTLP and stdout log export enabled. + config: + prompt: "OTEL QA marker: reply exactly `OTEL-QA-OK`. Do not repeat OTEL-QA-SECRET." + expectedReply: OTEL-QA-OK + +flow: + steps: + - name: emits a traced qa-channel turn with OTLP and stdout logs + actions: + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - set: startCursor + value: + expr: state.getSnapshot().messages.length + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:otel-both-log-smoke + message: + expr: config.prompt + timeoutMs: + expr: liveTurnTimeoutMs(env, 30000) + - call: waitForCondition + saveAs: outbound + args: + - lambda: + expr: "state.getSnapshot().messages.slice(startCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && String(candidate.text ?? '').trim().length > 0).at(-1)" + - expr: liveTurnTimeoutMs(env, 30000) + - expr: "env.providerMode === 'mock-openai' ? 100 : 250" + - assert: + expr: "String(outbound.text ?? '').trim().length > 0" + message: "expected non-empty qa output" + - assert: + expr: "String(outbound.text ?? '').includes(config.expectedReply)" + message: "expected qa output to include the response sentinel" + - set: gatewayLogText + value: + expr: "String(readGatewayLogs() ?? '')" + - set: stdoutDiagnosticLogs + value: + expr: "gatewayLogText.split('\\n').map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter((entry) => entry?.signal === 'openclaw.diagnostic.log')" + - assert: + expr: "stdoutDiagnosticLogs.length > 0" + message: "expected at least one stdout diagnostics-otel JSONL log record" + - assert: + expr: "stdoutDiagnosticLogs.every((record) => typeof record.ts === 'string' && typeof record['service.name'] === 'string' && typeof record.severityText === 'string' && typeof record.severityNumber === 'number' && Object.hasOwn(record, 'body') && record.attributes && typeof record.attributes === 'object' && !Array.isArray(record.attributes))" + message: "expected stdout diagnostics-otel records to keep the documented JSONL shape" + - assert: + expr: "!JSON.stringify(stdoutDiagnosticLogs).includes('OTEL-QA-SECRET') && !JSON.stringify(stdoutDiagnosticLogs).includes('agent:qa:otel-both-log-smoke')" + message: "expected stdout diagnostics-otel records to omit raw prompt/session content" + detailsExpr: "`stdout diagnostic log records=${stdoutDiagnosticLogs.length}`" diff --git a/qa/scenarios/runtime/otel-stdout-log-smoke.yaml b/qa/scenarios/runtime/otel-stdout-log-smoke.yaml new file mode 100644 index 00000000000..adbabe36076 --- /dev/null +++ b/qa/scenarios/runtime/otel-stdout-log-smoke.yaml @@ -0,0 +1,103 @@ +title: OTEL stdout log smoke + +scenario: + id: otel-stdout-log-smoke + surface: telemetry + coverage: + primary: + - telemetry.otel + secondary: + - harness.qa-lab + objective: Verify a QA-lab gateway run emits bounded OpenTelemetry traces and metrics while routing diagnostics-otel logs to stdout JSONL. + successCriteria: + - The diagnostics-otel plugin starts with trace, metric, and log export enabled. + - Diagnostic log records are written as stdout JSONL instead of OTLP logs. + - A minimal QA-channel agent turn completes. + - The run emits low-cardinality telemetry without content or raw diagnostic identifiers. + plugins: + - diagnostics-otel + gatewayRuntime: + preserveDebugArtifacts: true + gatewayConfigPatch: + logging: + file: .artifacts/qa-e2e/otel-smoke-gateway.jsonl + level: info + diagnostics: + enabled: true + otel: + enabled: true + protocol: http/protobuf + traces: true + metrics: true + logs: true + logsExporter: stdout + sampleRate: 1 + flushIntervalMs: 1000 + captureContent: + enabled: false + docsRefs: + - docs/gateway/opentelemetry.md + - docs/concepts/qa-e2e-automation.md + codeRefs: + - extensions/diagnostics-otel/src/service.ts + - test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts + execution: + kind: flow + summary: Emit minimal QA-lab telemetry with diagnostics-otel stdout log export enabled. + config: + prompt: "OTEL QA marker: reply exactly `OTEL-QA-OK`. Do not repeat OTEL-QA-SECRET." + expectedReply: OTEL-QA-OK + +flow: + steps: + - name: emits a traced qa-channel turn with stdout logs + actions: + - call: waitForGatewayHealthy + args: + - ref: env + - 60000 + - call: waitForQaChannelReady + args: + - ref: env + - 60000 + - call: reset + - set: startCursor + value: + expr: state.getSnapshot().messages.length + - call: runAgentPrompt + args: + - ref: env + - sessionKey: agent:qa:otel-stdout-log-smoke + message: + expr: config.prompt + timeoutMs: + expr: liveTurnTimeoutMs(env, 30000) + - call: waitForCondition + saveAs: outbound + args: + - lambda: + expr: "state.getSnapshot().messages.slice(startCursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === 'qa-operator' && String(candidate.text ?? '').trim().length > 0).at(-1)" + - expr: liveTurnTimeoutMs(env, 30000) + - expr: "env.providerMode === 'mock-openai' ? 100 : 250" + - assert: + expr: "String(outbound.text ?? '').trim().length > 0" + message: "expected non-empty qa output" + - assert: + expr: "String(outbound.text ?? '').includes(config.expectedReply)" + message: "expected qa output to include the response sentinel" + - set: gatewayLogText + value: + expr: "String(readGatewayLogs() ?? '')" + - set: stdoutDiagnosticLogs + value: + expr: "gatewayLogText.split('\\n').map((line) => { try { return JSON.parse(line); } catch { return null; } }).filter((entry) => entry?.signal === 'openclaw.diagnostic.log')" + - assert: + expr: "stdoutDiagnosticLogs.length > 0" + message: "expected at least one stdout diagnostics-otel JSONL log record" + - assert: + expr: "stdoutDiagnosticLogs.every((record) => typeof record.ts === 'string' && typeof record['service.name'] === 'string' && typeof record.severityText === 'string' && typeof record.severityNumber === 'number' && Object.hasOwn(record, 'body') && record.attributes && typeof record.attributes === 'object' && !Array.isArray(record.attributes))" + message: "expected stdout diagnostics-otel records to keep the documented JSONL shape" + - assert: + expr: "!JSON.stringify(stdoutDiagnosticLogs).includes('OTEL-QA-SECRET') && !JSON.stringify(stdoutDiagnosticLogs).includes('agent:qa:otel-stdout-log-smoke')" + message: "expected stdout diagnostics-otel records to omit raw prompt/session content" + detailsExpr: "`stdout diagnostic log records=${stdoutDiagnosticLogs.length}`" diff --git a/src/config/config-misc.test.ts b/src/config/config-misc.test.ts index 7d326c738be..d1d4f6445bb 100644 --- a/src/config/config-misc.test.ts +++ b/src/config/config-misc.test.ts @@ -422,6 +422,30 @@ describe("crestodian.rescue", () => { }); describe("diagnostics.otel.captureContent", () => { + it("accepts supported OTEL log exporters and rejects unknown values", () => { + for (const logsExporter of ["otlp", "stdout", "both"]) { + const result = OpenClawSchema.safeParse({ + diagnostics: { + otel: { + logs: true, + logsExporter, + }, + }, + }); + expect(result.success).toBe(true); + } + + const invalid = OpenClawSchema.safeParse({ + diagnostics: { + otel: { + logs: true, + logsExporter: "stderr", + }, + }, + }); + expect(invalid.success).toBe(false); + }); + it("accepts boolean and granular OTEL content capture config", () => { for (const captureContent of [ true, diff --git a/src/config/schema.help.quality.test.ts b/src/config/schema.help.quality.test.ts index ba9114e1560..73bd08ff768 100644 --- a/src/config/schema.help.quality.test.ts +++ b/src/config/schema.help.quality.test.ts @@ -457,6 +457,7 @@ const ENUM_EXPECTATIONS: Record = { "discovery.mdns.mode": ['"off"', '"minimal"', '"full"'], "wizard.lastRunMode": ['"local"', '"remote"'], "diagnostics.otel.protocol": ['"http/protobuf"', '"grpc"'], + "diagnostics.otel.logsExporter": ['"otlp"', '"stdout"', '"both"'], "logging.level": ['"silent"', '"fatal"', '"error"', '"warn"', '"info"', '"debug"', '"trace"'], "logging.consoleLevel": [ '"silent"', @@ -566,6 +567,7 @@ const FINAL_BACKLOG_TARGET_KEYS = [ "diagnostics.otel.headers", "diagnostics.otel.logsEndpoint", "diagnostics.otel.logs", + "diagnostics.otel.logsExporter", "diagnostics.otel.metricsEndpoint", "diagnostics.otel.metrics", "diagnostics.otel.sampleRate", diff --git a/src/config/schema.help.ts b/src/config/schema.help.ts index 4394fb8bf62..822e322482c 100644 --- a/src/config/schema.help.ts +++ b/src/config/schema.help.ts @@ -692,6 +692,8 @@ export const FIELD_HELP: Record = { "Enable metrics signal export to the configured OpenTelemetry collector endpoint. Keep enabled for runtime health dashboards, and disable only if metric volume must be minimized.", "diagnostics.otel.logs": "Enable log signal export through OpenTelemetry in addition to local logging sinks. Use this when centralized log correlation is required across services and agents.", + "diagnostics.otel.logsExporter": + 'Log export sink for diagnostics.otel.logs. Use "otlp" for the configured OTLP logs endpoint, "stdout" for one JSON record per stdout line in container log pipelines, and "both" when both sinks are required.', "diagnostics.otel.sampleRate": "Trace sampling rate (0-1) controlling how much trace traffic is exported to observability backends. Lower rates reduce overhead/cost, while higher rates improve debugging fidelity.", "diagnostics.otel.flushIntervalMs": diff --git a/src/config/schema.labels.ts b/src/config/schema.labels.ts index 5c32dda956d..35cc9102816 100644 --- a/src/config/schema.labels.ts +++ b/src/config/schema.labels.ts @@ -55,6 +55,7 @@ export const FIELD_LABELS: Record = { "diagnostics.otel.traces": "OpenTelemetry Traces Enabled", "diagnostics.otel.metrics": "OpenTelemetry Metrics Enabled", "diagnostics.otel.logs": "OpenTelemetry Logs Enabled", + "diagnostics.otel.logsExporter": "OpenTelemetry Logs Exporter", "diagnostics.otel.sampleRate": "OpenTelemetry Trace Sample Rate", "diagnostics.otel.flushIntervalMs": "OpenTelemetry Flush Interval (ms)", "diagnostics.otel.captureContent": "OpenTelemetry Content Capture", diff --git a/src/config/types.base.ts b/src/config/types.base.ts index 28a908d4543..5280f254b51 100644 --- a/src/config/types.base.ts +++ b/src/config/types.base.ts @@ -303,6 +303,8 @@ export type DiagnosticsOtelConfig = { traces?: boolean; metrics?: boolean; logs?: boolean; + /** Log export sink: OTLP by default, stdout JSONL, or both. */ + logsExporter?: "otlp" | "stdout" | "both"; /** Trace sample rate (0.0 - 1.0). */ sampleRate?: number; /** Metric export interval (ms). */ diff --git a/src/config/zod-schema.ts b/src/config/zod-schema.ts index 16116b2fd67..8681d2fd67e 100644 --- a/src/config/zod-schema.ts +++ b/src/config/zod-schema.ts @@ -547,6 +547,9 @@ export const OpenClawSchema = z traces: z.boolean().optional(), metrics: z.boolean().optional(), logs: z.boolean().optional(), + logsExporter: z + .union([z.literal("otlp"), z.literal("stdout"), z.literal("both")]) + .optional(), sampleRate: z.number().min(0).max(1).optional(), flushIntervalMs: z.number().int().nonnegative().optional(), captureContent: z diff --git a/test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts b/test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts index ea8f0f75e16..c0e97ea78ba 100644 --- a/test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts +++ b/test/e2e/qa-lab/runtime/qa-otel-smoke-runtime.ts @@ -3,7 +3,7 @@ import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { randomUUID } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; import { Socket } from "node:net"; import { tmpdir } from "node:os"; @@ -13,6 +13,7 @@ import { gunzipSync } from "node:zlib"; import { stripLeadingPackageManagerSeparator } from "../../../../scripts/lib/arg-utils.mjs"; type CollectorMode = "local" | "docker"; +type OtelLogsExporter = "otlp" | "stdout" | "both"; type OtlpAnyValue = { stringValue?: string; @@ -47,6 +48,7 @@ type OtlpSignal = "logs" | "metrics" | "traces"; type CliOptions = { collectorMode: CollectorMode; + logsExporter: OtelLogsExporter; outputDir: string; providerMode: string; scenarioId: string; @@ -82,7 +84,26 @@ type CapturedLogRecord = { traceId: string; }; +type StdoutDiagnosticLogRecord = { + signal: "openclaw.diagnostic.log"; + ts?: unknown; + "service.name"?: unknown; + severityText?: unknown; + severityNumber?: unknown; + body?: unknown; + attributes?: unknown; + trace_id?: unknown; + span_id?: unknown; + trace_flags?: unknown; + [key: string]: unknown; +}; + const DEFAULT_SCENARIO_ID = "otel-trace-smoke"; +const LOGS_EXPORTER_SCENARIO_IDS = { + otlp: "otel-trace-smoke", + stdout: "otel-stdout-log-smoke", + both: "otel-both-log-smoke", +} satisfies Record; const DEFAULT_DOCKER_COLLECTOR_IMAGE = process.env.OPENCLAW_QA_OTEL_COLLECTOR_IMAGE || "otel/opentelemetry-collector:0.104.0"; const OTLP_SIGNAL_PATHS = new Map([ @@ -133,6 +154,10 @@ const QA_SUITE_TIMEOUT_MS = readPositiveIntegerEnv( 10 * 60 * 1000, ); const QA_SUITE_KILL_GRACE_MS = readPositiveIntegerEnv("OPENCLAW_QA_OTEL_SUITE_KILL_GRACE_MS", 5000); +const MAX_STDOUT_DIAGNOSTIC_LINE_BYTES = readPositiveIntegerEnv( + "OPENCLAW_QA_OTEL_MAX_STDOUT_DIAGNOSTIC_LINE_BYTES", + 512 * 1024, +); function readPositiveIntegerEnv( name: string, @@ -167,7 +192,7 @@ function oversizedBodyError( } function usage(): string { - return `Usage: pnpm qa:otel:smoke [--collector local|docker] [--output-dir ] [--provider-mode ] [--scenario ] [--model ] [--alt-model ] + return `Usage: pnpm qa:otel:smoke [--collector local|docker] [--logs-exporter otlp|stdout|both] [--output-dir ] [--provider-mode ] [--scenario ] [--model ] [--alt-model ] Runs a QA-lab scenario with diagnostics-otel enabled, then asserts the emitted signal shape and privacy contract. The default collector is an in-process @@ -178,8 +203,10 @@ Collector container in front of the receiver. function parseArgs(argv: string[]): CliOptions { const args = stripLeadingPackageManagerSeparator(argv); + let scenarioExplicit = false; const options: CliOptions = { collectorMode: "local", + logsExporter: "otlp", outputDir: path.join(".artifacts", "qa-e2e", `otel-smoke-${Date.now().toString(36)}`), providerMode: "mock-openai", scenarioId: DEFAULT_SCENARIO_ID, @@ -208,10 +235,19 @@ function parseArgs(argv: string[]): CliOptions { throw new Error(`--collector must be local or docker, got ${JSON.stringify(value)}`); } options.collectorMode = value; + } else if (arg === "--logs-exporter") { + const value = readValue(); + if (value !== "otlp" && value !== "stdout" && value !== "both") { + throw new Error( + `--logs-exporter must be otlp, stdout, or both, got ${JSON.stringify(value)}`, + ); + } + options.logsExporter = value; } else if (arg === "--provider-mode") { options.providerMode = readValue(); } else if (arg === "--scenario") { options.scenarioId = readValue(); + scenarioExplicit = true; } else if (arg === "--model") { options.primaryModel = readValue(); } else if (arg === "--alt-model") { @@ -221,6 +257,22 @@ function parseArgs(argv: string[]): CliOptions { } } + const expectedScenarioId = LOGS_EXPORTER_SCENARIO_IDS[options.logsExporter]; + const knownLogsExporterScenarioIds = new Set(Object.values(LOGS_EXPORTER_SCENARIO_IDS)); + if ( + scenarioExplicit && + knownLogsExporterScenarioIds.has(options.scenarioId) && + options.scenarioId !== expectedScenarioId + ) { + throw new Error( + `--logs-exporter ${options.logsExporter} requires --scenario ${expectedScenarioId}; ` + + `got ${options.scenarioId}`, + ); + } + if (!scenarioExplicit) { + options.scenarioId = expectedScenarioId; + } + return options; } @@ -944,6 +996,94 @@ function createBoundedTextAccumulator(maxBytes: number) { }; } +function trimUtf8Tail(value: string, maxBytes: number): string { + const buffer = Buffer.from(value, "utf8"); + if (buffer.length <= maxBytes) { + return value; + } + return buffer.subarray(buffer.length - maxBytes).toString("utf8"); +} + +function objectValue(value: object, key: string): unknown { + return Reflect.get(value, key); +} + +function isStdoutDiagnosticLogRecord(value: unknown): value is StdoutDiagnosticLogRecord { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + objectValue(value, "signal") === "openclaw.diagnostic.log" + ); +} + +function parseStdoutDiagnosticLogLine(line: string): StdoutDiagnosticLogRecord | undefined { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) { + return undefined; + } + try { + const parsed: unknown = JSON.parse(trimmed); + return isStdoutDiagnosticLogRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function createStdoutDiagnosticLogCapture(maxLineBytes = MAX_STDOUT_DIAGNOSTIC_LINE_BYTES) { + const records: StdoutDiagnosticLogRecord[] = []; + const lines: string[] = []; + let pendingLine = ""; + + const appendLine = (line: string) => { + const record = parseStdoutDiagnosticLogLine(line); + if (!record) { + return; + } + records.push(record); + lines.push(line.trim()); + }; + + return { + records, + lines, + append(chunk: unknown): void { + const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk); + const parts = text.split(/\r?\n/u); + parts[0] = `${pendingLine}${parts[0]}`; + pendingLine = trimUtf8Tail(parts.pop() ?? "", maxLineBytes); + for (const part of parts) { + appendLine(trimUtf8Tail(part, maxLineBytes)); + } + }, + flush(): void { + const line = pendingLine; + pendingLine = ""; + appendLine(line); + }, + }; +} + +async function appendGatewayStdoutArtifactLogs(params: { + capture: ReturnType; + outputDir: string; +}): Promise { + const gatewayStdoutPath = path.join( + params.outputDir, + "artifacts", + "gateway-runtime", + "gateway.stdout.log", + ); + try { + params.capture.append(await readFile(gatewayStdoutPath, "utf8")); + params.capture.flush(); + } catch (error) { + if (!isErrnoCode(error, "ENOENT")) { + throw error; + } + } +} + async function stopDockerContainer(name: string): Promise { await new Promise((resolve) => { const child = spawn("docker", ["stop", name], { @@ -1347,27 +1487,33 @@ async function delay(ms: number): Promise { }); } -function hasRequiredSmokeSignals(receiver: ReturnType): boolean { +function hasRequiredSmokeSignals(params: { + logsExporter: OtelLogsExporter; + receiver: ReturnType; +}): boolean { + const expectsOtlpLogs = params.logsExporter === "otlp" || params.logsExporter === "both"; + const receiver = params.receiver; const spanNames = new Set(receiver.capturedSpans.map((span) => span.name)); const metricNames = new Set(receiver.capturedMetrics.map((metric) => metric.name)); return ( REQUIRED_SPAN_NAMES.every((name) => spanNames.has(name)) && receiver.capturedSpans.some(isLatestGenAiModelCallSpan) && REQUIRED_METRIC_NAMES.every((name) => metricNames.has(name)) && - receiver.capturedLogRecords.length > 0 && - ["traces", "metrics", "logs"].every((signal) => - receiver.capturedRequests.some((request) => request.signal === signal), - ) + (!expectsOtlpLogs || receiver.capturedLogRecords.length > 0) && + receiver.capturedRequests.some((request) => request.signal === "traces") && + receiver.capturedRequests.some((request) => request.signal === "metrics") && + (!expectsOtlpLogs || receiver.capturedRequests.some((request) => request.signal === "logs")) ); } async function waitForExpectedTelemetry( receiver: ReturnType, + logsExporter: OtelLogsExporter, timeoutMs: number, ): Promise { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - if (hasRequiredSmokeSignals(receiver)) { + if (hasRequiredSmokeSignals({ logsExporter, receiver })) { return; } await delay(250); @@ -1387,18 +1533,23 @@ function formatBoundedList(values: readonly string[], maxItems: number): string function assertSmoke(params: { childExitCode: number; disallowedBodyNeedles: string[]; + logsExporter: OtelLogsExporter; spans: CapturedSpan[]; metrics: CapturedMetric[]; logRecords: CapturedLogRecord[]; + stdoutLogRecords: StdoutDiagnosticLogRecord[]; + stdoutLogLines: string[]; requests: CapturedRequest[]; bodyText: Partial>; }) { const failures: string[] = []; const leakContexts: Partial> = {}; + const expectsOtlpLogs = params.logsExporter === "otlp" || params.logsExporter === "both"; + const expectsStdoutLogs = params.logsExporter === "stdout" || params.logsExporter === "both"; if (params.childExitCode !== 0) { failures.push(`qa suite exited with ${params.childExitCode}`); } - for (const signal of ["traces", "metrics", "logs"] as const) { + for (const signal of ["traces", "metrics"] as const) { const requests = params.requests.filter((request) => request.signal === signal); if (requests.length === 0) { failures.push(`no OTLP ${signal} requests were received`); @@ -1411,15 +1562,39 @@ function assertSmoke(params: { failures.push(`OTLP ${signal} request ${request.path} returned status ${request.status}`); } } + const logRequests = params.requests.filter((request) => request.signal === "logs"); + if (expectsOtlpLogs && logRequests.length === 0) { + failures.push("no OTLP logs requests were received"); + } + if (!expectsOtlpLogs && logRequests.length > 0) { + failures.push("OTLP logs requests were received for stdout logs exporter"); + } + for (const request of logRequests) { + if (request.bytes === 0) { + failures.push("empty OTLP logs request received"); + } + if (request.status < 200 || request.status >= 300) { + failures.push(`OTLP logs request ${request.path} returned status ${request.status}`); + } + } if (params.spans.length === 0) { failures.push("no OTLP trace spans were decoded"); } if (params.metrics.length === 0) { failures.push("no OTLP metrics were decoded"); } - if (params.logRecords.length === 0) { + if (expectsOtlpLogs && params.logRecords.length === 0) { failures.push("no OTLP log records were decoded"); } + if (!expectsOtlpLogs && params.logRecords.length > 0) { + failures.push("OTLP log records were decoded for stdout logs exporter"); + } + if (!expectsStdoutLogs && params.stdoutLogRecords.length > 0) { + failures.push("stdout diagnostic log records were captured for OTLP logs exporter"); + } + if (expectsStdoutLogs && params.stdoutLogRecords.length === 0) { + failures.push("no stdout diagnostic log records were captured"); + } const spanNames = new Set(params.spans.map((span) => span.name)); for (const name of REQUIRED_SPAN_NAMES) { @@ -1443,9 +1618,33 @@ function assertSmoke(params: { const correlatedLogRecords = params.logRecords.filter( (record) => record.traceId && record.spanId, ); - if (correlatedLogRecords.length === 0) { + if (expectsOtlpLogs && correlatedLogRecords.length === 0) { failures.push("no OTLP log records included trace/span correlation ids"); } + for (const record of params.stdoutLogRecords) { + if (typeof record.ts !== "string" || !/^\d{4}-\d{2}-\d{2}T/u.test(record.ts)) { + failures.push("stdout diagnostic log record missing ISO timestamp"); + } + if (typeof record["service.name"] !== "string" || record["service.name"].trim() === "") { + failures.push("stdout diagnostic log record missing service.name"); + } + if (typeof record.severityText !== "string" || record.severityText.trim() === "") { + failures.push("stdout diagnostic log record missing severityText"); + } + if (typeof record.severityNumber !== "number") { + failures.push("stdout diagnostic log record missing numeric severityNumber"); + } + if (!Object.hasOwn(record, "body")) { + failures.push("stdout diagnostic log record missing body"); + } + if ( + typeof record.attributes !== "object" || + record.attributes === null || + Array.isArray(record.attributes) + ) { + failures.push("stdout diagnostic log record missing attributes object"); + } + } const attributeKeys = collectAttributeKeys(params.spans); const disallowed = [...DISALLOWED_ATTRIBUTE_KEYS].filter((key) => attributeKeys.has(key)); @@ -1487,6 +1686,16 @@ function assertSmoke(params: { failures.push(`OTLP ${signal} payload leaked content: ${leakedNeedles.join(", ")}`); } } + const stdoutLogText = params.stdoutLogLines.join("\n"); + const stdoutLeakedNeedles = params.disallowedBodyNeedles.filter((needle) => + stdoutLogText.includes(needle), + ); + if (stdoutLeakedNeedles.length > 0) { + leakContexts.logs = findNeedleContexts(stdoutLogText, stdoutLeakedNeedles); + failures.push( + `stdout diagnostic log payload leaked content: ${stdoutLeakedNeedles.join(", ")}`, + ); + } return { passed: failures.length === 0, @@ -1504,6 +1713,7 @@ function assertSmoke(params: { metrics: params.requests.filter((request) => request.signal === "metrics").length, logs: params.requests.filter((request) => request.signal === "logs").length, }, + stdoutLogRecordCount: params.stdoutLogRecords.length, }; } @@ -1523,6 +1733,7 @@ async function main() { let collector: Awaited> | undefined; let childExitCode = 1; + const stdoutDiagnosticLogs = createStdoutDiagnosticLogCapture(); try { let exportPort = port; if (options.collectorMode === "docker") { @@ -1535,15 +1746,25 @@ async function main() { const child = spawnOpenClaw(buildQaArgs(options), buildQaEnv(exportPort)); const cleanupSignalRelay = relayParentSignalsToChild(child); - child.stdout?.on("data", (chunk) => process.stdout.write(chunk)); + child.stdout?.on("data", (chunk) => { + stdoutDiagnosticLogs.append(chunk); + process.stdout.write(chunk); + }); child.stderr?.on("data", (chunk) => process.stderr.write(chunk)); try { childExitCode = await waitForChild(child); } finally { cleanupSignalRelay(); + stdoutDiagnosticLogs.flush(); + } + if (stdoutDiagnosticLogs.records.length === 0) { + await appendGatewayStdoutArtifactLogs({ + capture: stdoutDiagnosticLogs, + outputDir: options.outputDir, + }); } if (childExitCode === 0) { - await waitForExpectedTelemetry(receiver, 15_000); + await waitForExpectedTelemetry(receiver, options.logsExporter, 15_000); } else { await delay(3000); } @@ -1558,9 +1779,12 @@ async function main() { const assertion = assertSmoke({ childExitCode, disallowedBodyNeedles: disallowedBodyNeedles(options), + logsExporter: options.logsExporter, spans: receiver.capturedSpans, metrics: receiver.capturedMetrics, logRecords: receiver.capturedLogRecords, + stdoutLogRecords: stdoutDiagnosticLogs.records, + stdoutLogLines: stdoutDiagnosticLogs.lines, requests: receiver.capturedRequests, bodyText: receiver.capturedBodyText, }); @@ -1571,10 +1795,12 @@ async function main() { scenarioId: options.scenarioId, providerMode: options.providerMode, collectorMode: options.collectorMode, + logsExporter: options.logsExporter, requests: receiver.capturedRequests, spanCount: receiver.capturedSpans.length, metricCount: receiver.capturedMetrics.length, logRecordCount: receiver.capturedLogRecords.length, + stdoutLogRecordCount: stdoutDiagnosticLogs.records.length, logRecordsWithTraceContext: receiver.capturedLogRecords.filter( (record) => record.traceId && record.spanId, ).length, @@ -1583,6 +1809,7 @@ async function main() { signalRequestCounts: assertion.signalRequestCounts, modelSpanCount: assertion.modelSpanCount, modelErrorSpanCount: assertion.modelErrorSpanCount, + stdoutLogRecordCountFromAssertion: assertion.stdoutLogRecordCount, disallowedAttributeKeys: assertion.disallowedAttributeKeys, contentAttributeKeys: assertion.contentAttributeKeys, leakContexts: assertion.leakContexts, @@ -1601,6 +1828,13 @@ async function main() { logBodyKinds: [ ...new Set(receiver.capturedLogRecords.map((record) => capturedValueKind(record.body))), ], + stdoutLogBodyKinds: [ + ...new Set( + stdoutDiagnosticLogs.records.map((record) => + Array.isArray(record.body) ? "array" : typeof record.body, + ), + ), + ], }; const summaryPath = path.join(options.outputDir, "otel-smoke-summary.json"); await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); @@ -1616,7 +1850,8 @@ async function main() { ); process.stderr.write( `qa-otel-smoke: captured decoded counts spans=${receiver.capturedSpans.length} ` + - `metrics=${receiver.capturedMetrics.length} logs=${receiver.capturedLogRecords.length}\n`, + `metrics=${receiver.capturedMetrics.length} logs=${receiver.capturedLogRecords.length} ` + + `stdoutLogs=${stdoutDiagnosticLogs.records.length}\n`, ); process.stderr.write( `qa-otel-smoke: captured span names: ${formatBoundedList(assertion.spanNames, 40)}\n`, @@ -1639,6 +1874,7 @@ async function main() { process.stdout.write( `qa-otel-smoke: passed spans=${receiver.capturedSpans.length} ` + `metrics=${receiver.capturedMetrics.length} logs=${receiver.capturedLogRecords.length} ` + + `stdoutLogs=${stdoutDiagnosticLogs.records.length} ` + `traces=${assertion.signalRequestCounts.traces} ` + `metricRequests=${assertion.signalRequestCounts.metrics} ` + `logRequests=${assertion.signalRequestCounts.logs}\n`, @@ -1651,6 +1887,7 @@ export const testing = { createBoundedTextAccumulator, decodeRequestBody, parseArgs, + parseStdoutDiagnosticLogLine, readPositiveIntegerEnv, readRequestBody, startLocalOtlpReceiver, diff --git a/test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts b/test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts index 7085f44876e..e34e91965a0 100644 --- a/test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts +++ b/test/e2e/qa-lab/runtime/qa-otel-smoke.e2e.test.ts @@ -42,6 +42,7 @@ describe("qa-otel-smoke receiver bounds", () => { }, childExitCode: 0, disallowedBodyNeedles: ["OTEL-QA-SECRET"], + logsExporter: "otlp", logRecords: [ { body: "diagnostics-otel: logs exporter enabled", @@ -82,6 +83,8 @@ describe("qa-otel-smoke receiver bounds", () => { logCount: 1, }, ], + stdoutLogLines: [], + stdoutLogRecords: [], spans: [ { name: "openclaw.run", parent: false, attributes: {} }, { name: "openclaw.harness.run", parent: true, attributes: {} }, @@ -110,15 +113,42 @@ describe("qa-otel-smoke receiver bounds", () => { "--provider-mode", "mock-openai", "--scenario", - "otel-trace-smoke", + "otel-stdout-log-smoke", + "--logs-exporter", + "stdout", ]), ).toMatchObject({ collectorMode: "docker", + logsExporter: "stdout", providerMode: "mock-openai", - scenarioId: "otel-trace-smoke", + scenarioId: "otel-stdout-log-smoke", }); }); + it("selects the matching scenario for the requested log exporter", () => { + expect(testing.parseArgs(["--logs-exporter", "otlp"]).scenarioId).toBe("otel-trace-smoke"); + expect(testing.parseArgs(["--logs-exporter", "stdout"]).scenarioId).toBe( + "otel-stdout-log-smoke", + ); + expect(testing.parseArgs(["--logs-exporter", "both"]).scenarioId).toBe("otel-both-log-smoke"); + }); + + it("rejects explicit scenarios that do not match the log exporter", () => { + expect(() => + testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "otel-trace-smoke"]), + ).toThrow("--logs-exporter stdout requires --scenario otel-stdout-log-smoke"); + }); + + it("allows explicit custom scenarios to own their exporter config", () => { + expect(testing.parseArgs(["--scenario", "custom-otel-smoke"]).scenarioId).toBe( + "custom-otel-smoke", + ); + expect( + testing.parseArgs(["--logs-exporter", "stdout", "--scenario", "custom-stdout-smoke"]) + .scenarioId, + ).toBe("custom-stdout-smoke"); + }); + it("parses body-size limit env values as strict positive integers", () => { expect(testing.readPositiveIntegerEnv("OTEL_TEST_LIMIT", 64, {})).toBe(64); expect( @@ -276,6 +306,7 @@ describe("qa-otel-smoke receiver bounds", () => { bodyText: {}, childExitCode: 0, disallowedBodyNeedles: [], + logsExporter: "otlp", logRecords: [], metrics: [], requests: [ @@ -290,6 +321,8 @@ describe("qa-otel-smoke receiver bounds", () => { logCount: 0, }, ], + stdoutLogLines: [], + stdoutLogRecords: [], spans: [], }); @@ -304,6 +337,77 @@ describe("qa-otel-smoke receiver bounds", () => { expect(assertion.failures).toEqual([]); }); + it("allows stdout diagnostic logs without OTLP log requests", () => { + const input = makePassingSmokeAssertionInput(); + input.logsExporter = "stdout"; + input.bodyText = {}; + input.logRecords = []; + input.requests = input.requests.filter((request) => request.signal !== "logs"); + input.stdoutLogRecords = [ + { + ts: "2026-06-18T00:00:00.000Z", + signal: "openclaw.diagnostic.log", + "service.name": "openclaw-qa-lab-otel-smoke", + severityText: "INFO", + severityNumber: 9, + body: "log", + attributes: { + "openclaw.log.level": "INFO", + }, + }, + ]; + input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])]; + + const assertion = testing.assertSmoke(input); + + expect(assertion.passed).toBe(true); + expect(assertion.failures).toEqual([]); + expect(assertion.signalRequestCounts.logs).toBe(0); + expect(assertion.stdoutLogRecordCount).toBe(1); + }); + + it("fails stdout diagnostic mode when no stdout log records are captured", () => { + const input = makePassingSmokeAssertionInput(); + input.logsExporter = "stdout"; + input.bodyText = {}; + input.logRecords = []; + input.requests = input.requests.filter((request) => request.signal !== "logs"); + input.stdoutLogRecords = []; + input.stdoutLogLines = []; + + const assertion = testing.assertSmoke(input); + + expect(assertion.passed).toBe(false); + expect(assertion.failures).toContain("no stdout diagnostic log records were captured"); + expect(assertion.signalRequestCounts.logs).toBe(0); + expect(assertion.stdoutLogRecordCount).toBe(0); + }); + + it("fails stdout diagnostic mode when OTLP log requests are still emitted", () => { + const input = makePassingSmokeAssertionInput(); + input.logsExporter = "stdout"; + input.logRecords = []; + input.stdoutLogRecords = [ + { + ts: "2026-06-18T00:00:00.000Z", + signal: "openclaw.diagnostic.log", + "service.name": "openclaw-qa-lab-otel-smoke", + severityText: "INFO", + severityNumber: 9, + body: "log", + attributes: {}, + }, + ]; + input.stdoutLogLines = [JSON.stringify(input.stdoutLogRecords[0])]; + + const assertion = testing.assertSmoke(input); + + expect(assertion.passed).toBe(false); + expect(assertion.failures).toContain( + "OTLP logs requests were received for stdout logs exporter", + ); + }); + it("still fails when OTLP log payload text leaks scenario content", () => { const input = makePassingSmokeAssertionInput(); input.bodyText = { @@ -317,6 +421,39 @@ describe("qa-otel-smoke receiver bounds", () => { expect(assertion.leakContexts.logs?.[0]).toContain("[needle]"); }); + it("still fails when stdout diagnostic log payload text leaks scenario content", () => { + const input = makePassingSmokeAssertionInput(); + input.logsExporter = "stdout"; + input.bodyText = {}; + input.logRecords = []; + input.requests = input.requests.filter((request) => request.signal !== "logs"); + input.stdoutLogRecords = [ + { + ts: "2026-06-18T00:00:00.000Z", + signal: "openclaw.diagnostic.log", + "service.name": "openclaw-qa-lab-otel-smoke", + severityText: "INFO", + severityNumber: 9, + body: "log", + attributes: {}, + }, + ]; + input.stdoutLogLines = [ + JSON.stringify({ + ...input.stdoutLogRecords[0], + body: "diagnostics-otel: log payload contains OTEL-QA-SECRET", + }), + ]; + + const assertion = testing.assertSmoke(input); + + expect(assertion.passed).toBe(false); + expect(assertion.failures).toContain( + "stdout diagnostic log payload leaked content: OTEL-QA-SECRET", + ); + expect(assertion.leakContexts.logs?.[0]).toContain("[needle]"); + }); + it("still requires OTLP log records to carry trace correlation", () => { const input = makePassingSmokeAssertionInput(); input.logRecords = [