vrr/openclaw
2
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-09 15:59:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 22

Fold Telegram RTT sampling into live QA evidence (#92550)

Browse source 
* refactor(qa): fold telegram rtt into live evidence

* test: default package telegram rtt samples

* refactor(qa-lab): fold telegram rtt into live evidence

* fix(qa-lab): keep package telegram rtt optional for focused runs

* fix(qa-lab): avoid stale rtt evidence on failed samples

* fix(qa-lab): pass telegram live env into credential leasing

* fix(qa-lab): update telegram canary remediation artifacts

* docs(qa): remove stale telegram observed artifact guidance

* fix(qa-lab): clarify telegram empty-reply remediation

* fix(qa-lab): honor telegram rtt timeout

* ci(qa): drop stale telegram capture env

* refactor: align telegram evidence coverage fields

* fix: ignore stale telegram observed artifacts

* fix: preserve telegram rtt coverage mapping

* fix: omit unused telegram rtt catch binding

* docs: document telegram rtt check selector
This commit is contained in:
Dallin Romney 2026-06-14 02:02:33 -07:00 • committed by GitHub
parent 439a9e97fd
commit 1affe4fcdf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 729 additions and 4184 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/mantis-telegram-live.yml vendored
View file

@ -379,7 +379,6 @@ jobs:
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
CRABBOX_COORDINATOR: ${{ secrets.CRABBOX_COORDINATOR }}
CRABBOX_COORDINATOR_TOKEN: ${{ secrets.CRABBOX_COORDINATOR_TOKEN }}
OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR: ${{ secrets.OPENCLAW_QA_MANTIS_CRABBOX_COORDINATOR }}

1
.github/workflows/npm-telegram-beta-e2e.yml vendored
View file

@ -220,7 +220,6 @@ jobs:
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
INPUT_SCENARIO: ${{ inputs.scenario }}
PACKAGE_ARTIFACT_NAME: ${{ inputs.package_artifact_name || '' }}
run: |

1
.github/workflows/openclaw-release-checks.yml vendored
View file

@ -1412,7 +1412,6 @@ jobs:
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
run: |
set -euo pipefail

1
.github/workflows/qa-live-transports-convex.yml vendored
View file

@ -532,7 +532,6 @@ jobs:
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "1800000"
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1"
INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.scenario || '' }}
run: |
set -euo pipefail

17
docs/concepts/mantis.md
View file

@ -247,12 +247,13 @@ of only a bot-to-bot Slack transcript.
evidence pipeline. It checks out the trusted candidate ref in a separate
worktree, runs `pnpm openclaw qa telegram --credential-source convex
--credential-role ci`, writes a `mantis-evidence.json` manifest from the
Telegram QA summary and observed-message artifact, renders the redacted
transcript HTML through a Crabbox desktop browser, generates a motion-trimmed GIF
with `crabbox media preview`, and posts the inline PR evidence comment when a PR
number is available. This lane is transcript-visual rather than logged-in
Telegram Web proof: the Telegram Bot API gives stable live message evidence, but
Telegram Web login state is not required for normal Mantis automation.
Telegram QA summary, `qa-evidence.json`, and report artifacts, renders the
redacted evidence HTML through a Crabbox desktop browser, generates a
motion-trimmed GIF with `crabbox media preview`, and posts the inline PR
evidence comment when a PR number is available. This lane is QA-evidence visual
rather than logged-in Telegram Web proof: the Telegram Bot API gives stable live
message evidence, but Telegram Web login state is not required for normal Mantis
automation.
`Mantis Telegram Desktop Proof` is the agentic native Telegram Desktop
before/after wrapper. A maintainer can trigger it from a PR comment with
@ -494,8 +495,8 @@ zero:
- `pnpm openclaw qa discord` already runs a live Discord lane with driver and
SUT bots.
- The live transport runner already writes reports and observed-message
artifacts under `.artifacts/qa-e2e/`.
- The live transport runner already writes reports, QA evidence, and
transport-specific artifacts under `.artifacts/qa-e2e/`.
- Convex credential leases already provide exclusive access to shared live
transport credentials.
- The browser control service already supports screenshots, snapshots,

56
docs/concepts/qa-e2e-automation.md
View file

@ -318,17 +318,17 @@ Matrix has a [dedicated page](/concepts/qa-matrix) because of its scenario count
These lanes register through `extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts` and accept the same flags:
| Flag | Default | Description |
| ------------------------------------- | -------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
| `--scenario <id>` | - | Run only this scenario. Repeatable. |
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/<transport>-<timestamp>` | Where reports/summary/observed messages and the output log are written. Relative paths resolve against `--repo-root`. |
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
| `--fast` | off | Provider fast mode where supported. |
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
| Flag | Default | Description |
| ------------------------------------- | -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- |
| `--scenario <id>` | - | Run only this scenario. Repeatable. |
| `--output-dir <path>` | `<repo>/.artifacts/qa-e2e/<transport>-<timestamp>` | Where reports, summaries, evidence, transport-specific artifacts, and the output log are written. Relative paths resolve against `--repo-root`. |
| `--repo-root <path>` | `process.cwd()` | Repository root when invoking from a neutral cwd. |
| `--sut-account <id>` | `sut` | Temporary account id inside the QA gateway config. |
| `--provider-mode <mode>` | `live-frontier` | `mock-openai` or `live-frontier` (legacy `live-openai` still works). |
| `--model <ref>` / `--alt-model <ref>` | provider default | Primary/alternate model refs. |
| `--fast` | off | Provider fast mode where supported. |
| `--credential-source <env\|convex>` | `env` | See [Convex credential pool](#convex-credential-pool). |
| `--credential-role <maintainer\|ci>` | `ci` in CI, `maintainer` otherwise | Role used when `--credential-source convex`. |
Each lane exits non-zero on any failed scenario. `--allow-failures` writes artifacts without setting a failing exit code.
@ -346,10 +346,6 @@ Required env when `--credential-source env`:
- `OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN`
- `OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN`
Optional:
- `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1` keeps message bodies in observed-message artifacts (default redacts).
Scenarios (`extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts`):
- `telegram-canary`
@ -375,26 +371,26 @@ Output artifacts:
- `telegram-qa-report.md`
- `qa-evidence.json` - evidence entries for the live transport checks, including profile, coverage, provider, channel, artifacts, result, and RTT fields.
- `telegram-qa-observed-messages.json` - bodies redacted unless `OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT=1`.
Package RTT comparison uses the same Telegram credential contract while keeping
its RTT sample controls on the RTT harness path:
Package Telegram runs use the same Telegram credential contract. Repeated RTT
measurement is part of the normal package Telegram live lane; the RTT
distribution is folded into `qa-evidence.json` under `result.timing` for the
selected RTT check.
```bash
pnpm rtt openclaw@beta \
--credential-source convex \
--credential-role maintainer \
--samples 20 \
--sample-timeout-ms 30000
OPENCLAW_QA_CREDENTIAL_SOURCE=convex \
pnpm test:docker:npm-telegram-live
```
When `--credential-source convex` is set, the RTT Docker wrapper leases a
`kind: "telegram"` credential, exports the leased group/driver/SUT bot env into
the installed-package run, heartbeats the lease, and releases it on shutdown.
`--samples` and `--sample-timeout-ms` still feed
`OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES` and
`OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS`, so `result.json` remains comparable
across env-backed and Convex-backed RTT runs.
When `OPENCLAW_QA_CREDENTIAL_SOURCE=convex` is set, the package live wrapper
leases a `kind: "telegram"` credential, exports the leased group/driver/SUT bot
env into the installed-package run, heartbeats the lease, and releases it on
shutdown. The package wrapper defaults to 20 RTT checks of
`telegram-mentioned-message-reply`, a 30s RTT timeout, and Convex role
`maintainer` outside CI when Convex is selected. Override
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`, `OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`,
or `OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune RTT measurement without
creating a separate RTT command or Telegram-specific summary format.
### Discord QA

20
docs/help/testing.md
View file

@ -218,17 +218,27 @@ inside every shard.
`OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ=/path/to/openclaw-current.tgz` or
`OPENCLAW_CURRENT_PACKAGE_TGZ` to test a resolved local tarball instead of
installing from the registry.
- Emits repeated RTT timing in `qa-evidence.json` by default with
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES=20`. Override
`OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES`,
`OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS`, or
`OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES` to tune the RTT run.
`OPENCLAW_NPM_TELEGRAM_RTT_CHECKS` accepts a comma-separated list of
Telegram QA check IDs to sample; when unset, the default RTT-capable check
is `telegram-mentioned-message-reply`.
- Uses the same Telegram env credentials or Convex credential source as
`pnpm openclaw qa telegram`. For CI/release automation, set
`OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE=convex` plus
`OPENCLAW_QA_CONVEX_SITE_URL` and the role secret. If
`OPENCLAW_QA_CONVEX_SITE_URL` and a role secret. If
`OPENCLAW_QA_CONVEX_SITE_URL` and a Convex role secret are present in CI,
the Docker wrapper selects Convex automatically.
- The wrapper validates Telegram or Convex credential env on the host before
Docker build/install work. Set `OPENCLAW_NPM_TELEGRAM_SKIP_CREDENTIAL_PREFLIGHT=1`
only when deliberately debugging pre-credential setup.
- `OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE=ci|maintainer` overrides the shared
`OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only.
`OPENCLAW_QA_CREDENTIAL_ROLE` for this lane only. When Convex credentials
are selected and no role is set, the wrapper uses `ci` in CI and
`maintainer` outside CI.
- GitHub Actions exposes this lane as the manual maintainer workflow
`NPM Telegram Beta E2E`. It does not run on merge. The workflow uses the
`qa-live-shared` environment and Convex CI credential leases.
@ -344,11 +354,11 @@ gh workflow run package-acceptance.yml --ref main \
want artifacts without a failing exit code.
- Requires two distinct bots in the same private group, with the SUT bot exposing a Telegram username.
- For stable bot-to-bot observation, enable Bot-to-Bot Communication Mode in `@BotFather` for both bots and ensure the driver bot can observe group bot traffic.
- Writes a Telegram QA report, summary, and observed-messages artifact under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply.
- Writes a Telegram QA report, summary, and `qa-evidence.json` under `.artifacts/qa-e2e/...`. Replying scenarios include RTT from driver send request to observed SUT reply.
`Mantis Telegram Live` is the PR-evidence wrapper around this lane. It runs the
candidate ref with Convex-leased Telegram credentials, renders the redacted
observed-message transcript in a Crabbox desktop browser, records MP4 evidence,
candidate ref with Convex-leased Telegram credentials, renders the redacted QA
report/evidence bundle in a Crabbox desktop browser, records MP4 evidence,
generates a motion-trimmed GIF, uploads the artifact bundle, and posts inline PR
evidence through the Mantis GitHub App when `pr_number` is set. Maintainers can
start it from the Actions UI through `Mantis Scenario` (`scenario_id:

45
extensions/qa-lab/src/evidence-summary.test.ts
View file

@ -133,7 +133,6 @@ describe("evidence summary", () => {
artifactPaths: [
{ kind: "summary", path: QA_EVIDENCE_FILENAME },
{ kind: "report", path: "telegram-qa-report.md" },
{ kind: "transport-observations", path: "telegram-qa-observed-messages.json" },
],
env: {
OPENCLAW_QA_RUNNER: "crabbox",
@ -206,11 +205,6 @@ describe("evidence summary", () => {
path: "telegram-qa-report.md",
source: "telegram-live-transport",
},
{
kind: "transport-observations",
path: "telegram-qa-observed-messages.json",
source: "telegram-live-transport",
},
],
}),
result: {
@ -226,6 +220,45 @@ describe("evidence summary", () => {
]);
});
it("preserves aggregate live transport timing", () => {
const evidence = buildLiveTransportEvidenceSummary({
artifactPaths: [{ kind: "summary", path: QA_EVIDENCE_FILENAME }],
generatedAt: "2026-06-07T12:05:00.000Z",
primaryModel: "openai/gpt-5.5",
providerMode: "live-frontier",
checks: [
{
id: "telegram-mentioned-message-reply",
coverageIds: ["channels.telegram.mention-gating"],
title: "Telegram mentioned message gets a reply",
status: "pass",
details: "5 samples collected.",
rttMs: 2000,
timing: {
rttMs: 1200,
avgMs: 1300,
p50Ms: 1200,
p95Ms: 1800,
maxMs: 2200,
samples: 5,
failedSamples: 1,
},
},
],
transportId: "telegram",
});
expect(evidence.entries[0]?.result.timing).toEqual({
rttMs: 1200,
avgMs: 1300,
p50Ms: 1200,
p95Ms: 1800,
maxMs: 2200,
samples: 5,
failedSamples: 1,
});
});
it("builds Vitest runner evidence entries", () => {
const evidence = buildVitestEvidenceSummary({
artifactPaths: [

18
extensions/qa-lab/src/evidence-summary.ts
View file

@ -184,6 +184,7 @@ type QaEvidenceScenarioResultInput = {
name: string;
status: QaEvidenceStatusInput;
details?: string;
timing?: QaEvidenceTiming;
rttMs?: number;
rttMeasurement?: {
finalMatchedReplyRttMs?: number;
@ -195,6 +196,7 @@ type QaEvidenceLiveTransportCheckInput = {
title: string;
status: QaEvidenceStatusInput;
details: string;
timing?: QaEvidenceTiming;
rttMs?: number;
rttMeasurement?: {
finalMatchedReplyRttMs?: number;
@ -203,7 +205,10 @@ type QaEvidenceLiveTransportCheckInput = {
artifactPaths?: Readonly<Record<string, string>>;
};
type QaEvidenceRttInput = Pick<QaEvidenceScenarioResultInput, "rttMeasurement" | "rttMs">;
type QaEvidenceRttInput = Pick<
QaEvidenceScenarioResultInput,
"rttMeasurement" | "rttMs" | "timing"
>;
type QaEvidenceTestTargetInput = {
id: string;
@ -423,8 +428,17 @@ function failureForResult(result: {
}
function timingForRttResult(check: QaEvidenceRttInput) {
const timing: QaEvidenceTiming = { ...check.timing };
const rttMs = check.rttMeasurement?.finalMatchedReplyRttMs ?? check.rttMs;
return typeof rttMs === "number" && Number.isFinite(rttMs) && rttMs > 0 ? { rttMs } : undefined;
if (
timing.rttMs === undefined &&
typeof rttMs === "number" &&
Number.isFinite(rttMs) &&
rttMs > 0
) {
timing.rttMs = rttMs;
}
return Object.keys(timing).length > 0 ? timing : undefined;
}
function timingForTestResult(result: QaEvidenceTestResultInput) {

24
extensions/qa-lab/src/live-transports/shared/live-transport-result.ts Normal file
View file

@ -0,0 +1,24 @@
// Qa Lab plugin module implements shared live-transport result shapes.
import type { QaEvidenceTiming } from "../../evidence-summary.js";
export type LiveTransportRttMeasurement = {
finalMatchedReplyRttMs: number;
requestStartedAt: string;
responseObservedAt: string;
source: "request-to-observed-message";
};
export type LiveTransportCheckResult = {
id: string;
title: string;
status: "pass" | "fail";
details: string;
coverageIds?: readonly string[];
timing?: QaEvidenceTiming;
rttMs?: number;
requestStartedAt?: string;
responseObservedAt?: string;
rttMeasurement?: LiveTransportRttMeasurement;
sentMessageId?: number;
responseMessageId?: number;
};

83
extensions/qa-lab/src/live-transports/shared/live-transport-rtt.ts Normal file
View file

@ -0,0 +1,83 @@
// Qa Lab plugin module implements shared live-transport RTT behavior.
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import type { QaEvidenceTiming } from "../../evidence-summary.js";
export type LiveTransportRttOptions<CheckId extends string = string> = {
count: number;
timeoutMs: number;
maxFailures: number;
checkIds: Set<CheckId>;
};
export type LiveTransportRttSample = {
rttMs?: number;
status: "pass" | "fail";
};
function normalizePositiveRttInteger(value: number | undefined) {
if (
typeof value !== "number" ||
!Number.isSafeInteger(value) ||
value <= 0 ||
value > MAX_TIMER_TIMEOUT_MS
) {
return undefined;
}
return value;
}
export function normalizeLiveTransportRttOptions<CheckId extends string>(params: {
count?: number;
defaultCheckIds: readonly CheckId[];
knownCheckIds: ReadonlySet<CheckId>;
maxFailures?: number;
rawCheckIds?: readonly string[];
timeoutMs?: number;
unknownCheckMessage: (checkId: string) => string;
}): LiveTransportRttOptions<CheckId> | undefined {
const count = normalizePositiveRttInteger(params.count);
if (count === undefined) {
return undefined;
}
const rawCheckIds =
params.rawCheckIds && params.rawCheckIds.length > 0
? params.rawCheckIds
: params.defaultCheckIds;
const checkIds = new Set<CheckId>();
for (const checkId of rawCheckIds) {
if (!params.knownCheckIds.has(checkId as CheckId)) {
throw new Error(params.unknownCheckMessage(checkId));
}
checkIds.add(checkId as CheckId);
}
return {
count,
maxFailures: normalizePositiveRttInteger(params.maxFailures) ?? count,
checkIds,
timeoutMs: normalizePositiveRttInteger(params.timeoutMs) ?? 30_000,
};
}
export function percentile(sortedValues: readonly number[], percentileValue: number) {
if (sortedValues.length === 0) {
return undefined;
}
const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1;
return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)];
}
export function summarizeLiveTransportRttSamples(samples: readonly LiveTransportRttSample[]) {
const passed = samples.filter((sample) => sample.status === "pass" && sample.rttMs !== undefined);
const sorted = passed.map((sample) => sample.rttMs as number).toSorted((a, b) => a - b);
const sum = sorted.reduce((total, value) => total + value, 0);
const timing: QaEvidenceTiming = {
rttMs: percentile(sorted, 50),
avgMs: sorted.length > 0 ? Math.round(sum / sorted.length) : undefined,
p50Ms: percentile(sorted, 50),
p95Ms: percentile(sorted, 95),
maxMs: sorted.at(-1),
samples: samples.length,
failedSamples: samples.length - passed.length,
};
return { passed: passed.length, failed: samples.length - passed.length, timing };
}

1
extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts
View file

@ -22,7 +22,6 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions)
printLiveTransportQaArtifacts("Telegram QA", {
report: result.reportPath,
summary: result.summaryPath,
"observed messages": result.observedMessagesPath,
});
if (!runOptions.allowFailures) {
const failedScenarioCount = await readQaSuiteFailedScenarioCountFromFile(result.summaryPath);

260
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.test.ts
View file

@ -2,6 +2,7 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import { afterEach, describe, expect, it, vi } from "vitest";
import { summarizeLiveTransportRttSamples } from "../shared/live-transport-rtt.js";
import {
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
findMissingLiveTransportStandardScenarios,
@ -159,6 +160,78 @@ describe("telegram live qa runtime", () => {
}
});
it("normalizes Telegram RTT options", () => {
expect(testing.normalizeTelegramQaRttOptions({})).toBeUndefined();
expect(
testing.normalizeTelegramQaRttOptions({
count: 3,
timeoutMs: 45_000,
}),
).toEqual({
count: 3,
maxFailures: 3,
checkIds: new Set(["telegram-mentioned-message-reply"]),
timeoutMs: 45_000,
});
expect(
testing.normalizeTelegramQaRttOptions({
checkIds: ["telegram-mentioned-message-reply"],
count: 3,
maxFailures: 1,
}),
).toEqual({
count: 3,
maxFailures: 1,
checkIds: new Set(["telegram-mentioned-message-reply"]),
timeoutMs: 30_000,
});
});
it("rejects unknown Telegram RTT checks", () => {
expect(() =>
testing.normalizeTelegramQaRttOptions({
checkIds: ["telegram-rtt-only"],
count: 1,
}),
).toThrow("unknown Telegram QA RTT check: telegram-rtt-only");
});
it("summarizes live transport RTT timing", () => {
expect(
summarizeLiveTransportRttSamples([
{ status: "pass", rttMs: 1000 },
{ status: "pass", rttMs: 2000 },
{ status: "pass", rttMs: 4000 },
{ status: "fail" },
]),
).toEqual({
passed: 3,
failed: 1,
timing: {
rttMs: 2000,
avgMs: 2333,
p50Ms: 2000,
p95Ms: 4000,
maxMs: 4000,
samples: 4,
failedSamples: 1,
},
});
expect(summarizeLiveTransportRttSamples([{ status: "fail" }, { status: "fail" }])).toEqual({
passed: 0,
failed: 2,
timing: {
rttMs: undefined,
avgMs: undefined,
p50Ms: undefined,
p95Ms: undefined,
maxMs: undefined,
samples: 2,
failedSamples: 2,
},
});
});
it("sanitizes and truncates Telegram live progress details", () => {
expect(testing.sanitizeTelegramQaProgressValue("scenario\nid\tvalue")).toBe(
"scenario id value",
@ -580,6 +653,10 @@ describe("telegram live qa runtime", () => {
.find((scenario) => scenario.id === "telegram-mentioned-message-reply")
?.buildRun("sut_bot").steps[0].replyToLatestSutMessage,
).toBe(true);
expect(
scenarios.find((scenario) => scenario.id === "telegram-mentioned-message-reply")
?.evidenceCoverageIds,
).toEqual(["channels.telegram.mention-gating"]);
const replyChainStep = requireScenario(scenarios, "telegram-reply-chain-exact-marker").buildRun(
"sut_bot",
).steps[0];
@ -1140,157 +1217,6 @@ describe("telegram live qa runtime", () => {
expect(observedMessages[0]?.scenarioId).toBe("telegram-whoami-command");
});
it("redacts observed message content by default in artifacts", () => {
expect(
testing.buildObservedMessagesArtifact({
includeContent: false,
redactMetadata: false,
observedMessages: [
{
updateId: 1,
messageId: 9,
chatId: -100123,
senderId: 42,
senderIsBot: true,
senderUsername: "driver_bot",
text: "secret text",
caption: "secret caption",
replyToMessageId: 8,
timestamp: 1_700_000_000_000,
inlineButtons: ["Approve"],
mediaKinds: ["photo"],
},
],
}),
).toEqual([
{
updateId: 1,
messageId: 9,
chatId: -100123,
senderId: 42,
senderIsBot: true,
senderUsername: "driver_bot",
replyToMessageId: 8,
timestamp: 1_700_000_000_000,
inlineButtons: ["Approve"],
mediaKinds: ["photo"],
},
]);
});
it("keeps observed message content in public mode when capture is requested", () => {
const redacted = testing.buildObservedMessagesArtifact({
includeContent: true,
redactMetadata: true,
observedMessages: [
{
updateId: 1,
messageId: 9,
chatId: -100123,
senderId: 42,
senderIsBot: true,
senderUsername: "driver_bot",
text: "secret text",
caption: "secret caption",
replyToMessageId: 8,
timestamp: 1_700_000_000_000,
inlineButtons: ["Approve"],
mediaKinds: ["photo"],
},
],
});
expect(redacted).toEqual([
{
senderIsBot: true,
inlineButtonCount: 1,
mediaKinds: ["photo"],
text: "secret text",
caption: "secret caption",
},
]);
expect(redacted[0]).not.toHaveProperty("timestamp");
expect(redacted[0]).not.toHaveProperty("inlineButtons");
expect(redacted[0]).not.toHaveProperty("senderId");
expect(redacted[0]).not.toHaveProperty("senderUsername");
});
it("keeps raw timestamp and inline button text when metadata redaction is disabled", () => {
expect(
testing.buildObservedMessagesArtifact({
includeContent: true,
redactMetadata: false,
observedMessages: [
{
updateId: 1,
messageId: 9,
chatId: -100123,
senderId: 42,
senderIsBot: true,
senderUsername: "driver_bot",
text: "secret text",
caption: "secret caption",
replyToMessageId: 8,
timestamp: 1_700_000_000_000,
inlineButtons: ["Approve"],
mediaKinds: ["photo"],
},
],
}),
).toEqual([
{
updateId: 1,
messageId: 9,
chatId: -100123,
senderId: 42,
senderIsBot: true,
timestamp: 1_700_000_000_000,
inlineButtons: ["Approve"],
senderUsername: "driver_bot",
replyToMessageId: 8,
text: "secret text",
caption: "secret caption",
mediaKinds: ["photo"],
},
]);
});
it("adds scenario context to observed message artifacts", () => {
expect(
testing.buildObservedMessagesArtifact({
includeContent: false,
redactMetadata: true,
observedMessages: [
{
updateId: 11,
messageId: 21,
chatId: -100123,
senderId: 88,
senderIsBot: true,
senderUsername: "sut_bot",
scenarioId: "telegram-commands-command",
scenarioTitle: "Telegram commands list reply",
matchedScenario: false,
text: "noise from previous turn",
replyToMessageId: 19,
timestamp: 1_700_000_003_000,
inlineButtons: [],
mediaKinds: [],
},
],
}),
).toEqual([
{
scenarioId: "telegram-commands-command",
scenarioTitle: "Telegram commands list reply",
matchedScenario: false,
senderIsBot: true,
inlineButtonCount: 0,
mediaKinds: [],
},
]);
});
it("prints Telegram scenario RTT in the Markdown report", () => {
expect(
testing.renderTelegramQaMarkdown({
@ -1313,6 +1239,38 @@ describe("telegram live qa runtime", () => {
).toContain("- RTT: 4321ms");
});
it("prints Telegram repeated RTT timing in the Markdown report", () => {
const report = testing.renderTelegramQaMarkdown({
cleanupIssues: [],
credentialSource: "env",
groupId: "-100123",
redactMetadata: false,
startedAt: "2026-04-23T00:00:00.000Z",
finishedAt: "2026-04-23T00:00:10.000Z",
scenarios: [
{
id: "telegram-mentioned-message-reply",
title: "Telegram mentioned message gets a reply",
status: "pass",
details: "reply matched; 3/4 RTT checks passed",
rttMs: 2000,
timing: {
avgMs: 2333,
p50Ms: 2000,
p95Ms: 4000,
maxMs: 4000,
samples: 4,
failedSamples: 1,
},
},
],
});
expect(report).toContain("- Samples: 3/4");
expect(report).toContain("- P50: 2000ms");
expect(report).toContain("- P95: 4000ms");
});
it("formats phase-specific canary diagnostics with context", () => {
const error = new Error(
"SUT bot did not send any group reply after the canary command within 30s.",

466
extensions/qa-lab/src/live-transports/telegram/telegram-live.runtime.ts
View file

@ -1,9 +1,7 @@
// Qa Lab plugin module implements telegram live behavior.
import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
@ -12,9 +10,12 @@ import {
} from "openclaw/plugin-sdk/number-runtime";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { isRecord, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
import { z } from "zod";
import { QA_EVIDENCE_FILENAME, buildLiveTransportEvidenceSummary } from "../../evidence-summary.js";
import {
QA_EVIDENCE_FILENAME,
buildLiveTransportEvidenceSummary,
type QaEvidenceTiming,
} from "../../evidence-summary.js";
import { startQaGatewayChild } from "../../gateway-child.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
import {
@ -33,6 +34,13 @@ import {
redactQaLiveLaneIssues,
} from "../shared/live-artifacts.js";
import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js";
import type { LiveTransportCheckResult } from "../shared/live-transport-result.js";
import {
normalizeLiveTransportRttOptions,
summarizeLiveTransportRttSamples,
type LiveTransportRttOptions,
type LiveTransportRttSample,
} from "../shared/live-transport-rtt.js";
import {
collectLiveTransportStandardScenarioCoverage,
selectLiveTransportScenarios,
@ -91,8 +99,10 @@ type TelegramQaScenarioRun = {
type TelegramQaScenarioDefinition = LiveTransportScenarioDefinition<TelegramQaScenarioId> & {
buildRun: (sutUsername: string) => TelegramQaScenarioRun;
buildRttRun?: (params: { rttIndex: number; sutUsername: string }) => TelegramQaScenarioRun;
defaultEnabled?: boolean;
defaultProviderModes?: readonly QaProviderMode[];
evidenceCoverageIds?: readonly string[];
regressionRefs?: readonly string[];
rationale: string;
};
@ -115,44 +125,26 @@ type TelegramObservedMessage = {
mediaKinds: string[];
};
type TelegramObservedMessageArtifact = {
updateId?: number;
messageId?: number;
chatId?: number;
senderId?: number;
senderIsBot: boolean;
senderUsername?: string;
scenarioId?: string;
scenarioTitle?: string;
matchedScenario?: boolean;
text?: string;
caption?: string;
replyToMessageId?: number;
inlineButtonCount?: number;
timestamp?: number;
inlineButtons?: string[];
mediaKinds: string[];
};
const DEFAULT_TELEGRAM_QA_CANARY_TIMEOUT_MS = 30_000;
type TelegramQaScenarioResult = {
id: string;
standardId?: string;
title: string;
status: "pass" | "fail";
type TelegramQaScenarioResult = LiveTransportCheckResult;
function telegramLiveTransportCoverageIds(scenario: TelegramQaScenarioDefinition) {
if (scenario.evidenceCoverageIds) {
return scenario.evidenceCoverageIds;
}
return scenario.standardId ? [`channels.telegram.${scenario.standardId}`] : [];
}
type TelegramQaRttOptions = LiveTransportRttOptions<TelegramQaScenarioId>;
type TelegramQaRttResult = {
details: string;
rttMs?: number;
requestStartedAt?: string;
responseObservedAt?: string;
rttMeasurement?: {
finalMatchedReplyRttMs: number;
requestStartedAt: string;
responseObservedAt: string;
source: "request-to-observed-message";
};
sentMessageId?: number;
responseMessageId?: number;
driverOffset: number;
failed: number;
latestSutMessageId?: number;
passed: number;
timing: QaEvidenceTiming;
};
type TelegramQaCanaryPhase = "sut_reply_timeout" | "sut_reply_not_threaded" | "sut_reply_empty";
@ -161,7 +153,6 @@ type TelegramQaRunResult = {
outputDir: string;
reportPath: string;
summaryPath: string;
observedMessagesPath: string;
gatewayDebugDirPath?: string;
scenarios: TelegramQaScenarioResult[];
};
@ -420,6 +411,7 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
{
id: "telegram-mentioned-message-reply",
title: "Telegram mentioned message gets a reply",
evidenceCoverageIds: ["channels.telegram.mention-gating"],
rationale: "Bot-to-bot group mention routing must produce a threaded SUT reply.",
timeoutMs: 45_000,
buildRun: (sutUsername) =>
@ -428,6 +420,16 @@ const TELEGRAM_QA_SCENARIOS: TelegramQaScenarioDefinition[] = [
input: `@${sutUsername} Telegram QA mention routing check. Reply with a short acknowledgement.`,
replyToLatestSutMessage: true,
}),
buildRttRun: ({ rttIndex, sutUsername }) => {
const marker = `QA-TELEGRAM-RTT-${rttIndex}-${randomUUID().slice(0, 8).toUpperCase()}`;
return telegramQaStepRun({
expectReply: true,
input: `@${sutUsername} Telegram RTT check ${rttIndex}. Reply exactly: ${marker}`,
expectedTextIncludes: [marker],
matchText: marker,
replyToLatestSutMessage: true,
});
},
},
{
id: "telegram-reply-chain-exact-marker",
@ -534,12 +536,10 @@ const TELEGRAM_QA_ENV_KEYS = [
"OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN",
"OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
] as const;
const TELEGRAM_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT";
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
const QA_SUITE_PROGRESS_ENV = "OPENCLAW_QA_SUITE_PROGRESS";
const TELEGRAM_QA_PROGRESS_DETAIL_LIMIT = 240;
const TELEGRAM_QA_PROGRESS_PREFIX = "[qa-telegram-live]";
const execFileAsync = promisify(execFile);
const telegramQaCredentialPayloadSchema = z.object({
groupId: z.string().trim().min(1),
@ -617,6 +617,44 @@ function resolveTelegramQaScenarioTimeoutMs(
return parsePositiveTelegramQaEnvMs(env, "OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", fallbackMs);
}
function normalizeTelegramQaRttOptions(params: {
count?: number;
checkIds?: readonly string[];
maxFailures?: number;
timeoutMs?: number;
}): TelegramQaRttOptions | undefined {
const knownScenarioIds = new Set(TELEGRAM_QA_SCENARIOS.map((scenario) => scenario.id));
return normalizeLiveTransportRttOptions({
count: params.count,
defaultCheckIds: ["telegram-mentioned-message-reply"],
knownCheckIds: knownScenarioIds,
maxFailures: params.maxFailures,
rawCheckIds: params.checkIds,
timeoutMs: params.timeoutMs,
unknownCheckMessage: (checkId) => `unknown Telegram QA RTT check: ${checkId}`,
});
}
function assertTelegramQaRttCheckSupport(params: {
rttOptions?: TelegramQaRttOptions;
scenarios: TelegramQaScenarioDefinition[];
}) {
if (!params.rttOptions) {
return;
}
const selectedScenarioIds = new Set(params.scenarios.map((scenario) => scenario.id));
for (const scenarioId of params.rttOptions.checkIds) {
if (!selectedScenarioIds.has(scenarioId)) {
throw new Error(`Telegram QA RTT check ${scenarioId} is not selected.`);
}
}
for (const scenario of params.scenarios) {
if (params.rttOptions.checkIds.has(scenario.id) && !scenario.buildRttRun) {
throw new Error(`Telegram QA scenario ${scenario.id} does not support RTT measurement.`);
}
}
}
function formatTelegramQaTimeoutSeconds(timeoutMs: number) {
return `${Math.round(timeoutMs / 1_000)}s`;
}
@ -1277,6 +1315,23 @@ function renderTelegramQaMarkdown(params: {
if (scenario.rttMs !== undefined) {
lines.push(`- RTT: ${scenario.rttMs}ms`);
}
if (scenario.timing?.samples !== undefined) {
lines.push(
`- Samples: ${scenario.timing.samples - (scenario.timing.failedSamples ?? 0)}/${scenario.timing.samples}`,
);
if (scenario.timing.avgMs !== undefined) {
lines.push(`- Avg: ${scenario.timing.avgMs}ms`);
}
if (scenario.timing.p50Ms !== undefined) {
lines.push(`- P50: ${scenario.timing.p50Ms}ms`);
}
if (scenario.timing.p95Ms !== undefined) {
lines.push(`- P95: ${scenario.timing.p95Ms}ms`);
}
if (scenario.timing.maxMs !== undefined) {
lines.push(`- Max: ${scenario.timing.maxMs}ms`);
}
}
lines.push("");
}
if (params.gatewayDebugDirPath) {
@ -1296,50 +1351,6 @@ function renderTelegramQaMarkdown(params: {
return lines.join("\n");
}
function buildObservedMessagesArtifact(params: {
observedMessages: TelegramObservedMessage[];
includeContent: boolean;
redactMetadata: boolean;
}) {
return params.observedMessages.map<TelegramObservedMessageArtifact>((message) => {
const scenarioContext = {
...(message.scenarioId ? { scenarioId: message.scenarioId } : {}),
...(message.scenarioTitle ? { scenarioTitle: message.scenarioTitle } : {}),
...(typeof message.matchedScenario === "boolean"
? { matchedScenario: message.matchedScenario }
: {}),
};
const base = params.redactMetadata
? {
...scenarioContext,
senderIsBot: message.senderIsBot,
inlineButtonCount: message.inlineButtons.length,
mediaKinds: message.mediaKinds,
}
: {
...scenarioContext,
senderIsBot: message.senderIsBot,
timestamp: message.timestamp,
inlineButtons: message.inlineButtons,
mediaKinds: message.mediaKinds,
updateId: message.updateId,
messageId: message.messageId,
chatId: message.chatId,
senderId: message.senderId,
senderUsername: message.senderUsername,
replyToMessageId: message.replyToMessageId,
};
if (!params.includeContent) {
return base;
}
return {
...base,
text: message.text,
caption: message.caption,
};
});
}
function shouldRunTelegramScenarioByDefault(
scenario: TelegramQaScenarioDefinition,
providerMode: QaProviderMode,
@ -1447,16 +1458,19 @@ function resolveTelegramQaScenarioSteps(run: TelegramQaScenarioRun): TelegramQaS
async function runTelegramQaScenarioStep(params: {
driverOffset: number;
driverToken: string;
env: NodeJS.ProcessEnv;
groupId: string;
latestSutMessageId?: number;
observedMessages: TelegramObservedMessage[];
replyTimeoutMs?: number;
scenario: TelegramQaScenarioDefinition;
step: TelegramQaScenarioStep;
sutBotId: number;
}) {
const fallbackTimeoutMs = params.step.timeoutMs ?? params.scenario.timeoutMs;
const stepTimeoutMs = params.step.expectReply
? resolveTelegramQaScenarioTimeoutMs(params.step.timeoutMs ?? params.scenario.timeoutMs)
: (params.step.timeoutMs ?? params.scenario.timeoutMs);
? (params.replyTimeoutMs ?? resolveTelegramQaScenarioTimeoutMs(fallbackTimeoutMs, params.env))
: fallbackTimeoutMs;
const requestStartedAtMs = Date.now();
const sent = await sendGroupMessage(
params.driverToken,
@ -1507,6 +1521,82 @@ async function runTelegramQaScenarioStep(params: {
}
}
async function runTelegramQaRttChecks(params: {
driverOffset: number;
driverToken: string;
env: NodeJS.ProcessEnv;
groupId: string;
latestSutMessageId?: number;
observedMessages: TelegramObservedMessage[];
rttOptions: TelegramQaRttOptions;
scenario: TelegramQaScenarioDefinition;
sutBotId: number;
sutUsername: string;
}): Promise<TelegramQaRttResult> {
if (!params.scenario.buildRttRun) {
throw new Error(`Telegram QA scenario ${params.scenario.id} does not support RTT measurement.`);
}
let driverOffset = params.driverOffset;
let latestSutMessageId = params.latestSutMessageId;
const samples: LiveTransportRttSample[] = [];
let failures = 0;
let passed = 0;
for (let index = 1; passed < params.rttOptions.count; index += 1) {
const run = params.scenario.buildRttRun({
rttIndex: index,
sutUsername: params.sutUsername,
});
const steps = resolveTelegramQaScenarioSteps(run);
if (steps.length !== 1) {
throw new Error(`Telegram QA RTT check ${params.scenario.id} must have one step.`);
}
try {
driverOffset = await flushTelegramUpdates(params.driverToken);
const stepResult = await runTelegramQaScenarioStep({
driverOffset,
driverToken: params.driverToken,
env: params.env,
groupId: params.groupId,
latestSutMessageId,
observedMessages: params.observedMessages,
replyTimeoutMs: params.rttOptions.timeoutMs,
scenario: params.scenario,
step: steps[0],
sutBotId: params.sutBotId,
});
if (!stepResult.matched) {
throw new Error("RTT check did not expect a reply");
}
driverOffset = stepResult.matched.nextOffset;
latestSutMessageId = stepResult.matched.message.messageId;
const rttMs = stepResult.matched.observedAtMs - stepResult.requestStartedAtMs;
samples.push({
status: "pass",
rttMs,
});
passed += 1;
} catch {
failures += 1;
samples.push({
status: "fail",
});
}
if (failures >= params.rttOptions.maxFailures) {
break;
}
}
const summary = summarizeLiveTransportRttSamples(samples);
return {
details: `${summary.passed}/${samples.length} RTT checks passed`,
driverOffset,
failed: summary.failed,
latestSutMessageId,
passed: summary.passed,
timing: summary.timing,
};
}
function classifyCanaryReply(params: {
message: TelegramObservedMessage;
groupId: string;
@ -1646,13 +1736,13 @@ function canaryFailureMessage(params: {
return [
"1. Check whether the SUT bot is replying in the group without threading to the driver message.",
"2. Confirm the Telegram native command path preserves reply-to behavior for group commands.",
"3. Inspect the observed messages artifact for the mismatched SUT message id and reply target.",
"3. Inspect telegram-qa-report.md and gateway debug logs for the mismatched SUT message id and reply target.",
];
case "sut_reply_empty":
return [
"1. Inspect the observed messages artifact to confirm whether the SUT sent media-only or blank text.",
"2. Check whether the Telegram native command response path produced an empty or suppressed reply.",
"3. Confirm the SUT command completed successfully in gateway logs.",
"1. Check whether the Telegram native command response path produced an empty or suppressed reply.",
"2. Confirm the SUT command completed successfully in gateway logs.",
"3. Inspect telegram-qa-report.md for the matched message ids and phase context.",
];
default:
return [
@ -1679,80 +1769,28 @@ function canaryFailureMessage(params: {
].join("\n");
}
async function runInstalledOpenClawTelegramOnboardingPreflight(params: {
openClawCommand: string;
providerMode: ReturnType<typeof normalizeQaProviderMode>;
sutToken: string;
}) {
const tempRoot = await fs.mkdtemp(
path.join(resolvePreferredOpenClawTmpDir(), "openclaw-npm-telegram-"),
);
const homeDir = path.join(tempRoot, "home");
const stateDir = path.join(homeDir, ".openclaw");
await fs.mkdir(stateDir, { recursive: true });
const tokenPath = path.join(tempRoot, "sut-token.txt");
await fs.writeFile(tokenPath, params.sutToken, { encoding: "utf8", mode: 0o600 });
const env = {
...process.env,
HOME: homeDir,
OPENCLAW_HOME: stateDir,
OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"),
OPENCLAW_STATE_DIR: stateDir,
OPENCLAW_GATEWAY_TOKEN: "npm-telegram-live-onboard",
...(params.providerMode === "live-frontier"
? {}
: { OPENAI_API_KEY: process.env.OPENAI_API_KEY ?? "sk-openclaw-npm-telegram-preflight" }),
};
try {
await execFileAsync(
params.openClawCommand,
[
"onboard",
"--non-interactive",
"--accept-risk",
"--mode",
"local",
"--auth-choice",
"openai-api-key",
"--secret-input-mode",
"ref",
"--gateway-port",
"18789",
"--gateway-bind",
"loopback",
"--skip-daemon",
"--skip-ui",
"--skip-skills",
"--skip-health",
"--json",
],
{ env },
);
await execFileAsync(
params.openClawCommand,
["channels", "add", "--channel", "telegram", "--token-file", tokenPath],
{ env },
);
await execFileAsync(params.openClawCommand, ["doctor", "--non-interactive"], { env });
} finally {
await fs.rm(tempRoot, { recursive: true, force: true }).catch(() => {});
}
}
export async function runTelegramQaLive(params: {
env?: NodeJS.ProcessEnv;
repoRoot?: string;
outputDir?: string;
sutOpenClawCommand?: string;
preflightInstalledOnboarding?: boolean;
providerMode?: QaProviderModeInput;
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;
scenarioIds?: string[];
rttCount?: number;
rttTimeoutMs?: number;
maxRttFailures?: number;
rttCheckIds?: string[];
sutAccountId?: string;
credentialSource?: string;
credentialRole?: string;
redactPublicMetadata?: boolean;
progressEnabled?: boolean;
canaryTimeoutMs?: number;
}): Promise<TelegramQaRunResult> {
const env = params.env ?? process.env;
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const outputDir =
params.outputDir ??
@ -1766,17 +1804,25 @@ export async function runTelegramQaLive(params: {
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findScenario(params.scenarioIds, providerMode);
const progressEnabled = shouldLogTelegramQaLiveProgress();
const rttOptions = normalizeTelegramQaRttOptions({
checkIds: params.rttCheckIds,
count: params.rttCount,
maxFailures: params.maxRttFailures,
timeoutMs: params.rttTimeoutMs,
});
assertTelegramQaRttCheckSupport({ rttOptions, scenarios });
const progressEnabled = params.progressEnabled ?? shouldLogTelegramQaLiveProgress(env);
writeTelegramQaProgress(
progressEnabled,
`run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"}`,
`run start: scenarios=${scenarios.length} providerMode=${providerMode} fastMode=${params.fastMode === true ? "on" : "off"} rttChecks=${rttOptions?.count ?? 0}`,
);
const credentialLease = await acquireQaCredentialLease({
env,
kind: "telegram",
source: params.credentialSource,
role: params.credentialRole,
resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(),
resolveEnvPayload: () => resolveTelegramQaRuntimeEnv(env),
parsePayload: parseTelegramQaCredentialPayload,
});
const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease);
@ -1790,11 +1836,11 @@ export async function runTelegramQaLive(params: {
const runtimeEnv = credentialLease.payload;
const observedMessages: TelegramObservedMessage[] = [];
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
const includeObservedMessageContent = isTruthyOptIn(process.env[TELEGRAM_QA_CAPTURE_CONTENT_ENV]);
const redactPublicMetadata =
params.redactPublicMetadata ?? isTruthyOptIn(env[QA_REDACT_PUBLIC_METADATA_ENV]);
writeTelegramQaProgress(
progressEnabled,
`runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"} captureContent=${includeObservedMessageContent ? "on" : "off"}`,
`runtime: redactMetadata=${redactPublicMetadata ? "on" : "off"}`,
);
const startedAt = new Date().toISOString();
const scenarioResults: TelegramQaScenarioResult[] = [];
@ -1803,16 +1849,6 @@ export async function runTelegramQaLive(params: {
let preservedGatewayDebugArtifacts = false;
let canaryFailure: string | null = null;
try {
if (params.sutOpenClawCommand && params.preflightInstalledOnboarding === true) {
writeTelegramQaProgress(progressEnabled, "installed package onboarding preflight start");
await runInstalledOpenClawTelegramOnboardingPreflight({
openClawCommand: params.sutOpenClawCommand,
providerMode,
sutToken: runtimeEnv.sutToken,
});
writeTelegramQaProgress(progressEnabled, "installed package onboarding preflight pass");
}
const driverIdentity = await getBotIdentity(runtimeEnv.driverToken);
const sutIdentity = await getBotIdentity(runtimeEnv.sutToken);
const sutUsername = sutIdentity.username?.trim();
@ -1866,13 +1902,13 @@ export async function runTelegramQaLive(params: {
groupId: runtimeEnv.groupId,
sutUsername,
sutBotId: sutIdentity.id,
timeoutMs: resolveTelegramQaCanaryTimeoutMs(),
timeoutMs: params.canaryTimeoutMs ?? resolveTelegramQaCanaryTimeoutMs(env),
observedMessages,
});
latestSutMessageId = canaryTiming.responseMessageId;
scenarioResults.push({
id: "telegram-canary",
standardId: "canary",
coverageIds: ["channels.telegram.canary"],
title: "Telegram canary",
status: "pass",
details: redactPublicMetadata
@ -1903,7 +1939,7 @@ export async function runTelegramQaLive(params: {
});
scenarioResults.push({
id: "telegram-canary",
standardId: "canary",
coverageIds: ["channels.telegram.canary"],
title: "Telegram canary",
status: "fail",
details: canaryFailure,
@ -1946,6 +1982,7 @@ export async function runTelegramQaLive(params: {
const stepResult = await runTelegramQaScenarioStep({
driverOffset,
driverToken: runtimeEnv.driverToken,
env,
groupId: runtimeEnv.groupId,
latestSutMessageId,
observedMessages,
@ -1999,7 +2036,7 @@ export async function runTelegramQaLive(params: {
if (!lastMatched || !firstRequestStartedAt || lastSentMessageId === undefined) {
const result = {
id: scenario.id,
standardId: scenario.standardId,
coverageIds: telegramLiveTransportCoverageIds(scenario),
title: scenario.title,
status: "pass",
details: "no reply",
@ -2021,35 +2058,64 @@ export async function runTelegramQaLive(params: {
: `; observed ${lastStep.expectedSutMessageCountRange[0]}-${lastStep.expectedSutMessageCountRange[1]} SUT message(s)`
: `; observed ${lastStep.expectedSutMessageCount} SUT message(s)`
: `; ${scenarioSteps.filter((step) => step.expectReply).length} command replies matched`;
let resultStatus: "pass" | "fail" = "pass";
let details = redactPublicMetadata
? `reply matched in ${rttMs}ms${suffix}`
: `reply message ${lastMatched.message.messageId} matched in ${rttMs}ms${suffix}`;
let resultRttMs: number | undefined = rttMs;
let timing: QaEvidenceTiming | undefined;
if (rttOptions?.checkIds.has(scenario.id)) {
const rttResult = await runTelegramQaRttChecks({
driverOffset,
driverToken: runtimeEnv.driverToken,
env,
groupId: runtimeEnv.groupId,
latestSutMessageId,
observedMessages,
rttOptions,
scenario,
sutBotId: sutIdentity.id,
sutUsername,
});
driverOffset = rttResult.driverOffset;
latestSutMessageId = rttResult.latestSutMessageId ?? latestSutMessageId;
timing = rttResult.timing;
resultRttMs = rttResult.timing.p50Ms;
details = `${details}; ${rttResult.details}`;
if (rttResult.passed < rttOptions.count) {
resultStatus = "fail";
}
}
const result = {
id: scenario.id,
standardId: scenario.standardId,
coverageIds: telegramLiveTransportCoverageIds(scenario),
title: scenario.title,
status: "pass",
details: redactPublicMetadata
? `reply matched in ${rttMs}ms${suffix}`
: `reply message ${lastMatched.message.messageId} matched in ${rttMs}ms${suffix}`,
rttMs,
status: resultStatus,
details,
rttMs: resultRttMs,
timing,
requestStartedAt: firstRequestStartedAt,
responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(),
rttMeasurement: {
finalMatchedReplyRttMs: rttMs,
requestStartedAt: new Date(lastRequestStartedAtMs).toISOString(),
responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(),
source: "request-to-observed-message",
},
rttMeasurement: timing
? undefined
: {
finalMatchedReplyRttMs: rttMs,
requestStartedAt: new Date(lastRequestStartedAtMs).toISOString(),
responseObservedAt: new Date(lastMatched.observedAtMs).toISOString(),
source: "request-to-observed-message",
},
sentMessageId: redactPublicMetadata ? undefined : lastSentMessageId,
responseMessageId: redactPublicMetadata ? undefined : lastMatched.message.messageId,
} satisfies TelegramQaScenarioResult;
scenarioResults.push(result);
writeTelegramQaProgress(
progressEnabled,
`scenario pass ${scenarioIndexLabel}: ${scenarioIdForLog}`,
`scenario ${resultStatus} ${scenarioIndexLabel}: ${scenarioIdForLog}`,
);
} catch (error) {
const result = {
id: scenario.id,
standardId: scenario.standardId,
coverageIds: telegramLiveTransportCoverageIds(scenario),
title: scenario.title,
status: "fail",
details: formatErrorMessage(error),
@ -2100,21 +2166,16 @@ export async function runTelegramQaLive(params: {
}
const reportPath = path.join(outputDir, "telegram-qa-report.md");
const summaryPath = path.join(outputDir, QA_EVIDENCE_FILENAME);
const observedMessagesPath = path.join(outputDir, "telegram-qa-observed-messages.json");
const evidence = buildLiveTransportEvidenceSummary({
artifactPaths: [
{ kind: "summary", path: path.basename(summaryPath) },
{ kind: "report", path: path.basename(reportPath) },
{ kind: "transport-observations", path: path.basename(observedMessagesPath) },
],
env: process.env,
env,
generatedAt: finishedAt,
primaryModel,
providerMode,
checks: scenarioResults.map(({ standardId, ...check }) => ({
...check,
coverageIds: standardId ? [`channels.telegram.${standardId}`] : undefined,
})),
checks: scenarioResults,
transportId: "telegram",
});
await fs.writeFile(
@ -2135,23 +2196,9 @@ export async function runTelegramQaLive(params: {
encoding: "utf8",
mode: 0o600,
});
await fs.writeFile(
observedMessagesPath,
`${JSON.stringify(
buildObservedMessagesArtifact({
observedMessages,
includeContent: includeObservedMessageContent,
redactMetadata: redactPublicMetadata,
}),
null,
2,
)}\n`,
{ encoding: "utf8", mode: 0o600 },
);
const artifactPaths = {
report: reportPath,
summary: summaryPath,
observedMessages: observedMessagesPath,
...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}),
};
if (canaryFailure) {
@ -2176,7 +2223,6 @@ export async function runTelegramQaLive(params: {
outputDir,
reportPath,
summaryPath,
observedMessagesPath,
...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}),
scenarios: scenarioResults,
};
@ -2186,7 +2232,6 @@ export const testing = {
TELEGRAM_QA_SCENARIOS,
TELEGRAM_QA_STANDARD_SCENARIO_IDS,
buildTelegramQaConfig,
buildObservedMessagesArtifact,
canaryFailureMessage,
callTelegramApi,
assertTelegramCanaryPresenceReply,
@ -2201,6 +2246,7 @@ export const testing = {
normalizeTelegramObservedMessage,
parseTelegramQaProgressBooleanEnv,
parseTelegramQaCredentialPayload,
normalizeTelegramQaRttOptions,
resolveTelegramQaCanaryTimeoutMs,
resolveTelegramQaScenarioTimeoutMs,
resolveTelegramQaRuntimeEnv,

1
package.json
View file

@ -1666,7 +1666,6 @@
"release:plugins:npm:plan": "node --import tsx scripts/plugin-npm-release-plan.ts",
"release:verify-beta": "node --import tsx scripts/release-verify-beta.ts",
"release:prep": "node scripts/release-preflight.mjs --fix",
"rtt": "node --import tsx scripts/rtt.ts",
"runtime-sidecars:check": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --check",
"runtime-sidecars:gen": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --write",
"start": "node openclaw.mjs",

13
scripts/e2e/npm-telegram-live-docker.sh
View file

@ -93,8 +93,12 @@ fi
credential_source="$(resolve_credential_source)"
credential_role="$(resolve_credential_role)"
if [ -z "$credential_role" ] && [ -n "${CI:-}" ] && [ "$credential_source" = "convex" ]; then
credential_role="ci"
if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then
if [ -n "${CI:-}" ]; then
credential_role="ci"
else
credential_role="maintainer"
fi
fi
validate_credential_preflight() {
@ -205,7 +209,6 @@ for key in \
OPENCLAW_QA_ALLOW_INSECURE_HTTP \
OPENCLAW_QA_REDACT_PUBLIC_METADATA \
OPENCLAW_QA_PACKAGE_SOURCE_SHA \
OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT \
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS \
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS \
OPENCLAW_QA_SUITE_PROGRESS \
@ -213,6 +216,10 @@ for key in \
OPENCLAW_NPM_TELEGRAM_MODEL \
OPENCLAW_NPM_TELEGRAM_ALT_MODEL \
OPENCLAW_NPM_TELEGRAM_SCENARIOS \
OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES \
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS \
OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS \
OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES \
OPENCLAW_NPM_TELEGRAM_SKIP_HOTPATH \
OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT \
OPENCLAW_NPM_TELEGRAM_ALLOW_FAILURES; do

45
scripts/e2e/npm-telegram-live-runner.ts
View file

@ -17,6 +17,21 @@ function splitCsv(value: string | undefined) {
.filter((entry) => entry.length > 0);
}
function parsePositiveIntegerEnv(env: NodeJS.ProcessEnv, name: string) {
const raw = env[name]?.trim();
if (!raw) {
return undefined;
}
if (!/^\d+$/u.test(raw)) {
throw new Error(`invalid ${name}: ${raw}`);
}
const value = Number(raw);
if (!Number.isSafeInteger(value) || value <= 0) {
throw new Error(`invalid ${name}: ${raw}`);
}
return value;
}
function resolveCredentialSource(env: NodeJS.ProcessEnv) {
return env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE;
}
@ -25,6 +40,27 @@ function resolveCredentialRole(env: NodeJS.ProcessEnv) {
return env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE ?? env.OPENCLAW_QA_CREDENTIAL_ROLE;
}
const DEFAULT_RTT_CHECK_ID = "telegram-mentioned-message-reply";
function resolveRttOptions(env: NodeJS.ProcessEnv, selectedScenarioIds: readonly string[] = []) {
const explicitCheckIds = splitCsv(env.OPENCLAW_NPM_TELEGRAM_RTT_CHECKS);
if (
explicitCheckIds.length === 0 &&
selectedScenarioIds.length > 0 &&
!selectedScenarioIds.includes(DEFAULT_RTT_CHECK_ID)
) {
return {};
}
const rttCount = parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES") ?? 20;
return {
rttCount,
rttTimeoutMs: parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS"),
maxRttFailures:
parsePositiveIntegerEnv(env, "OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES") ?? rttCount,
rttCheckIds: explicitCheckIds,
};
}
async function shouldFailPackageTelegramRun(
result: { summaryPath: string },
env: NodeJS.ProcessEnv = process.env,
@ -74,16 +110,18 @@ async function main() {
const outputDir =
process.env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR?.trim() ||
path.join(repoRoot, ".artifacts", "qa-e2e", `npm-telegram-live-${Date.now().toString(36)}`);
const scenarioIds = splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS);
const result = await runTelegramQaLive({
env: process.env,
repoRoot,
outputDir,
sutOpenClawCommand,
preflightInstalledOnboarding: true,
providerMode: process.env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE,
primaryModel: process.env.OPENCLAW_NPM_TELEGRAM_MODEL,
alternateModel: process.env.OPENCLAW_NPM_TELEGRAM_ALT_MODEL,
fastMode: parseBoolean(process.env.OPENCLAW_NPM_TELEGRAM_FAST),
scenarioIds: splitCsv(process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS),
scenarioIds,
...resolveRttOptions(process.env, scenarioIds),
sutAccountId: process.env.OPENCLAW_NPM_TELEGRAM_SUT_ACCOUNT,
credentialSource: resolveCredentialSource(process.env),
credentialRole: resolveCredentialRole(process.env),
@ -91,7 +129,6 @@ async function main() {
process.stdout.write(`Package Telegram QA report: ${result.reportPath}\n`);
process.stdout.write(`Package Telegram QA summary: ${result.summaryPath}\n`);
process.stdout.write(`Package Telegram QA observed messages: ${result.observedMessagesPath}\n`);
if (await shouldFailPackageTelegramRun(result)) {
process.exitCode = 1;
}
@ -116,8 +153,10 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
}
export const testing = {
parsePositiveIntegerEnv,
resolveCredentialRole,
resolveCredentialSource,
resolveRttOptions,
shouldFailPackageTelegramRun,
};
export { testing as __testing };

118
scripts/e2e/npm-telegram-rtt-config.mjs
View file

@ -1,118 +0,0 @@
#!/usr/bin/env node
// Writes npm Telegram RTT config fixtures.
import fs from "node:fs";
const [configPath, mockPort, groupId, driverToken, sutToken, packageVersion] =
process.argv.slice(2);
if (!configPath || !mockPort || !groupId || !driverToken || !sutToken || !packageVersion) {
throw new Error(
"usage: npm-telegram-rtt-config.mjs <config> <mock-port> <group-id> <driver-token> <sut-token> <package-version>",
);
}
const driverId = driverToken.split(":", 1)[0];
const config = fs.existsSync(configPath) ? JSON.parse(fs.readFileSync(configPath, "utf8")) : {};
function supportsVisibleReplies(version) {
const match = /(\d{4})\.(\d+)\.(\d+)/u.exec(version);
if (!match) {
return false;
}
const [, year, month, day] = match.map(Number);
return year > 2026 || (year === 2026 && (month > 4 || (month === 4 && day >= 27)));
}
config.gateway = {
mode: "local",
port: 18789,
bind: "loopback",
auth: { mode: "none" },
};
config.models = config.models ?? {};
config.models.providers = config.models.providers ?? {};
config.models.providers.openai = {
api: "openai-responses",
agentRuntime: { id: "openclaw" },
apiKey: {
source: "env",
provider: "default",
id: "OPENAI_API_KEY",
},
baseUrl: `http://127.0.0.1:${mockPort}/v1`,
request: { allowPrivateNetwork: true },
models: [
{
id: "gpt-5.5",
name: "gpt-5.5",
api: "openai-responses",
contextWindow: 128000,
},
],
};
config.agents = config.agents ?? {};
config.agents.defaults = config.agents.defaults ?? {};
config.agents.defaults.model = { primary: "openai/gpt-5.5" };
config.agents.defaults.models = {
"openai/gpt-5.5": {
agentRuntime: { id: "openclaw" },
params: {
transport: "sse",
openaiWsWarmup: false,
},
},
};
config.agents.list = [
{
id: "main",
default: true,
name: "Main",
workspace: "~/workspace",
model: { primary: "openai/gpt-5.5" },
},
];
config.plugins = config.plugins ?? {};
config.plugins.enabled = true;
config.plugins.allow = ["telegram", "openai"];
config.plugins.entries = {
telegram: { enabled: true },
openai: { enabled: true },
};
config.channels = config.channels ?? {};
config.channels.telegram = {
enabled: true,
botToken: {
source: "env",
provider: "default",
id: "TELEGRAM_BOT_TOKEN",
},
streaming: { mode: "off" },
replyToMode: "first",
dmPolicy: "allowlist",
allowFrom: [driverId],
defaultTo: driverId,
groupPolicy: "allowlist",
groupAllowFrom: [driverId],
groups: {
[groupId]: {
requireMention: false,
allowFrom: [driverId],
},
},
};
if (supportsVisibleReplies(packageVersion)) {
config.messages = {
...config.messages,
groupChat: {
...config.messages?.groupChat,
visibleReplies: "automatic",
},
};
}
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`);

505
scripts/e2e/npm-telegram-rtt-credentials.mjs
View file

@ -1,505 +0,0 @@
#!/usr/bin/env node
// Issues and writes npm Telegram RTT credential fixtures.
import fs from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { readBoundedResponseText } from "./lib/bounded-response-text.mjs";
const DEFAULT_ENDPOINT_PREFIX = "/qa-credentials/v1";
const DEFAULT_ACQUIRE_TIMEOUT_MS = 90_000;
const DEFAULT_HTTP_BODY_MAX_BYTES = 1024 * 1024;
const DEFAULT_HTTP_TIMEOUT_MS = 15_000;
const DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;
const DEFAULT_LEASE_TTL_MS = 20 * 60 * 1_000;
const DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES = 64 * 1024 * 1024;
const DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS = 4096;
const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1";
const RETRY_BACKOFF_MS = [500, 1_000, 2_000, 4_000, 5_000];
const RETRYABLE_ACQUIRE_CODES = new Set(["POOL_EXHAUSTED", "NO_CREDENTIAL_AVAILABLE"]);
function parseArgs(argv) {
const command = argv[2];
const opts = new Map();
for (let index = 3; index < argv.length; index += 1) {
const arg = argv[index];
if (!arg.startsWith("--")) {
throw new Error(`Unexpected argument: ${arg}`);
}
const value = argv[++index];
if (!value) {
throw new Error(`${arg} requires a value.`);
}
opts.set(arg.slice(2), value);
}
if (!command || !["acquire", "heartbeat", "release"].includes(command)) {
throw new Error(
"Usage: npm-telegram-rtt-credentials.mjs acquire|heartbeat|release --lease-file PATH [--credential-env-file PATH]",
);
}
return { command, opts };
}
function requireOption(opts, key) {
const value = opts.get(key)?.trim();
if (!value) {
throw new Error(`Missing --${key}.`);
}
return value;
}
function requireString(record, key) {
const value = record[key];
if (typeof value !== "string" || value.trim().length === 0) {
throw new Error(`Credential payload is missing ${key}.`);
}
return value.trim();
}
class BrokerError extends Error {
constructor(message, options = {}) {
super(message);
this.name = "BrokerError";
this.code = options.code;
this.retryAfterMs = options.retryAfterMs;
}
}
function taggedError(message, code) {
return Object.assign(new Error(message), { code });
}
function parsePositiveInteger(value, fallback, label) {
const raw = value?.trim();
if (!raw) {
return fallback;
}
const parsed = Number(raw);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer; got: ${value}`);
}
return parsed;
}
const CHUNKED_PAYLOAD_MAX_BYTES = parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES,
DEFAULT_CHUNKED_PAYLOAD_MAX_BYTES,
"OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES",
);
const CHUNKED_PAYLOAD_MAX_CHUNKS = parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS,
DEFAULT_CHUNKED_PAYLOAD_MAX_CHUNKS,
"OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS",
);
function normalizeCredentialRole() {
const raw =
process.env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE?.trim() ||
process.env.OPENCLAW_QA_CREDENTIAL_ROLE?.trim() ||
(process.env.CI ? "ci" : "maintainer");
const normalized = raw.toLowerCase();
if (normalized === "ci" || normalized === "maintainer") {
return normalized;
}
throw new Error(`Credential role must be maintainer or ci; got: ${raw}`);
}
function normalizeEndpointPrefix() {
const raw = process.env.OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX?.trim() || DEFAULT_ENDPOINT_PREFIX;
if (!raw.startsWith("/") || raw.startsWith("//") || raw.includes("\\") || raw.includes("..")) {
throw new Error(
"OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX must be an absolute path like /qa-credentials/v1.",
);
}
return raw.replace(/\/+$/u, "") || "/";
}
function resolveConfig() {
const siteUrl = process.env.OPENCLAW_QA_CONVEX_SITE_URL?.trim();
if (!siteUrl) {
throw new Error("Missing OPENCLAW_QA_CONVEX_SITE_URL for --credential-source convex.");
}
const parsed = new URL(siteUrl);
const allowInsecure = /^(1|true|yes)$/iu.test(process.env.OPENCLAW_QA_ALLOW_INSECURE_HTTP ?? "");
const isLoopback = parsed.hostname === "localhost" || parsed.hostname === "127.0.0.1";
if (
parsed.protocol !== "https:" &&
!(parsed.protocol === "http:" && allowInsecure && isLoopback)
) {
throw new Error("OPENCLAW_QA_CONVEX_SITE_URL must use https://.");
}
const role = normalizeCredentialRole();
const authToken =
role === "ci"
? process.env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim()
: process.env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim();
if (!authToken) {
throw new Error(
role === "ci"
? "Missing OPENCLAW_QA_CONVEX_SECRET_CI for CI credential access."
: "Missing OPENCLAW_QA_CONVEX_SECRET_MAINTAINER for maintainer credential access.",
);
}
const endpointPrefix = normalizeEndpointPrefix();
const ownerId =
process.env.OPENCLAW_QA_CREDENTIAL_OWNER_ID?.trim() ||
`npm-telegram-rtt-${process.pid}-${Date.now()}`;
const joinEndpoint = (endpoint) =>
`${siteUrl.replace(/\/+$/u, "")}${endpointPrefix}/${endpoint.replace(/^\/+/u, "")}`;
return {
acquireUrl: joinEndpoint("acquire"),
acquireTimeoutMs: parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS,
DEFAULT_ACQUIRE_TIMEOUT_MS,
"OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS",
),
heartbeatIntervalMs: parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS,
DEFAULT_HEARTBEAT_INTERVAL_MS,
"OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS",
),
heartbeatUrl: joinEndpoint("heartbeat"),
httpBodyMaxBytes: parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES,
DEFAULT_HTTP_BODY_MAX_BYTES,
"OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES",
),
httpTimeoutMs: parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS,
DEFAULT_HTTP_TIMEOUT_MS,
"OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS",
),
leaseTtlMs: parsePositiveInteger(
process.env.OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS,
DEFAULT_LEASE_TTL_MS,
"OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS",
),
ownerId,
payloadChunkUrl: joinEndpoint("payload-chunk"),
releaseUrl: joinEndpoint("release"),
role,
siteUrl,
authToken,
};
}
function parseBrokerPayload(rawPayload, response) {
if (!rawPayload.trim()) {
return response.ok ? { status: "ok" } : {};
}
try {
return JSON.parse(rawPayload);
} catch (error) {
throw new Error("Convex credential broker returned invalid JSON.", { cause: error });
}
}
async function postBroker(params) {
const controller = new AbortController();
const timeoutMs = Math.max(1, params.timeoutMs);
const timeoutError = taggedError(`${params.label} timed out after ${timeoutMs}ms`, "ETIMEDOUT");
let timeout;
const timeoutPromise = new Promise((_, reject) => {
timeout = setTimeout(() => {
controller.abort(timeoutError);
reject(timeoutError);
}, timeoutMs);
timeout.unref?.();
});
try {
const response = await Promise.race([
fetch(params.url, {
method: "POST",
headers: {
authorization: `Bearer ${params.authToken}`,
"content-type": "application/json",
},
body: JSON.stringify(params.body),
signal: controller.signal,
}),
timeoutPromise,
]);
const rawPayload = await readBoundedResponseText(
response,
params.label,
params.bodyMaxBytes,
timeoutPromise,
);
const payload = parseBrokerPayload(rawPayload, response);
if (!response.ok || payload?.status === "error") {
const message =
typeof payload?.message === "string" && payload.message.trim()
? payload.message.trim()
: `HTTP ${response.status}`;
throw new BrokerError(message, {
code: typeof payload?.code === "string" ? payload.code : undefined,
retryAfterMs: Number.isInteger(payload?.retryAfterMs) ? payload.retryAfterMs : undefined,
});
}
if (payload?.status !== "ok") {
throw new Error("Convex credential broker returned an invalid response.");
}
return payload;
} finally {
clearTimeout(timeout);
}
}
function parseChunkedPayloadMarker(payload) {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
return undefined;
}
if (payload[CHUNKED_PAYLOAD_MARKER] !== true) {
return undefined;
}
if (!Number.isInteger(payload.chunkCount) || payload.chunkCount < 1) {
throw new Error("Chunked credential payload has invalid chunkCount.");
}
if (payload.chunkCount > CHUNKED_PAYLOAD_MAX_CHUNKS) {
throw new Error(`Chunked credential payload exceeds ${CHUNKED_PAYLOAD_MAX_CHUNKS} chunks.`);
}
if (!Number.isInteger(payload.byteLength) || payload.byteLength < 0) {
throw new Error("Chunked credential payload has invalid byteLength.");
}
if (payload.byteLength > CHUNKED_PAYLOAD_MAX_BYTES) {
throw new Error(`Chunked credential payload exceeds ${CHUNKED_PAYLOAD_MAX_BYTES} bytes.`);
}
return { byteLength: payload.byteLength, chunkCount: payload.chunkCount };
}
function parseTelegramCredentialPayload(payload) {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
throw new Error("Telegram credential payload must be an object.");
}
const groupId = requireString(payload, "groupId");
if (!/^-?\d+$/u.test(groupId)) {
throw new Error("Telegram credential payload groupId must be a numeric Telegram chat id.");
}
return {
groupId,
driverToken: requireString(payload, "driverToken"),
sutToken: requireString(payload, "sutToken"),
};
}
async function resolveCredentialPayload(config, acquired) {
const marker = parseChunkedPayloadMarker(acquired.payload);
if (!marker) {
return parseTelegramCredentialPayload(acquired.payload);
}
const chunks = [];
let serializedLength = 0;
for (let index = 0; index < marker.chunkCount; index += 1) {
const chunk = await postBroker({
authToken: config.authToken,
bodyMaxBytes: config.httpBodyMaxBytes,
label: "credential broker payload-chunk",
timeoutMs: config.httpTimeoutMs,
url: config.payloadChunkUrl,
body: {
kind: "telegram",
ownerId: config.ownerId,
actorRole: config.role,
credentialId: requireString(acquired, "credentialId"),
leaseToken: requireString(acquired, "leaseToken"),
index,
},
});
const data = requireString(chunk, "data");
serializedLength += data.length;
if (serializedLength > marker.byteLength) {
throw new Error("Chunked credential payload exceeded declared byteLength.");
}
chunks.push(data);
}
const serialized = chunks.join("");
if (serializedLength !== marker.byteLength) {
throw new Error("Chunked credential payload length mismatch.");
}
return parseTelegramCredentialPayload(JSON.parse(serialized));
}
function shellQuote(value) {
return `'${value.replaceAll("'", "'\\''")}'`;
}
async function writeCredentialEnv(pathname, payload) {
await fs.writeFile(
pathname,
[
`export OPENCLAW_QA_TELEGRAM_GROUP_ID=${shellQuote(payload.groupId)}`,
`export OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN=${shellQuote(payload.driverToken)}`,
`export OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN=${shellQuote(payload.sutToken)}`,
"",
].join("\n"),
{ mode: 0o600 },
);
}
async function readLease(pathname) {
return JSON.parse(await fs.readFile(pathname, "utf8"));
}
function leaseTtlMsFromLease(config, lease) {
const value = lease.leaseTtlMs;
if (value === undefined || value === null) {
return config.leaseTtlMs;
}
return parsePositiveInteger(String(value), config.leaseTtlMs, "leaseTtlMs");
}
async function acquire(opts) {
const config = resolveConfig();
const leaseFile = requireOption(opts, "lease-file");
const envFile = requireOption(opts, "credential-env-file");
const acquired = await acquireWithRetry(config);
const lease = {
kind: "telegram",
ownerId: config.ownerId,
actorRole: config.role,
credentialId: requireString(acquired, "credentialId"),
leaseToken: requireString(acquired, "leaseToken"),
heartbeatIntervalMs: acquired.heartbeatIntervalMs ?? config.heartbeatIntervalMs,
leaseTtlMs: acquired.leaseTtlMs ?? config.leaseTtlMs,
};
try {
await writeCredentialEnv(envFile, await resolveCredentialPayload(config, acquired));
await fs.writeFile(leaseFile, `${JSON.stringify(lease, null, 2)}\n`, { mode: 0o600 });
} catch (error) {
await releaseLease(config, lease).catch(() => {});
throw error;
}
process.stdout.write(
`${JSON.stringify({ status: "ok", credentialId: lease.credentialId, ownerId: lease.ownerId }, null, 2)}\n`,
);
}
async function acquireWithRetry(config) {
const startedAt = Date.now();
let attempt = 0;
while (true) {
attempt += 1;
const attemptElapsedMs = Date.now() - startedAt;
const attemptRemainingMs = config.acquireTimeoutMs - attemptElapsedMs;
if (attemptRemainingMs <= 0) {
throw taggedError(
`credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`,
"ETIMEDOUT",
);
}
try {
return await postBroker({
authToken: config.authToken,
bodyMaxBytes: config.httpBodyMaxBytes,
label: "credential broker acquire",
timeoutMs: Math.min(config.httpTimeoutMs, attemptRemainingMs),
url: config.acquireUrl,
body: {
kind: "telegram",
ownerId: config.ownerId,
actorRole: config.role,
leaseTtlMs: config.leaseTtlMs,
heartbeatIntervalMs: config.heartbeatIntervalMs,
},
});
} catch (error) {
const code = error instanceof BrokerError ? error.code : undefined;
const retryable = code ? RETRYABLE_ACQUIRE_CODES.has(code) : false;
const elapsedMs = Date.now() - startedAt;
if (!retryable) {
throw error;
}
if (elapsedMs >= config.acquireTimeoutMs) {
throw taggedError(
`credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`,
"ETIMEDOUT",
);
}
const fallbackDelay = RETRY_BACKOFF_MS[Math.min(attempt - 1, RETRY_BACKOFF_MS.length - 1)];
const retryAfterMs = error instanceof BrokerError ? error.retryAfterMs : undefined;
const delayMs = retryAfterMs ?? fallbackDelay;
const remainingMs = config.acquireTimeoutMs - elapsedMs;
if (delayMs >= remainingMs) {
throw taggedError(
`credential broker acquire timed out after ${config.acquireTimeoutMs}ms before retry`,
"ETIMEDOUT",
);
}
await new Promise((resolve) => {
setTimeout(resolve, delayMs);
});
}
}
}
async function releaseLease(config, lease) {
await postBroker({
authToken: config.authToken,
bodyMaxBytes: config.httpBodyMaxBytes,
label: "credential broker release",
timeoutMs: config.httpTimeoutMs,
url: config.releaseUrl,
body: {
kind: requireString(lease, "kind"),
ownerId: requireString(lease, "ownerId"),
actorRole: requireString(lease, "actorRole"),
credentialId: requireString(lease, "credentialId"),
leaseToken: requireString(lease, "leaseToken"),
},
});
}
async function release(opts) {
const config = resolveConfig();
const leaseFile = requireOption(opts, "lease-file");
const lease = await readLease(leaseFile);
await releaseLease(config, lease);
await fs.rm(leaseFile, { force: true });
}
async function heartbeat(opts) {
const config = resolveConfig();
const leaseFile = requireOption(opts, "lease-file");
while (true) {
const lease = await readLease(leaseFile);
await postBroker({
authToken: config.authToken,
bodyMaxBytes: config.httpBodyMaxBytes,
label: "credential broker heartbeat",
timeoutMs: config.httpTimeoutMs,
url: config.heartbeatUrl,
body: {
kind: requireString(lease, "kind"),
ownerId: requireString(lease, "ownerId"),
actorRole: requireString(lease, "actorRole"),
credentialId: requireString(lease, "credentialId"),
leaseTtlMs: leaseTtlMsFromLease(config, lease),
leaseToken: requireString(lease, "leaseToken"),
},
});
const intervalMs = parsePositiveInteger(
String(lease.heartbeatIntervalMs ?? config.heartbeatIntervalMs),
config.heartbeatIntervalMs,
"heartbeatIntervalMs",
);
await new Promise((resolve) => {
setTimeout(resolve, intervalMs);
});
}
}
async function main(argv = process.argv) {
const { command, opts } = parseArgs(argv);
if (command === "acquire") {
await acquire(opts);
} else if (command === "heartbeat") {
await heartbeat(opts);
} else {
await release(opts);
}
}
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
await main();
}
export { parseChunkedPayloadMarker };

454
scripts/e2e/npm-telegram-rtt-docker.sh
View file

@ -1,454 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh"
IMAGE_NAME="$(docker_e2e_resolve_image "openclaw-npm-telegram-rtt-e2e" OPENCLAW_NPM_TELEGRAM_RTT_E2E_IMAGE)"
DOCKER_TARGET="${OPENCLAW_NPM_TELEGRAM_DOCKER_TARGET:-build}"
PACKAGE_SPEC="${OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC:-openclaw@beta}"
PACKAGE_TGZ="${OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ:-${OPENCLAW_CURRENT_PACKAGE_TGZ:-}}"
PACKAGE_LABEL="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-}"
RUN_ID="${OPENCLAW_NPM_TELEGRAM_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"
OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt/$RUN_ID}"
resolve_credential_source() {
if [ -n "${OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE:-}" ]; then
printf "%s" "$OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE"
return 0
fi
if [ -n "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" ]; then
printf "%s" "$OPENCLAW_QA_CREDENTIAL_SOURCE"
return 0
fi
if [ -n "${CI:-}" ] && [ -n "${OPENCLAW_QA_CONVEX_SITE_URL:-}" ]; then
if [ -n "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ] || [ -n "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then
printf "convex"
fi
fi
}
resolve_credential_role() {
if [ -n "${OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE:-}" ]; then
printf "%s" "$OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE"
return 0
fi
if [ -n "${OPENCLAW_QA_CREDENTIAL_ROLE:-}" ]; then
printf "%s" "$OPENCLAW_QA_CREDENTIAL_ROLE"
fi
}
validate_openclaw_package_spec() {
local spec="$1"
if [[ "$spec" =~ ^openclaw@(main|alpha|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$ ]]; then
return 0
fi
echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC must be openclaw@main, openclaw@alpha, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: $spec" >&2
exit 1
}
resolve_package_tgz() {
local candidate="$1"
if [ -z "$candidate" ]; then
return 0
fi
if [ ! -f "$candidate" ]; then
echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ must point to an existing .tgz file; got: $candidate" >&2
exit 1
fi
case "$candidate" in
*.tgz) ;;
*)
echo "OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ must point to a .tgz file; got: $candidate" >&2
exit 1
;;
esac
local dir
local base
dir="$(cd "$(dirname "$candidate")" && pwd)"
base="$(basename "$candidate")"
printf "%s/%s" "$dir" "$base"
}
package_mount_args=()
package_install_source="$PACKAGE_SPEC"
package_source_kind="npm-package"
resolved_package_tgz="$(resolve_package_tgz "$PACKAGE_TGZ")"
if [ -n "$resolved_package_tgz" ]; then
package_install_source="/package-under-test/$(basename "$resolved_package_tgz")"
package_source_kind="packed-tarball"
package_mount_args=(-v "$resolved_package_tgz:$package_install_source:ro")
else
validate_openclaw_package_spec "$PACKAGE_SPEC"
fi
if [ -z "$PACKAGE_LABEL" ]; then
if [ -n "$resolved_package_tgz" ]; then
PACKAGE_LABEL="$(basename "$resolved_package_tgz")"
else
PACKAGE_LABEL="$PACKAGE_SPEC"
fi
fi
credential_source="$(resolve_credential_source)"
credential_role="$(resolve_credential_role)"
if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then
if [ -n "${CI:-}" ]; then
credential_role="ci"
else
credential_role="maintainer"
fi
fi
validate_credential_source() {
case "$credential_source" in
"" | env | convex) ;;
*)
echo "OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE must be env or convex; got: $credential_source" >&2
exit 1
;;
esac
}
validate_credential_role() {
case "$credential_role" in
"" | maintainer | ci) ;;
*)
echo "OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE must be maintainer or ci; got: $credential_role" >&2
exit 1
;;
esac
}
validate_credential_source
validate_credential_role
validate_credential_preflight() {
if [ "$credential_source" = "convex" ]; then
if [ -z "${OPENCLAW_QA_CONVEX_SITE_URL:-}" ]; then
echo "Missing required env for Convex credential mode: OPENCLAW_QA_CONVEX_SITE_URL" >&2
exit 1
fi
if [ "$credential_role" = "ci" ]; then
if [ -z "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ]; then
echo "Missing required env for Convex ci credential mode: OPENCLAW_QA_CONVEX_SECRET_CI" >&2
exit 1
fi
return 0
fi
if [ "$credential_role" = "maintainer" ]; then
if [ -z "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then
echo "Missing required env for Convex maintainer credential mode: OPENCLAW_QA_CONVEX_SECRET_MAINTAINER" >&2
exit 1
fi
return 0
fi
if [ -z "${OPENCLAW_QA_CONVEX_SECRET_CI:-}" ] && [ -z "${OPENCLAW_QA_CONVEX_SECRET_MAINTAINER:-}" ]; then
echo "Missing required env for Convex credential mode: OPENCLAW_QA_CONVEX_SECRET_CI or OPENCLAW_QA_CONVEX_SECRET_MAINTAINER" >&2
exit 1
fi
return 0
fi
for key in \
OPENCLAW_QA_TELEGRAM_GROUP_ID \
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN \
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN; do
if [ -z "${!key:-}" ]; then
echo "Missing required env: $key" >&2
exit 1
fi
done
}
validate_credential_preflight
if [ -n "$credential_source" ]; then
export OPENCLAW_QA_CREDENTIAL_SOURCE="$credential_source"
fi
if [ -n "$credential_role" ]; then
export OPENCLAW_QA_CREDENTIAL_ROLE="$credential_role"
fi
if [ -z "$credential_source" ] || [ "$credential_source" = "env" ]; then
for key in \
OPENCLAW_QA_TELEGRAM_GROUP_ID \
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN \
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN; do
if [ -z "${!key:-}" ]; then
echo "Missing required env: $key" >&2
exit 1
fi
done
fi
for value in "$credential_source" "$credential_role"; do
if [[ "$value" == *[$'\n\r']* ]]; then
echo "Credential source and role must be single-line values." >&2
exit 1
fi
done
docker_e2e_build_or_reuse "$IMAGE_NAME" npm-telegram-rtt "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" "$DOCKER_TARGET"
mkdir -p "$ROOT_DIR/.artifacts/qa-e2e"
run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-npm-telegram-rtt.XXXXXX")"
npm_prefix_host="$(mktemp -d "$ROOT_DIR/.artifacts/qa-e2e/npm-telegram-rtt-prefix.XXXXXX")"
trap 'rm -f "$run_log"; rm -rf "$npm_prefix_host"' EXIT
docker_env=(
-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0
-e OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE="$package_install_source"
-e OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL="$PACKAGE_LABEL"
-e OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR="$OUTPUT_DIR"
-e OPENCLAW_QA_PACKAGE_SOURCE="$package_install_source"
-e OPENCLAW_QA_PACKAGE_SOURCE_KIND="$package_source_kind"
-e OPENCLAW_QA_TELEGRAM_GROUP_ID
-e OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN
-e OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN
-e OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS="${OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS:-180000}"
-e OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS="${OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS:-180000}"
-e OPENCLAW_NPM_TELEGRAM_SCENARIOS="${OPENCLAW_NPM_TELEGRAM_SCENARIOS:-telegram-mentioned-message-reply}"
-e OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE="${OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE:-mock-openai}"
-e OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES="${OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES:-20}"
-e OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS="${OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS:-30000}"
-e OPENCLAW_NPM_TELEGRAM_MAX_FAILURES="${OPENCLAW_NPM_TELEGRAM_MAX_FAILURES:-${OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES:-20}}"
-e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"
)
forward_env_if_set() {
local key="$1"
if [ -n "${!key:-}" ]; then
docker_env+=(-e "$key")
fi
}
if [ -n "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" ]; then
docker_env+=(-e OPENCLAW_QA_CREDENTIAL_SOURCE="$OPENCLAW_QA_CREDENTIAL_SOURCE")
fi
if [ -n "${OPENCLAW_QA_CREDENTIAL_ROLE:-}" ]; then
docker_env+=(-e OPENCLAW_QA_CREDENTIAL_ROLE="$OPENCLAW_QA_CREDENTIAL_ROLE")
fi
install_env=("${docker_env[@]}")
for key in \
OPENCLAW_QA_CONVEX_SITE_URL \
OPENCLAW_QA_CONVEX_SECRET_CI \
OPENCLAW_QA_CONVEX_SECRET_MAINTAINER \
OPENCLAW_QA_CREDENTIAL_LEASE_TTL_MS \
OPENCLAW_QA_CREDENTIAL_HEARTBEAT_INTERVAL_MS \
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS \
OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS \
OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES \
OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES \
OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS \
OPENCLAW_QA_PACKAGE_SOURCE_SHA \
OPENCLAW_QA_CONVEX_ENDPOINT_PREFIX \
OPENCLAW_QA_CREDENTIAL_OWNER_ID \
OPENCLAW_QA_ALLOW_INSECURE_HTTP; do
forward_env_if_set "$key"
done
run_logged() {
if ! "$@" >"$run_log" 2>&1; then
docker_e2e_print_log "$run_log"
exit 1
fi
docker_e2e_print_log "$run_log"
>"$run_log"
}
echo "Installing ${PACKAGE_LABEL} from ${package_install_source}..."
run_logged docker_e2e_docker_run_cmd run --rm \
"${install_env[@]}" \
${package_mount_args[@]+"${package_mount_args[@]}"} \
-v "$npm_prefix_host:/npm-global" \
-i "$IMAGE_NAME" bash -s <<'EOF'
set -euo pipefail
export NPM_CONFIG_PREFIX="/npm-global"
export PATH="$NPM_CONFIG_PREFIX/bin:$PATH"
install_source="${OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE:?missing OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE}"
package_label="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-$install_source}"
npm_install_timeout="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"
run_npm_install() {
if [ -z "$npm_install_timeout" ] || [ "$npm_install_timeout" = "0" ]; then
npm install -g "$install_source" --no-fund --no-audit
return
fi
local timeout_bin=""
if command -v timeout >/dev/null 2>&1; then
timeout_bin="timeout"
elif command -v gtimeout >/dev/null 2>&1; then
timeout_bin="gtimeout"
fi
if [ -z "$timeout_bin" ]; then
echo "timeout or gtimeout is required for OPENCLAW_E2E_NPM_INSTALL_TIMEOUT=$npm_install_timeout" >&2
return 127
fi
if "$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1; then
"$timeout_bin" --kill-after=30s "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit
else
"$timeout_bin" "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit
fi
}
run_npm_install
command -v openclaw
openclaw --version
node -p "require('/npm-global/lib/node_modules/openclaw/package.json').version"
EOF
echo "Running package Telegram RTT Docker E2E ($PACKAGE_LABEL)..."
run_logged docker_e2e_docker_run_cmd run --rm \
"${docker_env[@]}" \
-v "$ROOT_DIR/scripts:/app/scripts:ro" \
-v "$ROOT_DIR/.artifacts:/app/.artifacts" \
-v "$npm_prefix_host:/npm-global" \
-i "$IMAGE_NAME" bash -s <<'EOF'
set -euo pipefail
source scripts/lib/openclaw-e2e-instance.sh
export HOME="$(mktemp -d "/tmp/openclaw-npm-telegram-rtt.XXXXXX")"
export NPM_CONFIG_PREFIX="/npm-global"
export PATH="$NPM_CONFIG_PREFIX/bin:$PATH"
export OPENAI_API_KEY="sk-openclaw-rtt"
export GATEWAY_AUTH_TOKEN_REF="openclaw-rtt"
export OPENCLAW_DISABLE_BONJOUR="1"
install_source="${OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE:?missing OPENCLAW_NPM_TELEGRAM_INSTALL_SOURCE}"
package_label="${OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL:-$install_source}"
mock_port="${OPENCLAW_NPM_TELEGRAM_MOCK_PORT:-44080}"
config_path="$HOME/.openclaw/openclaw.json"
gateway_log="/tmp/openclaw-npm-telegram-rtt-gateway.log"
mock_log="/tmp/openclaw-npm-telegram-rtt-mock.log"
export MOCK_PORT="$mock_port"
credential_env_file=""
credential_lease_file=""
credential_heartbeat_pid=""
rtt_shell_pid="$$"
dump_logs() {
local status="$1"
if [ "$status" -eq 0 ]; then
return
fi
echo "package Telegram RTT failed with exit code $status" >&2
for file in \
"$mock_log" \
"$gateway_log"; do
if [ -f "$file" ]; then
echo "--- $file ---" >&2
openclaw_e2e_print_log "$file" >&2
fi
done
}
cleanup() {
local status="$?"
kill ${gateway_pid:-} ${mock_pid:-} ${credential_heartbeat_pid:-} 2>/dev/null || true
if [ -n "$credential_lease_file" ] && [ -f "$credential_lease_file" ]; then
node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs release --lease-file "$credential_lease_file" >/dev/null 2>&1 || true
fi
rm -f "$credential_env_file" "$credential_lease_file"
dump_logs "$status"
exit "$status"
}
start_credential_heartbeat() {
(
set +e
node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs heartbeat --lease-file "$credential_lease_file" &
local heartbeat_child_pid="$!"
trap 'kill "$heartbeat_child_pid" 2>/dev/null || true; wait "$heartbeat_child_pid" 2>/dev/null || true; exit 0' TERM INT
wait "$heartbeat_child_pid"
local heartbeat_status="$?"
echo "Convex credential heartbeat exited with status $heartbeat_status" >&2
kill -TERM "$rtt_shell_pid" 2>/dev/null || true
exit "$heartbeat_status"
) &
credential_heartbeat_pid="$!"
}
trap cleanup EXIT
trap 'exit 1' TERM INT
if [ "${OPENCLAW_QA_CREDENTIAL_SOURCE:-}" = "convex" ]; then
credential_env_file="$(mktemp "/tmp/openclaw-npm-telegram-rtt-credential-env.XXXXXX")"
credential_lease_file="$(mktemp "/tmp/openclaw-npm-telegram-rtt-credential-lease.XXXXXX")"
rm -f "$credential_env_file" "$credential_lease_file"
node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs acquire \
--credential-env-file "$credential_env_file" \
--lease-file "$credential_lease_file"
# shellcheck source=/dev/null
source "$credential_env_file"
start_credential_heartbeat
fi
export TELEGRAM_BOT_TOKEN="${OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN:?missing OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN}"
command -v openclaw
openclaw --version
installed_version="$(node -p "require('/npm-global/lib/node_modules/openclaw/package.json').version")"
node /app/scripts/e2e/mock-openai-server.mjs >"$mock_log" 2>&1 &
mock_pid="$!"
mock_ready=0
for _ in $(seq 1 60); do
if node --input-type=module -e '
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), 1000);
try {
const response = await fetch(process.argv[1], { signal: controller.signal });
process.exit(response.ok ? 0 : 1);
} catch {
process.exit(1);
} finally {
clearTimeout(timer);
}
' "http://127.0.0.1:${mock_port}/health"; then
mock_ready=1
break
fi
sleep 1
done
if [ "$mock_ready" != "1" ]; then
echo "Mock OpenAI server did not become ready" >&2
openclaw_e2e_print_log "$mock_log" >&2
exit 1
fi
mkdir -p "$(dirname "$config_path")" "$HOME/.openclaw/workspace" "$HOME/.openclaw/agents/main/sessions" "$HOME/workspace"
node /app/scripts/e2e/npm-telegram-rtt-config.mjs \
"$config_path" \
"$mock_port" \
"$OPENCLAW_QA_TELEGRAM_GROUP_ID" \
"$OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN" \
"$OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN" \
"$installed_version"
openclaw gateway run --verbose >"$gateway_log" 2>&1 &
gateway_pid="$!"
for _ in $(seq 1 120); do
if ! kill -0 "$gateway_pid" 2>/dev/null; then
echo "gateway exited before readiness" >&2
exit 1
fi
if bash -c ":</dev/tcp/127.0.0.1/18789" >/dev/null 2>&1; then
break
fi
sleep 1
done
if ! bash -c ":</dev/tcp/127.0.0.1/18789" >/dev/null 2>&1; then
echo "gateway did not open port 18789" >&2
exit 1
fi
node /app/scripts/e2e/npm-telegram-rtt-driver.mjs
EOF
echo "package Telegram RTT Docker E2E passed ($PACKAGE_LABEL)"

592
scripts/e2e/npm-telegram-rtt-driver.mjs
View file

@ -1,592 +0,0 @@
#!/usr/bin/env node
// Drives npm Telegram RTT test messages through the fixture server.
import fs from "node:fs/promises";
import path from "node:path";
import { readBoundedResponseText } from "./lib/bounded-response-text.mjs";
const groupId = process.env.OPENCLAW_QA_TELEGRAM_GROUP_ID;
const driverToken = process.env.OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN;
const sutToken = process.env.OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN;
const outputDir = process.env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR ?? ".artifacts/rtt/raw";
const providerMode = process.env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE?.trim() || "mock-openai";
const primaryModel = process.env.OPENCLAW_NPM_TELEGRAM_MODEL?.trim() || null;
const telegramApiBaseUrl = (
process.env.OPENCLAW_QA_TELEGRAM_API_BASE_URL ?? "https://api.telegram.org"
).replace(/\/+$/u, "");
const timeoutMs = readPositiveIntEnv("OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", 180000);
const canaryTimeoutMs = readPositiveIntEnv("OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", timeoutMs);
const warmSampleCount = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES", 20);
const sampleTimeoutMs = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS", 30000);
const botApiTimeoutMs = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS", 30000);
const botApiBodyMaxBytes = readPositiveIntEnv(
"OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES",
1024 * 1024,
);
const maxWarmFailures = readPositiveIntEnv("OPENCLAW_NPM_TELEGRAM_MAX_FAILURES", warmSampleCount);
const successMarker = process.env.OPENCLAW_NPM_TELEGRAM_SUCCESS_MARKER ?? "OPENCLAW_E2E_OK";
const supportedScenarioIds = new Set(["telegram-mentioned-message-reply"]);
const requestedScenarioIds = (
process.env.OPENCLAW_NPM_TELEGRAM_SCENARIOS ?? "telegram-mentioned-message-reply"
)
.split(",")
.map((value) => value.trim())
.filter(Boolean);
if (requestedScenarioIds.length === 0) {
throw new Error("OPENCLAW_NPM_TELEGRAM_SCENARIOS must include at least one RTT scenario");
}
const unknownScenarioIds = requestedScenarioIds.filter(
(scenarioId) => !supportedScenarioIds.has(scenarioId),
);
if (unknownScenarioIds.length > 0) {
throw new Error(`unknown OPENCLAW_NPM_TELEGRAM_SCENARIOS: ${unknownScenarioIds.join(", ")}`);
}
const scenarioIds = new Set(requestedScenarioIds);
if (!groupId || !driverToken || !sutToken) {
throw new Error(
"missing Telegram env: OPENCLAW_QA_TELEGRAM_GROUP_ID, OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN, OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
);
}
function readPositiveIntEnv(name, fallback) {
const text = String(process.env[name] ?? fallback).trim();
if (!/^\d+$/u.test(text)) {
throw new Error(`invalid ${name}: ${text}`);
}
const value = Number(text);
if (!Number.isSafeInteger(value) || value <= 0) {
throw new Error(`invalid ${name}: ${text}`);
}
return value;
}
function taggedError(message, code) {
return Object.assign(new Error(message), { code });
}
function parseJsonPayload(rawPayload, label) {
try {
return JSON.parse(rawPayload);
} catch (error) {
throw new Error(`${label} returned invalid JSON`, { cause: error });
}
}
async function fetchTelegramJson(url, init, label) {
const controller = new AbortController();
const timeoutError = taggedError(`${label} timed out after ${botApiTimeoutMs}ms`, "ETIMEDOUT");
let timeout;
const timeoutPromise = new Promise((_, reject) => {
timeout = setTimeout(() => {
controller.abort(timeoutError);
reject(timeoutError);
}, botApiTimeoutMs);
timeout.unref?.();
});
try {
const response = await Promise.race([
fetch(url, {
...init,
signal: controller.signal,
}),
timeoutPromise,
]);
const rawPayload = await readBoundedResponseText(
response,
label,
botApiBodyMaxBytes,
timeoutPromise,
);
const payload = parseJsonPayload(rawPayload, label);
return { payload, response };
} finally {
if (timeout) {
clearTimeout(timeout);
}
}
}
class TelegramBot {
constructor(token) {
this.baseUrl = `${telegramApiBaseUrl}/bot${token}`;
}
async call(method, body) {
const { payload, response } = await fetchTelegramJson(
`${this.baseUrl}/${method}`,
{
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(body),
},
`Telegram Bot API ${method}`,
);
if (!response.ok || payload.ok !== true) {
throw new Error(`${method} failed: ${JSON.stringify(payload)}`);
}
return payload.result;
}
getMe() {
return this.call("getMe", {});
}
sendMessage(params) {
return this.call("sendMessage", params);
}
getUpdates(params) {
return this.call("getUpdates", params);
}
}
const driver = new TelegramBot(driverToken);
const sut = new TelegramBot(sutToken);
const observedMessages = [];
let driverUpdateOffset = 0;
function sleep(ms) {
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}
function messageText(message) {
return message.text ?? message.caption ?? "";
}
async function flushUpdates(bot) {
let updates = await bot.getUpdates({
timeout: 0,
allowed_updates: ["message", "edited_message"],
});
let nextOffset;
while (updates.length > 0) {
const lastUpdateId = updates.at(-1).update_id;
nextOffset = lastUpdateId + 1;
updates = await bot.getUpdates({
offset: nextOffset,
timeout: 0,
allowed_updates: ["message", "edited_message"],
});
}
return nextOffset;
}
async function waitForSutReply(params) {
const deadline = Date.now() + params.timeoutMs;
while (Date.now() < deadline) {
const updates = await driver.getUpdates({
offset: driverUpdateOffset,
timeout: 5,
allowed_updates: ["message", "edited_message"],
});
for (const update of updates) {
driverUpdateOffset = Math.max(driverUpdateOffset, update.update_id + 1);
const message = update.message ?? update.edited_message;
if (!message || String(message.chat?.id) !== String(groupId)) {
continue;
}
observedMessages.push({
updateType: update.edited_message ? "edited_message" : "message",
updateId: update.update_id,
messageId: message.message_id,
fromId: message.from?.id,
fromUsername: message.from?.username,
replyToMessageId: message.reply_to_message?.message_id,
text: messageText(message),
scenarioId: params.scenarioId,
scenarioTitle: params.scenarioTitle,
sampleIndex: params.sampleIndex,
});
if (message.from?.id !== params.sutId) {
continue;
}
if (message.date < params.startedUnixSeconds) {
continue;
}
const text = messageText(message);
if (params.matchText && !text.includes(params.matchText)) {
continue;
}
const replyMatches = message.reply_to_message?.message_id === params.requestMessageId;
const textMatches = params.matchText ? text.includes(params.matchText) : false;
if (replyMatches || textMatches) {
return message;
}
}
}
throw new Error(`timed out after ${params.timeoutMs}ms waiting for Telegram message`);
}
async function runScenario(params) {
const startedAt = new Date();
const startedUnixSeconds = Math.floor(startedAt.getTime() / 1000);
const sendParams = {
chat_id: groupId,
text: params.input,
disable_notification: true,
};
if (params.replyToMessageId) {
sendParams.reply_parameters = { message_id: params.replyToMessageId };
}
const request = await driver.sendMessage(sendParams);
try {
const reply = await waitForSutReply({
allowAnySutReply: params.allowAnySutReply,
matchText: params.matchText,
requestMessageId: request.message_id,
scenarioId: params.id,
scenarioTitle: params.title,
sampleIndex: params.sampleIndex,
startedUnixSeconds,
sutId: params.sutId,
timeoutMs: params.timeoutMs,
});
const rttMs = Date.now() - startedAt.getTime();
return {
id: params.id,
title: params.title,
status: "pass",
details: `observed SUT message ${reply.message_id}`,
messageId: reply.message_id,
rttMs,
};
} catch (error) {
return {
id: params.id,
title: params.title,
status: "fail",
details: error instanceof Error ? error.message : String(error),
};
}
}
function percentile(sortedValues, percentileValue) {
if (sortedValues.length === 0) {
return undefined;
}
const index = Math.ceil((percentileValue / 100) * sortedValues.length) - 1;
return sortedValues[Math.min(Math.max(index, 0), sortedValues.length - 1)];
}
function summarizeSamples(samples) {
const passed = samples.filter((sample) => sample.status === "pass" && sample.rttMs !== undefined);
const sorted = passed.map((sample) => sample.rttMs).toSorted((a, b) => a - b);
const sum = sorted.reduce((total, value) => total + value, 0);
return {
total: samples.length,
passed: passed.length,
failed: samples.length - passed.length,
avgMs: sorted.length > 0 ? Math.round(sum / sorted.length) : undefined,
p50Ms: percentile(sorted, 50),
p95Ms: percentile(sorted, 95),
maxMs: sorted.at(-1),
};
}
function splitModelRef(modelRef) {
if (!modelRef) {
return { provider: "openai", model: null, ref: null };
}
const slashIndex = modelRef.indexOf("/");
if (slashIndex <= 0 || slashIndex === modelRef.length - 1) {
return { provider: "openai", model: modelRef, ref: modelRef };
}
return {
provider: modelRef.slice(0, slashIndex),
model: modelRef.slice(slashIndex + 1),
ref: modelRef,
};
}
function buildProviderEvidence() {
const split = splitModelRef(primaryModel);
const live = providerMode !== "mock-openai";
return {
id: split.provider || "openai",
live,
model: {
name: split.model,
ref: split.ref,
},
...(live ? { auth: providerMode } : { fixture: providerMode }),
};
}
function buildPackageSourceEvidence() {
const spec = process.env.OPENCLAW_QA_PACKAGE_SOURCE?.trim() || undefined;
const sha = process.env.OPENCLAW_QA_PACKAGE_SOURCE_SHA?.trim() || undefined;
const kind =
process.env.OPENCLAW_QA_PACKAGE_SOURCE_KIND?.trim() ||
(spec?.endsWith(".tgz") ? "packed-tarball" : spec ? "npm-package" : "source-checkout");
return {
kind,
...(spec ? { spec } : {}),
...(sha ? { sha } : {}),
};
}
function coverageIdForScenario(scenarioId) {
if (scenarioId === "telegram-canary") {
return "channels.telegram.canary";
}
if (scenarioId === "telegram-mentioned-message-reply") {
return "channels.telegram.mention-gating";
}
return undefined;
}
function timingForScenario(scenario) {
const timing = {};
if (typeof scenario.rttMs === "number" && Number.isFinite(scenario.rttMs) && scenario.rttMs > 0) {
timing.rttMs = scenario.rttMs;
}
if (scenario.stats) {
for (const key of ["avgMs", "p50Ms", "p95Ms", "maxMs"]) {
const value = scenario.stats[key];
if (typeof value === "number" && Number.isFinite(value) && value > 0) {
timing[key] = value;
}
}
if (
typeof scenario.stats.total === "number" &&
Number.isFinite(scenario.stats.total) &&
scenario.stats.total > 0
) {
timing.samples = scenario.stats.total;
}
if (
typeof scenario.stats.failed === "number" &&
Number.isFinite(scenario.stats.failed) &&
scenario.stats.failed >= 0
) {
timing.failedSamples = scenario.stats.failed;
}
}
return Object.keys(timing).length > 0 ? timing : undefined;
}
function buildScenarioCoverage(scenarioId) {
const liveCoverage = {
id: "channels.telegram.live",
role: "live-transport",
surfaceIds: ["channels.telegram"],
categoryIds: ["channels.telegram.live"],
};
const coverageId = coverageIdForScenario(scenarioId);
if (!coverageId) {
return [liveCoverage];
}
return [
liveCoverage,
{
id: coverageId,
role: "live-transport-coverage",
surfaceIds: ["channels.telegram"],
categoryIds: ["channels.telegram.live"],
},
];
}
function buildEvidenceSummary(params) {
const provider = buildProviderEvidence();
const generatedAt = new Date().toISOString();
return {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt,
entries: params.scenarios.map((scenario) => {
const timing = timingForScenario(scenario);
return {
test: {
kind: "live-transport-check",
id: scenario.id,
title: scenario.title,
},
mapping: {
profile: "release",
coverage: buildScenarioCoverage(scenario.id),
},
execution: {
runner: "docker",
environment: {
ref: process.env.OPENCLAW_QA_REF?.trim() || process.env.GITHUB_SHA?.trim() || null,
os: process.platform,
nodeVersion: process.version,
},
provider,
channel: {
id: "telegram",
live: true,
driver: "native",
},
packageSource: buildPackageSourceEvidence(),
artifacts: [
{
kind: "summary",
path: "qa-evidence.json",
source: "telegram-rtt",
},
{
kind: "report",
path: "telegram-qa-report.md",
source: "telegram-rtt",
},
{
kind: "transport-observations",
path: "telegram-qa-observed-messages.json",
source: "telegram-rtt",
},
],
},
result: {
status: scenario.status,
...(scenario.status === "pass" ? {} : { failure: { reason: scenario.details } }),
...(timing ? { timing } : {}),
},
};
}),
};
}
async function runWarmScenario(params) {
const samples = [];
let failures = 0;
let passed = 0;
for (let index = 0; passed < params.sampleCount; index += 1) {
const sampleMarker = `${successMarker}_${index + 1}`;
const sample = await runScenario({
allowAnySutReply: false,
id: params.id,
input: `@${params.sutUsername} RTT sample ${index + 1}. Reply with exactly ${sampleMarker}.`,
matchText: sampleMarker,
replyToMessageId: params.replyToMessageId,
sampleIndex: index + 1,
sutId: params.sutId,
timeoutMs: params.sampleTimeoutMs,
title: params.title,
});
if (sample.status === "fail") {
failures += 1;
} else {
passed += 1;
}
samples.push({
index: index + 1,
status: sample.status,
details: sample.details,
...(sample.rttMs === undefined ? {} : { rttMs: sample.rttMs }),
});
if (failures >= params.maxFailures) {
break;
}
if (passed < params.sampleCount) {
await sleep(500);
}
}
const stats = summarizeSamples(samples);
return {
id: params.id,
title: params.title,
status: stats.passed >= params.sampleCount ? "pass" : "fail",
details: `${stats.passed}/${stats.total} warm samples passed`,
rttMs: stats.p50Ms,
samples,
stats,
};
}
function reportMarkdown(summary) {
const lines = ["# Telegram RTT", ""];
for (const scenario of summary.scenarios) {
lines.push(`## ${scenario.title}`, "");
lines.push(`- Status: ${scenario.status}`);
lines.push(`- Details: ${scenario.details}`);
if (scenario.rttMs !== undefined) {
lines.push(`- RTT: ${scenario.rttMs}ms`);
}
if (scenario.stats) {
lines.push(`- Samples: ${scenario.stats.passed}/${scenario.stats.total}`);
if (scenario.stats.avgMs !== undefined) {
lines.push(`- Avg: ${scenario.stats.avgMs}ms`);
}
if (scenario.stats.p50Ms !== undefined) {
lines.push(`- P50: ${scenario.stats.p50Ms}ms`);
}
if (scenario.stats.p95Ms !== undefined) {
lines.push(`- P95: ${scenario.stats.p95Ms}ms`);
}
if (scenario.stats.maxMs !== undefined) {
lines.push(`- Max: ${scenario.stats.maxMs}ms`);
}
}
lines.push("");
}
return lines.join("\n");
}
async function main() {
await fs.mkdir(outputDir, { recursive: true });
const [driverMe, sutMe] = await Promise.all([driver.getMe(), sut.getMe()]);
driverUpdateOffset = (await flushUpdates(driver)) ?? driverUpdateOffset;
const scenarios = [];
const canary = await runScenario({
allowAnySutReply: false,
id: "telegram-canary",
input: `/status@${sutMe.username}`,
sutId: sutMe.id,
timeoutMs: canaryTimeoutMs,
title: "Telegram canary",
});
scenarios.push(canary);
if (scenarioIds.has("telegram-mentioned-message-reply")) {
scenarios.push(
await runWarmScenario({
id: "telegram-mentioned-message-reply",
maxFailures: maxWarmFailures,
replyToMessageId: canary.messageId,
sampleCount: warmSampleCount,
sampleTimeoutMs,
sutId: sutMe.id,
sutUsername: sutMe.username,
title: "Telegram normal reply",
}),
);
}
const failed = scenarios.filter((scenario) => scenario.status === "fail").length;
const reportSummary = {
provider: "telegram",
driver: { id: driverMe.id, username: driverMe.username },
sut: { id: sutMe.id, username: sutMe.username },
startedAt: new Date().toISOString(),
status: failed > 0 ? "fail" : "pass",
totals: { total: scenarios.length, failed, passed: scenarios.length - failed },
scenarios,
};
const evidenceSummary = buildEvidenceSummary({ scenarios });
await fs.writeFile(
path.join(outputDir, "qa-evidence.json"),
`${JSON.stringify(evidenceSummary, null, 2)}\n`,
);
await fs.writeFile(path.join(outputDir, "telegram-qa-report.md"), reportMarkdown(reportSummary));
await fs.writeFile(
path.join(outputDir, "telegram-qa-observed-messages.json"),
`${JSON.stringify(observedMessages, null, 2)}\n`,
);
if (failed > 0) {
process.exitCode = 1;
}
}
await main();

362
scripts/lib/rtt-harness.ts
View file

@ -1,362 +0,0 @@
// Rtt Harness script supports OpenClaw repository automation.
import { execFile, spawn } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { promisify } from "node:util";
import {
QA_EVIDENCE_FILENAME,
validateQaEvidenceSummaryJson,
type QaEvidenceSummaryJson,
type QaEvidenceTiming,
} from "../../extensions/qa-lab/src/evidence-summary.ts";
const execFileAsync = promisify(execFile);
export type RttProviderMode = "mock-openai" | "live-frontier";
export type RttCredentialSource = "env" | "convex";
export type RttCredentialRole = "maintainer" | "ci";
type RttResult = {
package: {
spec: string;
version: string;
};
run: {
id: string;
startedAt: string;
finishedAt: string;
durationMs: number;
status: "pass" | "fail";
};
mode: {
providerMode: RttProviderMode;
scenarios: string[];
};
rtt: {
canaryMs?: number;
mentionReplyMs?: number;
warmSamples?: number[];
avgMs?: number;
p50Ms?: number;
p95Ms?: number;
maxMs?: number;
failedSamples?: number;
};
artifacts: {
rawSummaryPath: string;
rawReportPath: string;
rawObservedMessagesPath: string;
resultPath: string;
};
};
const OPENCLAW_PACKAGE_SPEC_RE =
/^openclaw@(main|alpha|beta|latest|[0-9]{4}\.[1-9][0-9]*\.[1-9][0-9]*(-[1-9][0-9]*|-(alpha|beta)\.[1-9][0-9]*)?)$/u;
const REQUIRED_TELEGRAM_ENV = [
"OPENCLAW_QA_TELEGRAM_GROUP_ID",
"OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN",
"OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN",
] as const;
export function parseRttCredentialSource(value: string): RttCredentialSource {
const normalized = value.trim().toLowerCase();
if (normalized === "env" || normalized === "convex") {
return normalized;
}
throw new Error(`--credential-source must be env or convex; got: ${value}`);
}
export function parseRttCredentialRole(value: string): RttCredentialRole {
const normalized = value.trim().toLowerCase();
if (normalized === "maintainer" || normalized === "ci") {
return normalized;
}
throw new Error(`--credential-role must be maintainer or ci; got: ${value}`);
}
function resolveRttCredentialSource(
env: NodeJS.ProcessEnv,
credentialSource?: RttCredentialSource,
): RttCredentialSource {
if (credentialSource) {
return credentialSource;
}
const rawSource =
env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE ?? env.OPENCLAW_QA_CREDENTIAL_SOURCE;
if (rawSource?.trim()) {
return parseRttCredentialSource(rawSource);
}
if (
env.CI &&
env.OPENCLAW_QA_CONVEX_SITE_URL?.trim() &&
(env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim() || env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim())
) {
return "convex";
}
return "env";
}
function resolveRttCredentialRole(
env: NodeJS.ProcessEnv,
credentialRole?: RttCredentialRole,
): RttCredentialRole {
if (credentialRole) {
return credentialRole;
}
const rawRole = env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE ?? env.OPENCLAW_QA_CREDENTIAL_ROLE;
if (rawRole?.trim()) {
return parseRttCredentialRole(rawRole);
}
return env.CI ? "ci" : "maintainer";
}
export function validateOpenClawPackageSpec(spec: string) {
if (!OPENCLAW_PACKAGE_SPEC_RE.test(spec)) {
throw new Error(
`Package spec must be openclaw@main, openclaw@alpha, openclaw@beta, openclaw@latest, or an exact OpenClaw release version; got: ${spec}`,
);
}
return spec;
}
export function safeRunLabel(input: string) {
return input.replace(/[^a-zA-Z0-9.-]+/gu, "_").replace(/^_+|_+$/gu, "");
}
export function buildRunId(params: { now: Date; spec: string; index?: number }) {
const stamp = params.now.toISOString().replaceAll(":", "").replaceAll(".", "");
const suffix = params.index === undefined ? "" : `-${params.index + 1}`;
return `${stamp}-${safeRunLabel(params.spec)}${suffix}`;
}
export function extractRtt(summary: QaEvidenceSummaryJson) {
const entries = summary.entries ?? [];
const findEntry = (id: string) => entries.find((entry) => entry.test?.id === id);
const canary = findEntry("telegram-canary")?.result?.timing;
const mention = findEntry("telegram-mentioned-message-reply")?.result?.timing;
const rtt: RttResult["rtt"] = {
canaryMs: canary?.rttMs,
mentionReplyMs: mention?.p50Ms ?? mention?.rttMs,
};
appendRttTiming(rtt, mention);
return rtt;
}
function appendRttTiming(rtt: RttResult["rtt"], timing: QaEvidenceTiming | undefined) {
if (timing?.avgMs !== undefined) {
rtt.avgMs = timing.avgMs;
}
if (timing?.p50Ms !== undefined) {
rtt.p50Ms = timing.p50Ms;
}
if (timing?.p95Ms !== undefined) {
rtt.p95Ms = timing.p95Ms;
}
if (timing?.maxMs !== undefined) {
rtt.maxMs = timing.maxMs;
}
if (timing?.failedSamples !== undefined) {
rtt.failedSamples = timing.failedSamples;
}
}
export function createHarnessEnv(params: {
baseEnv: NodeJS.ProcessEnv;
credentialRole?: RttCredentialRole;
credentialSource?: RttCredentialSource;
packageTgz?: string;
providerMode: RttProviderMode;
scenarios: string[];
spec: string;
version: string;
rawOutputDir: string;
samples: number;
sampleTimeoutMs: number;
timeoutMs: number;
}) {
const packageSourceSpec = params.packageTgz ?? params.spec;
return {
...params.baseEnv,
OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC: params.spec,
...(params.packageTgz ? { OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ: params.packageTgz } : {}),
OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL: `${params.spec} (${params.version})`,
OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE: params.providerMode,
OPENCLAW_QA_PACKAGE_SOURCE: packageSourceSpec,
OPENCLAW_QA_PACKAGE_SOURCE_KIND: params.packageTgz ? "packed-tarball" : "npm-package",
...(params.credentialSource
? { OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE: params.credentialSource }
: {}),
...(params.credentialRole
? { OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: params.credentialRole }
: {}),
OPENCLAW_NPM_TELEGRAM_SCENARIOS: params.scenarios.join(","),
OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: params.rawOutputDir,
OPENCLAW_NPM_TELEGRAM_FAST: params.baseEnv.OPENCLAW_NPM_TELEGRAM_FAST ?? "1",
OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: String(params.samples),
OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS: String(params.sampleTimeoutMs),
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: String(params.timeoutMs),
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: String(params.timeoutMs),
};
}
export function assertRequiredEnv(
env: NodeJS.ProcessEnv,
options: {
credentialRole?: RttCredentialRole;
credentialSource?: RttCredentialSource;
} = {},
) {
const credentialSource = resolveRttCredentialSource(env, options.credentialSource);
if (credentialSource === "convex") {
const missing: string[] = [];
const credentialRole = resolveRttCredentialRole(env, options.credentialRole);
if (!env.OPENCLAW_QA_CONVEX_SITE_URL?.trim()) {
missing.push("OPENCLAW_QA_CONVEX_SITE_URL");
}
if (credentialRole === "ci" && !env.OPENCLAW_QA_CONVEX_SECRET_CI?.trim()) {
missing.push("OPENCLAW_QA_CONVEX_SECRET_CI");
}
if (credentialRole === "maintainer" && !env.OPENCLAW_QA_CONVEX_SECRET_MAINTAINER?.trim()) {
missing.push("OPENCLAW_QA_CONVEX_SECRET_MAINTAINER");
}
if (missing.length > 0) {
throw new Error(`Missing Convex Telegram QA credential env: ${missing.join(", ")}`);
}
return;
}
const missing = REQUIRED_TELEGRAM_ENV.filter((key) => !env[key]?.trim());
if (missing.length > 0) {
throw new Error(`Missing Telegram QA env: ${missing.join(", ")}`);
}
}
export async function assertHarnessRoot(harnessRoot: string) {
const scriptPath = path.join(harnessRoot, "scripts/e2e/npm-telegram-rtt-docker.sh");
try {
await fs.access(scriptPath);
} catch {
throw new Error(`Missing OpenClaw Telegram npm harness: ${scriptPath}`);
}
}
export async function assertDockerAvailable() {
try {
await execFileAsync("docker", ["version", "--format", "{{.Server.Version}}"], {
timeout: 10_000,
});
} catch {
throw new Error("Docker is required for RTT runs; install/start Docker and retry.");
}
}
export async function resolvePublishedVersion(spec: string) {
const { stdout } = await execFileAsync("npm", ["view", spec, "version", "--json"], {
timeout: 30_000,
});
const parsed = JSON.parse(stdout.trim()) as unknown;
if (typeof parsed !== "string" || parsed.trim().length === 0) {
throw new Error(`npm did not return a version for ${spec}.`);
}
return parsed.trim();
}
export async function resolveMainVersion(harnessRoot: string) {
const packageJson = JSON.parse(
await fs.readFile(path.join(harnessRoot, "package.json"), "utf8"),
) as { version?: unknown };
if (typeof packageJson.version !== "string" || packageJson.version.trim().length === 0) {
throw new Error("OpenClaw package.json must contain a non-empty version.");
}
const { stdout } = await execFileAsync("git", ["rev-parse", "--short=10", "HEAD"], {
cwd: harnessRoot,
timeout: 10_000,
});
return `${packageJson.version.trim()}+${stdout.trim()}`;
}
export async function readTelegramSummary(summaryPath: string) {
return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(summaryPath, "utf8")));
}
export async function resolveTelegramSummaryPath(outputDir: string) {
return path.join(outputDir, QA_EVIDENCE_FILENAME);
}
export async function writeJson(pathname: string, value: unknown) {
await fs.mkdir(path.dirname(pathname), { recursive: true });
await fs.writeFile(pathname, `${JSON.stringify(value, null, 2)}\n`);
}
export async function appendJsonl(pathname: string, value: unknown) {
await fs.mkdir(path.dirname(pathname), { recursive: true });
await fs.appendFile(pathname, `${JSON.stringify(value)}\n`);
}
export async function runHarness(params: { env: NodeJS.ProcessEnv; harnessRoot: string }) {
const scriptPath = path.join(params.harnessRoot, "scripts/e2e/npm-telegram-rtt-docker.sh");
const child = spawn("bash", [scriptPath], {
cwd: params.harnessRoot,
env: params.env,
stdio: "inherit",
});
const exitCode = await new Promise<number | null>((resolve, reject) => {
child.once("error", reject);
child.once("exit", resolve);
});
return exitCode ?? 1;
}
function rttSummaryFailed(summary: QaEvidenceSummaryJson, requestedScenarios: string[]) {
const entries = summary.entries ?? [];
const requiredScenarioIds = ["telegram-canary", ...requestedScenarios];
for (const scenarioId of requiredScenarioIds) {
const entry = entries.find((candidate) => candidate.test?.id === scenarioId);
if (!entry || entry.result?.status !== "pass") {
return true;
}
const timing = entry.result.timing;
const rttMs =
scenarioId === "telegram-mentioned-message-reply"
? (timing?.p50Ms ?? timing?.rttMs)
: timing?.rttMs;
if (typeof rttMs !== "number" || !Number.isFinite(rttMs)) {
return true;
}
}
return entries.some((entry) => entry.result?.status !== "pass");
}
export function buildRttResult(params: {
artifacts: RttResult["artifacts"];
finishedAt: Date;
providerMode: RttProviderMode;
rawSummary: QaEvidenceSummaryJson;
runId: string;
scenarios: string[];
spec: string;
startedAt: Date;
version: string;
}): RttResult {
const failed = rttSummaryFailed(params.rawSummary, params.scenarios);
return {
package: {
spec: params.spec,
version: params.version,
},
run: {
id: params.runId,
startedAt: params.startedAt.toISOString(),
finishedAt: params.finishedAt.toISOString(),
durationMs: params.finishedAt.getTime() - params.startedAt.getTime(),
status: failed ? "fail" : "pass",
},
mode: {
providerMode: params.providerMode,
scenarios: params.scenarios,
},
rtt: extractRtt(params.rawSummary),
artifacts: params.artifacts,
};
}

31
scripts/mantis/build-telegram-evidence.mjs
View file

@ -324,6 +324,7 @@ export function renderTelegramEvidenceHtml({ observedMessages, summary }) {
export function buildTelegramEvidenceManifest({
candidateRef,
candidateSha,
hasObservedMessages = true,
scenarioLabel,
summary,
summaryArtifactPath = "qa-evidence.json",
@ -381,13 +382,17 @@ export function buildTelegramEvidenceManifest({
path: summaryArtifactPath,
targetPath: "summary.json",
},
{
kind: "metadata",
lane: "run",
label: "Telegram observed messages",
path: "telegram-qa-observed-messages.json",
targetPath: "observed-messages.json",
},
...(hasObservedMessages
? [
{
kind: "metadata",
lane: "run",
label: "Telegram observed messages",
path: "telegram-qa-observed-messages.json",
targetPath: "observed-messages.json",
},
]
: []),
{
kind: "metadata",
lane: "run",
@ -449,16 +454,14 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
mkdirSync(outputDir, { recursive: true });
const evidenceSummaryPath = path.join(outputDir, "qa-evidence.json");
const legacySummaryPath = path.join(outputDir, "telegram-qa-summary.json");
const summaryPath = existsSync(evidenceSummaryPath) ? evidenceSummaryPath : legacySummaryPath;
const usesCurrentEvidenceSummary = existsSync(evidenceSummaryPath);
const summaryPath = usesCurrentEvidenceSummary ? evidenceSummaryPath : legacySummaryPath;
const observedPath = path.join(outputDir, "telegram-qa-observed-messages.json");
const reportPath = path.join(outputDir, "telegram-qa-report.md");
if (!existsSync(summaryPath)) {
throw new Error(`Missing Telegram QA evidence summary: ${evidenceSummaryPath}`);
}
if (!existsSync(observedPath)) {
throw new Error(`Missing Telegram observed messages: ${observedPath}`);
}
const summary = existsSync(evidenceSummaryPath)
const summary = usesCurrentEvidenceSummary
? readJson(evidenceSummaryPath)
: legacyTelegramSummaryToEvidenceSummary(readJson(legacySummaryPath));
const counts = evidenceCounts(summary);
@ -469,12 +472,14 @@ export function writeTelegramEvidence(rawArgs = process.argv.slice(2)) {
}
writeFileSync(reportPath, "# Mantis Telegram Live QA\n\nTelegram QA report was unavailable.\n");
}
const observedMessages = readJson(observedPath);
const hasLegacyObservedMessages = !usesCurrentEvidenceSummary && existsSync(observedPath);
const observedMessages = hasLegacyObservedMessages ? readJson(observedPath) : [];
const transcriptHtml = renderTelegramEvidenceHtml({ observedMessages, summary });
writeFileSync(path.join(outputDir, "telegram-live-transcript.html"), transcriptHtml, "utf8");
const manifest = buildTelegramEvidenceManifest({
candidateRef: args.candidate_ref,
candidateSha: args.candidate_sha,
hasObservedMessages: hasLegacyObservedMessages,
scenarioLabel: args.scenario_label,
summary,
summaryArtifactPath: path.basename(summaryPath),

277
scripts/rtt.ts
View file

@ -1,277 +0,0 @@
#!/usr/bin/env -S node --import tsx
// Rtt script supports OpenClaw repository automation.
import fs from "node:fs/promises";
import path from "node:path";
import {
appendJsonl,
assertDockerAvailable,
assertHarnessRoot,
assertRequiredEnv,
buildRttResult,
buildRunId,
createHarnessEnv,
readTelegramSummary,
resolveTelegramSummaryPath,
resolveMainVersion,
resolvePublishedVersion,
runHarness,
validateOpenClawPackageSpec,
writeJson,
parseRttCredentialRole,
parseRttCredentialSource,
type RttCredentialRole,
type RttCredentialSource,
type RttProviderMode,
} from "./lib/rtt-harness.ts";
const DEFAULT_SCENARIOS = ["telegram-mentioned-message-reply"];
const DEFAULT_PROVIDER_MODE = "mock-openai" satisfies RttProviderMode;
const DEFAULT_TIMEOUT_MS = 180_000;
const DEFAULT_SAMPLES = 20;
const DEFAULT_SAMPLE_TIMEOUT_MS = 30_000;
function usage() {
return [
"Usage: pnpm rtt <openclaw@spec> [--package-tgz PATH] [--provider mock-openai|live-frontier] [--credential-source env|convex] [--credential-role maintainer|ci] [--runs N] [--samples N] [--sample-timeout-ms N] [--timeout-ms N] [--harness-root PATH] [--output PATH]",
"",
"Examples:",
" pnpm rtt openclaw@main --package-tgz .artifacts/package/openclaw.tgz",
" pnpm rtt openclaw@beta",
" pnpm rtt openclaw@2026.4.30",
" pnpm rtt openclaw@latest --provider live-frontier",
].join("\n");
}
function parseProviderMode(value: string): RttProviderMode {
if (value === "mock-openai" || value === "live-frontier") {
return value;
}
throw new Error(`--provider must be mock-openai or live-frontier; got: ${value}`);
}
function parsePositiveInt(label: string, value: string) {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer; got: ${value}`);
}
return parsed;
}
function resolveHome(input: string) {
if (input === "~") {
return process.env.HOME ?? input;
}
if (input.startsWith("~/")) {
return path.join(process.env.HOME ?? "~", input.slice(2));
}
return input;
}
function readRequiredPathArg(argv: string[], index: number, flag: string) {
const value = argv[index + 1] ?? "";
if (!value.trim() || value.startsWith("--")) {
throw new Error(`${flag} requires a path.`);
}
return value;
}
function parseArgs(argv: string[]) {
let spec: string | undefined;
let credentialRole: RttCredentialRole | undefined;
let credentialSource: RttCredentialSource | undefined;
let packageTgz: string | undefined;
let providerMode = DEFAULT_PROVIDER_MODE;
let runs = 1;
let samples = DEFAULT_SAMPLES;
let sampleTimeoutMs = DEFAULT_SAMPLE_TIMEOUT_MS;
let harnessRoot = "~/Developer/clawdbot";
let output = "runs";
let timeoutMs = DEFAULT_TIMEOUT_MS;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--help" || arg === "-h") {
process.stdout.write(`${usage()}\n`);
process.exit(0);
}
if (arg === "--provider") {
providerMode = parseProviderMode(argv[++index] ?? "");
continue;
}
if (arg === "--credential-source") {
credentialSource = parseRttCredentialSource(argv[++index] ?? "");
continue;
}
if (arg === "--credential-role") {
credentialRole = parseRttCredentialRole(argv[++index] ?? "");
continue;
}
if (arg === "--package-tgz") {
const value = readRequiredPathArg(argv, index, "--package-tgz");
index += 1;
packageTgz = path.resolve(resolveHome(value));
continue;
}
if (arg === "--runs") {
runs = parsePositiveInt("--runs", argv[++index] ?? "");
continue;
}
if (arg === "--samples") {
samples = parsePositiveInt("--samples", argv[++index] ?? "");
continue;
}
if (arg === "--sample-timeout-ms") {
sampleTimeoutMs = parsePositiveInt("--sample-timeout-ms", argv[++index] ?? "");
continue;
}
if (arg === "--harness-root") {
harnessRoot = readRequiredPathArg(argv, index, "--harness-root");
index += 1;
continue;
}
if (arg === "--timeout-ms") {
timeoutMs = parsePositiveInt("--timeout-ms", argv[++index] ?? "");
continue;
}
if (arg === "--output") {
output = readRequiredPathArg(argv, index, "--output");
index += 1;
continue;
}
if (arg.startsWith("--")) {
throw new Error(`Unknown option: ${arg}`);
}
if (spec) {
throw new Error(`Unexpected extra argument: ${arg}`);
}
spec = arg;
}
if (!spec) {
throw new Error(`Missing package spec.\n${usage()}`);
}
return {
spec: validateOpenClawPackageSpec(spec),
options: {
packageTgz,
credentialRole,
credentialSource,
providerMode,
runs,
samples,
sampleTimeoutMs,
harnessRoot: path.resolve(resolveHome(harnessRoot)),
output: path.resolve(resolveHome(output)),
scenarios: DEFAULT_SCENARIOS,
timeoutMs,
},
};
}
async function runOne(params: {
index: number;
options: ReturnType<typeof parseArgs>["options"];
spec: string;
version: string;
}) {
const runId = buildRunId({ now: new Date(), spec: params.spec, index: params.index });
const runDir = path.join(params.options.output, runId);
const rawDir = path.join(runDir, "raw");
const resultPath = path.join(runDir, "result.json");
const harnessRawDir = path.join(params.options.harnessRoot, ".artifacts/rtt", runId, "raw");
const rawOutputDir = path.relative(params.options.harnessRoot, harnessRawDir);
const startedAt = new Date();
const env = createHarnessEnv({
baseEnv: process.env,
credentialRole: params.options.credentialRole,
credentialSource: params.options.credentialSource,
packageTgz: params.options.packageTgz,
providerMode: params.options.providerMode,
rawOutputDir,
samples: params.options.samples,
sampleTimeoutMs: params.options.sampleTimeoutMs,
scenarios: params.options.scenarios,
spec: params.spec,
timeoutMs: params.options.timeoutMs,
version: params.version,
});
process.stderr.write(`[rtt] run ${params.index + 1}/${params.options.runs}: ${params.spec}\n`);
const harnessExitCode = await runHarness({ env, harnessRoot: params.options.harnessRoot });
await readTelegramSummary(await resolveTelegramSummaryPath(harnessRawDir));
await fs.rm(rawDir, { recursive: true, force: true });
await fs.mkdir(path.dirname(rawDir), { recursive: true });
await fs.cp(harnessRawDir, rawDir, { recursive: true });
const rawSummaryPath = await resolveTelegramSummaryPath(rawDir);
const rawReportPath = path.join(rawDir, "telegram-qa-report.md");
const rawObservedMessagesPath = path.join(rawDir, "telegram-qa-observed-messages.json");
const rawSummary = await readTelegramSummary(rawSummaryPath);
const finishedAt = new Date();
const result = buildRttResult({
artifacts: {
rawSummaryPath,
rawReportPath,
rawObservedMessagesPath,
resultPath,
},
finishedAt,
providerMode: params.options.providerMode,
rawSummary,
runId,
scenarios: params.options.scenarios,
spec: params.spec,
startedAt,
version: params.version,
});
await writeJson(resultPath, result);
await appendJsonl(path.resolve("data/rtt.jsonl"), result);
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
return {
harnessExitCode,
result,
};
}
async function main() {
const { spec, options } = parseArgs(process.argv.slice(2));
assertRequiredEnv(process.env, {
credentialRole: options.credentialRole,
credentialSource: options.credentialSource,
});
await assertHarnessRoot(options.harnessRoot);
await assertDockerAvailable();
if (spec === "openclaw@main" && !options.packageTgz) {
throw new Error("openclaw@main requires --package-tgz.");
}
const version =
spec === "openclaw@main"
? await resolveMainVersion(options.harnessRoot)
: await resolvePublishedVersion(spec);
let failed = false;
for (let index = 0; index < options.runs; index += 1) {
const run = await runOne({ index, options, spec, version });
failed = failed || run.harnessExitCode !== 0 || run.result.run.status === "fail";
}
if (failed) {
process.exitCode = 1;
}
}
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch((error: unknown) => {
const message = error instanceof Error ? error.message : String(error);
process.stderr.write(`[rtt] ${message}\n`);
process.exitCode = 1;
});
}
export const testing = {
parseArgs,
parseProviderMode,
parsePositiveInt,
resolveHome,
};
export { testing as __testing };

60
test/fixtures/telegram-qa-summary-rtt.json vendored
View file

@ -1,60 +0,0 @@
{
"credentials": {
"kind": "telegram",
"source": "env"
},
"groupId": "-100123",
"startedAt": "2026-05-01T00:00:00.000Z",
"finishedAt": "2026-05-01T00:00:10.000Z",
"cleanupIssues": [],
"counts": {
"total": 2,
"passed": 2,
"failed": 0
},
"scenarios": [
{
"id": "telegram-canary",
"title": "Telegram canary",
"status": "pass",
"details": "reply matched in 1234ms",
"rttMs": 1234
},
{
"id": "telegram-mentioned-message-reply",
"title": "Telegram mentioned message gets a reply",
"status": "pass",
"details": "3/3 warm samples passed",
"rttMs": 5000,
"samples": [
{
"index": 1,
"status": "pass",
"details": "observed SUT message 101",
"rttMs": 4000
},
{
"index": 2,
"status": "pass",
"details": "observed SUT message 102",
"rttMs": 5000
},
{
"index": 3,
"status": "pass",
"details": "observed SUT message 103",
"rttMs": 7000
}
],
"stats": {
"total": 3,
"passed": 3,
"failed": 0,
"avgMs": 5333,
"p50Ms": 5000,
"p95Ms": 7000,
"maxMs": 7000
}
}
]
}

33
test/scripts/mantis-build-telegram-evidence.test.ts
View file

@ -167,7 +167,6 @@ describe("scripts/mantis/build-telegram-evidence", () => {
expect(manifest.comparison.candidate.sha).toBe("abc123");
expect(manifest.artifacts.map((artifact) => artifact.targetPath)).toEqual([
"summary.json",
"observed-messages.json",
"telegram-live-transcript.html",
"report.md",
"mantis-evidence.json",
@ -177,6 +176,35 @@ describe("scripts/mantis/build-telegram-evidence", () => {
);
});
it("does not require observed-message artifacts for current evidence summaries", () => {
const dir = makeTelegramOutput();
rmSync(path.join(dir, "telegram-qa-observed-messages.json"), { force: true });
const result = writeTelegramEvidence(["--output-dir", dir]);
expect(readFileSync(result.transcriptPath, "utf8")).toContain(
"No observed Telegram messages were recorded.",
);
const targetPaths = result.manifest.artifacts.map((artifact) => artifact.targetPath);
expect(targetPaths).toContain("summary.json");
expect(targetPaths).toContain("telegram-live-transcript.html");
expect(targetPaths).toContain("report.md");
expect(targetPaths).not.toContain("observed-messages.json");
});
it("ignores stale observed-message files beside current evidence summaries", () => {
const dir = makeTelegramOutput();
const result = writeTelegramEvidence(["--output-dir", dir]);
expect(readFileSync(result.transcriptPath, "utf8")).toContain(
"No observed Telegram messages were recorded.",
);
expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).not.toContain(
"observed-messages.json",
);
});
it("renders historical Telegram summaries when evidence summaries are absent", () => {
const dir = makeLegacyTelegramOutput();
@ -187,6 +215,9 @@ describe("scripts/mantis/build-telegram-evidence", () => {
expect(
result.manifest.artifacts.find((artifact) => artifact.targetPath === "summary.json"),
).toMatchObject({ path: "telegram-qa-summary.json" });
expect(result.manifest.artifacts.map((artifact) => artifact.targetPath)).toContain(
"observed-messages.json",
);
});
it("does not fabricate a required report artifact for passing Telegram summaries", () => {

51
test/scripts/npm-telegram-live.test.ts
View file

@ -82,8 +82,11 @@ describe("package Telegram live Docker E2E", () => {
expect(script).toContain('docker_e2e_print_log "$run_log"');
expect(script).not.toContain('cat "$run_log"');
expect(script).toContain('"${docker_env[@]}"');
expect(script).toContain('if [ -z "$credential_role" ] && [ -n "${CI:-}" ]');
expect(script).toContain(
'if [ -z "$credential_role" ] && [ "$credential_source" = "convex" ]; then',
);
expect(script).toContain('credential_role="ci"');
expect(script).toContain('credential_role="maintainer"');
});
it("bounds installed-package hot path OpenClaw commands", () => {
@ -142,6 +145,15 @@ describe("package Telegram live Docker E2E", () => {
);
});
it("forwards repeated RTT controls to the package Telegram live lane", () => {
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES");
expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS");
expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES");
expect(script).toContain("OPENCLAW_NPM_TELEGRAM_RTT_CHECKS");
});
it("keeps private QA harness imports local while using the installed package dist", () => {
const script = readFileSync(DOCKER_SCRIPT_PATH, "utf8");
const preparePackage = readFileSync(PREPARE_PACKAGE_PATH, "utf8");
@ -195,6 +207,43 @@ describe("package Telegram live Docker E2E", () => {
).toBe("ci");
});
it("defaults package Telegram RTT for the normal package live lane", () => {
expect(testing.resolveRttOptions({})).toEqual({
rttCount: 20,
rttTimeoutMs: undefined,
maxRttFailures: 20,
rttCheckIds: [],
});
});
it("does not force default RTT onto focused non-RTT scenario runs", () => {
expect(testing.resolveRttOptions({}, ["telegram-canary"])).toEqual({});
});
it("maps repeated RTT env onto package Telegram live options", () => {
expect(
testing.resolveRttOptions({
OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7",
OPENCLAW_NPM_TELEGRAM_RTT_TIMEOUT_MS: "45000",
OPENCLAW_NPM_TELEGRAM_RTT_MAX_FAILURES: "2",
OPENCLAW_NPM_TELEGRAM_RTT_CHECKS: "telegram-mentioned-message-reply",
}),
).toEqual({
rttCount: 7,
rttTimeoutMs: 45_000,
maxRttFailures: 2,
rttCheckIds: ["telegram-mentioned-message-reply"],
});
});
it("rejects invalid repeated RTT env", () => {
expect(() =>
testing.resolveRttOptions({
OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: "7samples",
}),
).toThrow("invalid OPENCLAW_NPM_TELEGRAM_RTT_SAMPLES: 7samples");
});
it("gates package Telegram status on the summary artifact", async () => {
const summaryPath = path.join(mkTempRoot(), "qa-evidence.json");
writeFileSync(

569
test/scripts/npm-telegram-rtt-driver.test.ts
View file

@ -1,569 +0,0 @@
// Npm Telegram Rtt Driver tests cover npm telegram rtt driver script behavior.
import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process";
import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
import { tmpdir } from "node:os";
import path from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { beforeAll, describe, expect, it } from "vitest";
import { createBoundedChildOutput } from "../helpers/bounded-child-output.js";
const DRIVER_SCRIPT = "scripts/e2e/npm-telegram-rtt-driver.mjs";
const QA_EVIDENCE_FILENAME = "qa-evidence.json";
type EvidenceSummaryForTest = {
kind?: string;
entries: Array<{
test: { id: string };
mapping?: {
coverage?: Array<{ id?: string }>;
};
result: {
status?: string;
failure?: { reason?: string };
};
}>;
};
function runDriver(env: Record<string, string>) {
return spawnSync(process.execPath, [DRIVER_SCRIPT], {
encoding: "utf8",
env: {
...process.env,
OPENCLAW_QA_TELEGRAM_API_BASE_URL: "http://127.0.0.1:9",
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token",
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token",
...env,
},
});
}
async function waitForFile(filePath: string, timeoutMs = 3000): Promise<string> {
const startedAt = Date.now();
while (Date.now() - startedAt < timeoutMs) {
if (existsSync(filePath)) {
return readFileSync(filePath, "utf8");
}
await delay(25);
}
throw new Error(`timed out waiting for ${filePath}`);
}
async function stopChild(child: ChildProcessWithoutNullStreams): Promise<void> {
if (child.exitCode !== null) {
return;
}
child.kill("SIGTERM");
const startedAt = Date.now();
while (child.exitCode === null && Date.now() - startedAt < 1000) {
await delay(25);
}
if (child.exitCode === null) {
child.kill("SIGKILL");
}
}
function startStalledJsonServer(portPath: string) {
return spawn(
process.execPath,
[
"--input-type=module",
"--eval",
[
'import net from "node:net";',
'import fs from "node:fs";',
'const server = net.createServer((socket) => socket.write("HTTP/1.1 200 OK\\r\\nContent-Type: application/json\\r\\n\\r\\n"));',
'server.listen(0, "127.0.0.1", () => {',
" const address = server.address();",
" fs.writeFileSync(process.env.PORT_FILE, String(address.port));",
"});",
"setInterval(() => {}, 1000);",
].join("\n"),
],
{
env: { ...process.env, PORT_FILE: portPath },
stdio: "pipe",
},
);
}
function startOversizedJsonServer(portPath: string) {
return spawn(
process.execPath,
[
"--input-type=module",
"--eval",
[
'import net from "node:net";',
'import fs from "node:fs";',
"const server = net.createServer((socket) => {",
' const body = JSON.stringify({ ok: true, result: { id: 1, username: "sut" }, padding: "x".repeat(128) });',
" socket.end(`HTTP/1.1 200 OK\\r\\nContent-Type: application/json\\r\\nContent-Length: ${Buffer.byteLength(body)}\\r\\n\\r\\n${body}`);",
"});",
'server.listen(0, "127.0.0.1", () => {',
" const address = server.address();",
" fs.writeFileSync(process.env.PORT_FILE, String(address.port));",
"});",
"setInterval(() => {}, 1000);",
].join("\n"),
],
{
env: { ...process.env, PORT_FILE: portPath },
stdio: "pipe",
},
);
}
type DriverCaseResult = {
result: ReturnType<typeof spawnSync>;
elapsedMs?: number;
};
type AsyncDriverResult = {
status: number | null;
signal: NodeJS.Signals | null;
stdout: string;
stderr: string;
timedOut: boolean;
};
type TelegramUpdate = {
update_id: number;
message: {
message_id: number;
date: number;
chat: { id: number };
from: { id: number; username: string };
reply_to_message?: { message_id: number };
text: string;
};
};
function runDriverAsync(env: Record<string, string>, timeoutMs = 5000): Promise<AsyncDriverResult> {
return new Promise((resolve) => {
const child = spawn(process.execPath, [DRIVER_SCRIPT], {
env: {
...process.env,
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token",
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token",
...env,
},
stdio: "pipe",
});
const stdout = createBoundedChildOutput();
const stderr = createBoundedChildOutput();
let settled = false;
const timeout = setTimeout(() => {
if (settled) {
return;
}
settled = true;
child.kill("SIGKILL");
resolve({
status: null,
signal: "SIGKILL",
stdout: stdout.text(),
stderr: stderr.text(),
timedOut: true,
});
}, timeoutMs);
timeout.unref?.();
child.stdout.on("data", (chunk) => {
stdout.append(chunk);
});
child.stderr.on("data", (chunk) => {
stderr.append(chunk);
});
child.on("close", (status, signal) => {
if (settled) {
return;
}
settled = true;
clearTimeout(timeout);
resolve({ status, signal, stdout: stdout.text(), stderr: stderr.text(), timedOut: false });
});
});
}
function readRequestJson(req: IncomingMessage): Promise<Record<string, unknown>> {
return new Promise((resolve, reject) => {
const chunks: Buffer[] = [];
req.on("data", (chunk) => {
chunks.push(Buffer.from(chunk));
});
req.on("error", reject);
req.on("end", () => {
const raw = Buffer.concat(chunks).toString("utf8").trim();
if (!raw) {
resolve({});
return;
}
try {
resolve(JSON.parse(raw) as Record<string, unknown>);
} catch (error) {
reject(error instanceof Error ? error : new Error(String(error)));
}
});
});
}
function writeTelegramJson(res: ServerResponse, payload: unknown): void {
const body = `${JSON.stringify(payload)}\n`;
res.writeHead(200, {
"content-length": Buffer.byteLength(body),
"content-type": "application/json",
});
res.end(body);
}
function closeServer(server: Server): Promise<void> {
return new Promise((resolve, reject) => {
server.close((error) => {
if (error) {
reject(error);
return;
}
resolve();
});
});
}
function readEvidenceSummary(outputDir: string) {
const summary = JSON.parse(
readFileSync(path.join(outputDir, QA_EVIDENCE_FILENAME), "utf8"),
) as EvidenceSummaryForTest;
expect(Array.isArray(summary.entries)).toBe(true);
return summary;
}
async function startTelegramApiServer(options: {
canaryReplyToRequest: boolean;
}): Promise<{ baseUrl: string; close: () => Promise<void> }> {
const pendingUpdates: TelegramUpdate[] = [];
let nextMessageId = 100;
let nextUpdateId = 1;
const chatId = -100123;
const nowUnixSeconds = () => Math.floor(Date.now() / 1000);
const pushSutUpdate = (params: { replyToMessageId?: number; text: string }) => {
pendingUpdates.push({
update_id: nextUpdateId++,
message: {
message_id: nextMessageId++,
date: nowUnixSeconds(),
chat: { id: chatId },
from: { id: 222, username: "sut_bot" },
...(params.replyToMessageId === undefined
? {}
: { reply_to_message: { message_id: params.replyToMessageId } }),
text: params.text,
},
});
};
const server = createServer((req, res) => {
void (async () => {
const match = req.url?.match(/^\/bot([^/]+)\/([^/?]+)/u);
if (!match) {
res.writeHead(404).end();
return;
}
const [, token, method] = match;
const body = await readRequestJson(req);
if (method === "getMe") {
writeTelegramJson(res, {
ok: true,
result:
token === "sut-token"
? { id: 222, username: "sut_bot" }
: { id: 111, username: "driver_bot" },
});
return;
}
if (method === "sendMessage") {
const messageId = nextMessageId++;
const text = typeof body.text === "string" ? body.text : "";
if (token === "driver-token" && text.startsWith("/status@")) {
pushSutUpdate({
replyToMessageId: options.canaryReplyToRequest ? messageId : undefined,
text: "status ok",
});
} else if (token === "driver-token" && text.includes("Reply with exactly ")) {
const marker = text.match(/Reply with exactly ([^.]+)\./u)?.[1] ?? "OPENCLAW_E2E_OK_1";
pushSutUpdate({ replyToMessageId: messageId, text: marker });
}
writeTelegramJson(res, {
ok: true,
result: {
message_id: messageId,
date: nowUnixSeconds(),
chat: { id: chatId },
text,
},
});
return;
}
if (method === "getUpdates") {
const offset = typeof body.offset === "number" ? body.offset : 0;
const updates = pendingUpdates.filter((update) => update.update_id >= offset);
if (updates.length === 0) {
await delay(25);
}
writeTelegramJson(res, { ok: true, result: updates });
return;
}
writeTelegramJson(res, { ok: false, description: `unexpected method ${method}` });
})().catch((error: unknown) => {
res.writeHead(500, { "content-type": "text/plain" }).end(String(error));
});
});
await new Promise<void>((resolve) => {
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("fake Telegram server did not bind a TCP port");
}
return {
baseUrl: `http://127.0.0.1:${address.port}`,
close: () => closeServer(server),
};
}
async function runStalledTelegramBodyCase(): Promise<DriverCaseResult> {
const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-driver-"));
const portPath = path.join(root, "port.txt");
const outputDir = path.join(root, "out");
const server = startStalledJsonServer(portPath);
try {
const port = Number.parseInt((await waitForFile(portPath)).trim(), 10);
const startedAt = Date.now();
const result = spawnSync(process.execPath, [DRIVER_SCRIPT], {
encoding: "utf8",
env: {
...process.env,
OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS: "100",
OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir,
OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1",
OPENCLAW_QA_TELEGRAM_API_BASE_URL: `http://127.0.0.1:${port}`,
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000",
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token",
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000",
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token",
},
killSignal: "SIGKILL",
timeout: 2500,
});
return { result, elapsedMs: Date.now() - startedAt };
} finally {
await stopChild(server);
rmSync(root, { force: true, recursive: true });
}
}
async function runOversizedTelegramBodyCase(): Promise<DriverCaseResult> {
const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-driver-"));
const portPath = path.join(root, "port.txt");
const outputDir = path.join(root, "out");
const server = startOversizedJsonServer(portPath);
try {
const port = Number.parseInt((await waitForFile(portPath)).trim(), 10);
const result = spawnSync(process.execPath, [DRIVER_SCRIPT], {
encoding: "utf8",
env: {
...process.env,
OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES: "16",
OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS: "1000",
OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir,
OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1",
OPENCLAW_QA_TELEGRAM_API_BASE_URL: `http://127.0.0.1:${port}`,
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000",
OPENCLAW_QA_TELEGRAM_DRIVER_BOT_TOKEN: "driver-token",
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000",
OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN: "sut-token",
},
killSignal: "SIGKILL",
timeout: 2500,
});
return { result };
} finally {
await stopChild(server);
rmSync(root, { force: true, recursive: true });
}
}
describe("npm Telegram RTT driver", () => {
let stalledBodyCase: DriverCaseResult;
let oversizedBodyCase: DriverCaseResult;
beforeAll(async () => {
[stalledBodyCase, oversizedBodyCase] = await Promise.all([
runStalledTelegramBodyCase(),
runOversizedTelegramBodyCase(),
]);
});
it("rejects loose numeric env values instead of parsing prefixes", () => {
for (const [name, value] of [
["OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS", "180000ms"],
["OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS", "1e3"],
["OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES", "20samples"],
["OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS", "30000ms"],
["OPENCLAW_NPM_TELEGRAM_BOT_API_TIMEOUT_MS", "100ms"],
["OPENCLAW_NPM_TELEGRAM_BOT_API_BODY_MAX_BYTES", "1mb"],
["OPENCLAW_NPM_TELEGRAM_MAX_FAILURES", "2failures"],
]) {
const result = runDriver({ [name]: value });
expect(result.status).not.toBe(0);
expect(result.stderr).toContain(`invalid ${name}: ${value}`);
}
});
it("rejects zero where positive numeric env values are required", () => {
const result = runDriver({ OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "0" });
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("invalid OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: 0");
});
it("rejects empty scenario selections before live Telegram calls", () => {
const result = runDriver({ OPENCLAW_NPM_TELEGRAM_SCENARIOS: "," });
expect(result.status).not.toBe(0);
expect(result.stderr).toContain(
"OPENCLAW_NPM_TELEGRAM_SCENARIOS must include at least one RTT scenario",
);
});
it("rejects unknown scenario selections before live Telegram calls", () => {
const result = runDriver({ OPENCLAW_NPM_TELEGRAM_SCENARIOS: "does-not-exist" });
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("unknown OPENCLAW_NPM_TELEGRAM_SCENARIOS: does-not-exist");
});
it("bounds stalled Telegram Bot API response bodies", async () => {
const { result, elapsedMs } = stalledBodyCase;
expect(result.error).toBeUndefined();
expect(result.signal).not.toBe("SIGKILL");
expect(result.status).not.toBe(0);
expect(result.stderr).toMatch(/abort|timed out|terminated/iu);
expect(elapsedMs).toBeLessThan(2500);
});
it("bounds oversized Telegram Bot API response bodies", async () => {
const { result } = oversizedBodyCase;
expect(result.error).toBeUndefined();
expect(result.signal).not.toBe("SIGKILL");
expect(result.status).not.toBe(0);
expect(result.stderr).toContain("Telegram Bot API getMe response body exceeded 16 bytes");
expect(result.stderr).not.toContain("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
});
it("rejects unrelated SUT messages during the Telegram canary", async () => {
const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-causal-"));
const server = await startTelegramApiServer({ canaryReplyToRequest: false });
try {
const outputDir = path.join(root, "out");
const result = await runDriverAsync({
OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir,
OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1",
OPENCLAW_QA_TELEGRAM_API_BASE_URL: server.baseUrl,
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "250",
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "250",
});
const summary = readEvidenceSummary(outputDir);
const canary = summary.entries.find((entry) => entry.test.id === "telegram-canary");
expect(result.timedOut).toBe(false);
expect(result.status).not.toBe(0);
expect(canary).toMatchObject({
test: { id: "telegram-canary" },
result: { status: "fail" },
});
expect(canary?.result.failure?.reason).toContain("timed out");
expect(existsSync(path.join(outputDir, "telegram-qa-summary.json"))).toBe(false);
} finally {
await server.close();
rmSync(root, { force: true, recursive: true });
}
});
it("accepts reply-threaded SUT messages during the Telegram canary", async () => {
const root = mkdtempSync(path.join(tmpdir(), "openclaw-telegram-rtt-causal-"));
const server = await startTelegramApiServer({ canaryReplyToRequest: true });
try {
const outputDir = path.join(root, "out");
const result = await runDriverAsync({
OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR: outputDir,
OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES: "1",
OPENCLAW_QA_PACKAGE_SOURCE: "/package-under-test/openclaw.tgz",
OPENCLAW_QA_PACKAGE_SOURCE_KIND: "packed-tarball",
OPENCLAW_QA_PACKAGE_SOURCE_SHA: "abc123",
OPENCLAW_QA_TELEGRAM_API_BASE_URL: server.baseUrl,
OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "1000",
OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS: "1000",
});
const summary = readEvidenceSummary(outputDir);
expect(result).toMatchObject({
signal: null,
status: 0,
timedOut: false,
});
expect(summary.kind).toBe("openclaw.qa.evidence-summary");
expect(summary.entries).toEqual(
expect.arrayContaining([
expect.objectContaining({
test: expect.objectContaining({ id: "telegram-canary" }),
mapping: expect.objectContaining({
coverage: expect.arrayContaining([
expect.objectContaining({ id: "channels.telegram.canary" }),
]),
}),
result: expect.objectContaining({ status: "pass" }),
}),
expect.objectContaining({
test: expect.objectContaining({ id: "telegram-mentioned-message-reply" }),
mapping: expect.objectContaining({
coverage: expect.arrayContaining([
expect.objectContaining({ id: "channels.telegram.mention-gating" }),
]),
}),
result: expect.objectContaining({
status: "pass",
timing: expect.objectContaining({
failedSamples: 0,
samples: 1,
}),
}),
}),
]),
);
expect(summary.entries[0]?.execution.packageSource).toEqual({
kind: "packed-tarball",
spec: "/package-under-test/openclaw.tgz",
sha: "abc123",
});
expect(existsSync(path.join(outputDir, "telegram-qa-summary.json"))).toBe(false);
} finally {
await server.close();
rmSync(root, { force: true, recursive: true });
}
});
});

808
test/scripts/rtt-harness.test.ts
View file

@ -1,808 +0,0 @@
// Rtt Harness tests cover rtt harness script behavior.
import { execFile } from "node:child_process";
import fs from "node:fs/promises";
import { createServer, type Server } from "node:http";
import os from "node:os";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { afterEach, describe, expect, it } from "vitest";
import {
appendJsonl,
assertRequiredEnv,
buildRttResult,
buildRunId,
createHarnessEnv,
extractRtt,
resolveTelegramSummaryPath,
safeRunLabel,
validateOpenClawPackageSpec,
} from "../../scripts/lib/rtt-harness.ts";
import { testing as cliTesting } from "../../scripts/rtt.ts";
const TEST_DIR = path.dirname(fileURLToPath(import.meta.url));
const DOCKER_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-docker.sh");
const CREDENTIAL_SCRIPT_PATH = path.resolve(
TEST_DIR,
"../../scripts/e2e/npm-telegram-rtt-credentials.mjs",
);
const CONFIG_SCRIPT_PATH = path.resolve(TEST_DIR, "../../scripts/e2e/npm-telegram-rtt-config.mjs");
const QA_EVIDENCE_FILENAME = "qa-evidence.json";
const CHUNKED_PAYLOAD_MARKER = "__openclawQaCredentialPayloadChunksV1";
const execFileAsync = promisify(execFile);
const tempDirs: string[] = [];
type EvidenceStatus = "pass" | "fail" | "blocked" | "skipped";
type EvidenceTiming = {
rttMs?: number;
avgMs?: number;
p50Ms?: number;
p95Ms?: number;
maxMs?: number;
samples?: number;
failedSamples?: number;
};
type EvidenceSummaryForTest = {
kind: "openclaw.qa.evidence-summary";
schemaVersion: 2;
generatedAt: string;
entries: Array<{
test: {
kind: string;
id: string;
title: string;
};
mapping: {
profile: string;
coverage: [];
};
execution: {
runner: string;
environment: { ref: null; os: string; nodeVersion: string };
provider: {
id: string;
live: boolean;
model: { name: null; ref: null };
fixture: string;
};
packageSource: { kind: string; spec: string };
artifacts: [];
};
result: {
status: EvidenceStatus;
timing?: EvidenceTiming;
};
}>;
};
function makeTelegramRttEvidenceSummary(
options: {
canaryStatus?: EvidenceStatus;
canaryTiming?: EvidenceTiming;
mentionStatus?: EvidenceStatus;
mentionTiming?: EvidenceTiming;
} = {},
): EvidenceSummaryForTest {
const canaryStatus = options.canaryStatus ?? "pass";
const canaryTiming = Object.hasOwn(options, "canaryTiming")
? options.canaryTiming
: { rttMs: 1234 };
const mentionStatus = options.mentionStatus ?? "pass";
const mentionTiming = Object.hasOwn(options, "mentionTiming")
? options.mentionTiming
: {
rttMs: 6000,
avgMs: 5333,
p50Ms: 5000,
p95Ms: 7000,
maxMs: 7000,
samples: 3,
failedSamples: 0,
};
const entry = (
id: string,
title: string,
status: EvidenceStatus,
timing: EvidenceTiming | undefined,
): EvidenceSummaryForTest["entries"][number] => {
const result = timing === undefined ? { status } : { status, timing };
return {
test: {
kind: "live-transport-check",
id,
title,
},
mapping: { profile: "release", coverage: [] },
execution: {
runner: "docker",
environment: { ref: null, os: "linux", nodeVersion: "v24.0.0" },
provider: {
id: "openai",
live: false,
model: { name: null, ref: null },
fixture: "mock-openai",
},
packageSource: { kind: "npm-package", spec: "openclaw@beta" },
artifacts: [],
},
result,
};
};
return {
kind: "openclaw.qa.evidence-summary",
schemaVersion: 2,
generatedAt: "2026-05-01T00:00:00.000Z",
entries: [
entry("telegram-canary", "Telegram canary", canaryStatus, canaryTiming),
entry(
"telegram-mentioned-message-reply",
"Telegram normal reply",
mentionStatus,
mentionTiming,
),
],
};
}
afterEach(async () => {
await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { recursive: true, force: true })));
});
async function listenOnLoopback(server: Server) {
await new Promise<void>((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", () => {
server.off("error", reject);
resolve();
});
});
const address = server.address();
if (!address || typeof address === "string") {
throw new Error("Expected TCP server address.");
}
return address;
}
function closeServer(server: Server) {
return new Promise<void>((resolve, reject) => {
server.close((error) => (error ? reject(error) : resolve()));
});
}
function credentialBrokerEnv(port: number) {
return {
...process.env,
OPENCLAW_QA_ALLOW_INSECURE_HTTP: "1",
OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "test-secret",
OPENCLAW_QA_CONVEX_SITE_URL: `http://127.0.0.1:${port}`,
OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "1000",
OPENCLAW_QA_CREDENTIAL_OWNER_ID: "test-owner",
OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE: "maintainer",
};
}
describe("RTT harness", () => {
it("validates OpenClaw package specs", () => {
expect(validateOpenClawPackageSpec("openclaw@main")).toBe("openclaw@main");
expect(validateOpenClawPackageSpec("openclaw@alpha")).toBe("openclaw@alpha");
expect(validateOpenClawPackageSpec("openclaw@beta")).toBe("openclaw@beta");
expect(validateOpenClawPackageSpec("openclaw@latest")).toBe("openclaw@latest");
expect(validateOpenClawPackageSpec("openclaw@2026.4.30")).toBe("openclaw@2026.4.30");
expect(validateOpenClawPackageSpec("openclaw@2026.4.30-beta.2")).toBe(
"openclaw@2026.4.30-beta.2",
);
expect(validateOpenClawPackageSpec("openclaw@2026.4.30-alpha.2")).toBe(
"openclaw@2026.4.30-alpha.2",
);
expect(() => validateOpenClawPackageSpec("@openclaw/openclaw@beta")).toThrow(
/Package spec must be/,
);
expect(() => validateOpenClawPackageSpec("openclaw@next")).toThrow(/Package spec must be/);
});
it("builds stable run labels", () => {
expect(safeRunLabel("openclaw@beta")).toBe("openclaw_beta");
expect(
buildRunId({
now: new Date("2026-05-01T03:04:05.678Z"),
spec: "openclaw@beta",
index: 1,
}),
).toBe("2026-05-01T030405678Z-openclaw_beta-2");
});
it("constructs harness env without dropping caller env", () => {
const env = createHarnessEnv({
baseEnv: {
OPENCLAW_QA_TELEGRAM_GROUP_ID: "-100123",
OPENCLAW_NPM_TELEGRAM_FAST: "0",
},
providerMode: "mock-openai",
rawOutputDir: ".artifacts/rtt/run/raw",
samples: 20,
sampleTimeoutMs: 30_000,
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@beta",
timeoutMs: 180_000,
version: "2026.4.30-beta.1",
});
expect(env.OPENCLAW_QA_TELEGRAM_GROUP_ID).toBe("-100123");
expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC).toBe("openclaw@beta");
expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_LABEL).toBe("openclaw@beta (2026.4.30-beta.1)");
expect(env.OPENCLAW_NPM_TELEGRAM_PROVIDER_MODE).toBe("mock-openai");
expect(env.OPENCLAW_QA_PACKAGE_SOURCE).toBe("openclaw@beta");
expect(env.OPENCLAW_QA_PACKAGE_SOURCE_KIND).toBe("npm-package");
expect(env.OPENCLAW_NPM_TELEGRAM_SCENARIOS).toBe("telegram-mentioned-message-reply");
expect(env.OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR).toBe(".artifacts/rtt/run/raw");
expect(env.OPENCLAW_NPM_TELEGRAM_FAST).toBe("0");
expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("20");
expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("30000");
expect(env.OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS).toBe("180000");
expect(env.OPENCLAW_QA_TELEGRAM_SCENARIO_TIMEOUT_MS).toBe("180000");
});
it("marks package tarball provenance in RTT evidence env", () => {
const env = createHarnessEnv({
baseEnv: {},
packageTgz: "/tmp/openclaw.tgz",
providerMode: "mock-openai",
rawOutputDir: ".artifacts/rtt/run/raw",
samples: 20,
sampleTimeoutMs: 30_000,
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@main",
timeoutMs: 180_000,
version: "2026.4.30+abc123",
});
expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_SPEC).toBe("openclaw@main");
expect(env.OPENCLAW_NPM_TELEGRAM_PACKAGE_TGZ).toBe("/tmp/openclaw.tgz");
expect(env.OPENCLAW_QA_PACKAGE_SOURCE).toBe("/tmp/openclaw.tgz");
expect(env.OPENCLAW_QA_PACKAGE_SOURCE_KIND).toBe("packed-tarball");
});
it("forwards Convex credential controls without dropping RTT sample controls", () => {
const env = createHarnessEnv({
baseEnv: {
OPENCLAW_QA_CONVEX_SITE_URL: "https://qa-credentials.example.convex.site",
OPENCLAW_QA_CONVEX_SECRET_MAINTAINER: "maintainer-secret",
},
credentialRole: "maintainer",
credentialSource: "convex",
providerMode: "mock-openai",
rawOutputDir: ".artifacts/rtt/run/raw",
samples: 7,
sampleTimeoutMs: 45_000,
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@beta",
timeoutMs: 180_000,
version: "2026.4.30-beta.1",
});
expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_SOURCE).toBe("convex");
expect(env.OPENCLAW_NPM_TELEGRAM_CREDENTIAL_ROLE).toBe("maintainer");
expect(env.OPENCLAW_NPM_TELEGRAM_WARM_SAMPLES).toBe("7");
expect(env.OPENCLAW_NPM_TELEGRAM_SAMPLE_TIMEOUT_MS).toBe("45000");
expect(() =>
assertRequiredEnv(env, { credentialRole: "maintainer", credentialSource: "convex" }),
).not.toThrow();
});
it("exports the Telegram bot token after Convex credentials are sourced", async () => {
const script = await fs.readFile(DOCKER_SCRIPT_PATH, "utf8");
const sourceIndex = script.indexOf('source "$credential_env_file"');
const tokenExportIndex = script.indexOf(
'export TELEGRAM_BOT_TOKEN="${OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN:?missing OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN}"',
);
const installEnvSnapshotIndex = script.indexOf('install_env=("${docker_env[@]}")');
const convexSecretForwardIndex = script.indexOf(
"OPENCLAW_QA_CONVEX_SECRET_CI",
installEnvSnapshotIndex,
);
const bodyLimitForwardIndex = script.indexOf(
"OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES",
installEnvSnapshotIndex,
);
const payloadByteLimitForwardIndex = script.indexOf(
"OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES",
installEnvSnapshotIndex,
);
const payloadChunkLimitForwardIndex = script.indexOf(
"OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_CHUNKS",
installEnvSnapshotIndex,
);
const packageInstallIndex = script.indexOf("npm install -g");
const credentialAcquireIndex = script.indexOf(
"node /app/scripts/e2e/npm-telegram-rtt-credentials.mjs acquire",
);
const heartbeatStartIndex = script.indexOf("start_credential_heartbeat", sourceIndex);
const driverIndex = script.indexOf("node /app/scripts/e2e/npm-telegram-rtt-driver.mjs");
expect(sourceIndex).toBeGreaterThanOrEqual(0);
expect(tokenExportIndex).toBeGreaterThan(sourceIndex);
expect(installEnvSnapshotIndex).toBeGreaterThanOrEqual(0);
expect(convexSecretForwardIndex).toBeGreaterThan(installEnvSnapshotIndex);
expect(bodyLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex);
expect(payloadByteLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex);
expect(payloadChunkLimitForwardIndex).toBeGreaterThan(installEnvSnapshotIndex);
expect(packageInstallIndex).toBeLessThan(credentialAcquireIndex);
expect(script).toContain(
'-e OPENCLAW_E2E_NPM_INSTALL_TIMEOUT="${OPENCLAW_E2E_NPM_INSTALL_TIMEOUT:-600s}"',
);
expect(script).toContain('-e OPENCLAW_QA_PACKAGE_SOURCE="$package_install_source"');
expect(script).toContain('-e OPENCLAW_QA_PACKAGE_SOURCE_KIND="$package_source_kind"');
expect(script).toContain("OPENCLAW_QA_PACKAGE_SOURCE_SHA");
expect(script).toContain(
'"$timeout_bin" --kill-after=30s "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit',
);
expect(script).toContain("elif command -v gtimeout >/dev/null 2>&1; then");
expect(script).toContain('timeout_bin="gtimeout"');
expect(script).toContain(
'echo "timeout or gtimeout is required for OPENCLAW_E2E_NPM_INSTALL_TIMEOUT=$npm_install_timeout" >&2',
);
expect(script).toContain('"$timeout_bin" --kill-after=1s 1s true >/dev/null 2>&1');
expect(script).toContain(
'"$timeout_bin" "$npm_install_timeout" npm install -g "$install_source" --no-fund --no-audit',
);
expect(script).not.toContain(
"running package install without OPENCLAW_E2E_NPM_INSTALL_TIMEOUT",
);
expect(script).toContain("run_logged docker_e2e_docker_run_cmd run --rm");
expect(script).not.toContain("run_logged docker run --rm");
expect(script).toContain("source scripts/lib/openclaw-e2e-instance.sh");
expect(script).toContain('docker_e2e_print_log "$run_log"');
expect(script).not.toContain('cat "$run_log"');
expect(heartbeatStartIndex).toBeGreaterThan(sourceIndex);
expect(heartbeatStartIndex).toBeLessThan(driverIndex);
expect(script).toContain("start_credential_heartbeat() {\n (\n set +e");
expect(script).toContain("Convex credential heartbeat exited with status");
expect(script).toContain('kill -TERM "$rtt_shell_pid"');
expect(script).toContain("const controller = new AbortController();");
expect(script).toContain("const timer = setTimeout(() => controller.abort(), 1000);");
expect(script).toContain('if [ "$mock_ready" != "1" ]; then');
expect(script).toContain("Mock OpenAI server did not become ready");
expect(script).toContain('openclaw_e2e_print_log "$mock_log"');
expect(script).toContain('openclaw_e2e_print_log "$file"');
expect(script).not.toContain('cat "$mock_log"');
expect(script).not.toContain("sed -n '1,260p'");
expect(script).not.toContain("fetch('http://127.0.0.1:${mock_port}/health')");
expect(script).not.toContain('export TELEGRAM_BOT_TOKEN="$OPENCLAW_QA_TELEGRAM_SUT_BOT_TOKEN"');
});
it("rejects oversized chunked RTT credential markers before hydration", async () => {
const credentialModule = (await import(
`${pathToFileURL(CREDENTIAL_SCRIPT_PATH).href}?case=chunk-marker-${Date.now()}`
)) as {
parseChunkedPayloadMarker(payload: unknown): unknown;
};
expect(() =>
credentialModule.parseChunkedPayloadMarker({
[CHUNKED_PAYLOAD_MARKER]: true,
byteLength: 1,
chunkCount: 4097,
}),
).toThrow("Chunked credential payload exceeds 4096 chunks.");
expect(() =>
credentialModule.parseChunkedPayloadMarker({
[CHUNKED_PAYLOAD_MARKER]: true,
byteLength: 64 * 1024 * 1024 + 1,
chunkCount: 1,
}),
).toThrow("Chunked credential payload exceeds 67108864 bytes.");
});
it("keeps RTT Docker artifacts isolated by default", async () => {
const script = await fs.readFile(DOCKER_SCRIPT_PATH, "utf8");
expect(script).toContain(
'RUN_ID="${OPENCLAW_NPM_TELEGRAM_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)-$$}"',
);
expect(script).toContain(
'OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt/$RUN_ID}"',
);
expect(script).toContain('-e OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR="$OUTPUT_DIR"');
expect(script).not.toContain(
'OUTPUT_DIR="${OPENCLAW_NPM_TELEGRAM_OUTPUT_DIR:-.artifacts/qa-e2e/npm-telegram-rtt}"',
);
});
it("keeps broker helper heartbeat handling aligned with QA leases", async () => {
const script = await fs.readFile(CREDENTIAL_SCRIPT_PATH, "utf8");
expect(script).toContain("leaseTtlMs: acquired.leaseTtlMs ?? config.leaseTtlMs");
expect(script).toContain("leaseTtlMs: leaseTtlMsFromLease(config, lease)");
});
it("bounds Convex credential broker response bodies", async () => {
const server = createServer((_request, response) => {
response.writeHead(500, { "content-type": "application/json" });
response.end(JSON.stringify({ status: "error", message: "x".repeat(128) }));
});
const { port } = await listenOnLoopback(server);
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-"));
tempDirs.push(tempDir);
try {
await execFileAsync(
process.execPath,
[
CREDENTIAL_SCRIPT_PATH,
"acquire",
"--lease-file",
path.join(tempDir, "lease.json"),
"--credential-env-file",
path.join(tempDir, "credentials.env"),
],
{
env: {
...credentialBrokerEnv(port),
OPENCLAW_QA_CREDENTIAL_HTTP_MAX_BODY_BYTES: "16",
},
maxBuffer: 128 * 1024,
},
);
throw new Error("Expected credential acquire to fail.");
} catch (error) {
const execError = error as Error & { stderr?: string };
expect(execError.stderr).toContain(
"credential broker acquire response body exceeded 16 bytes",
);
expect(execError.stderr).not.toContain("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
} finally {
await closeServer(server);
}
});
it("does not start another credential acquire after retry delay exhausts the deadline", async () => {
let requests = 0;
const server = createServer((_request, response) => {
requests += 1;
response.writeHead(503, { "content-type": "application/json" });
response.end(
JSON.stringify({
status: "error",
code: "POOL_EXHAUSTED",
message: "credential pool exhausted",
retryAfterMs: 1_000,
}),
);
});
const { port } = await listenOnLoopback(server);
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-retry-"));
tempDirs.push(tempDir);
const startedAt = Date.now();
try {
await execFileAsync(
process.execPath,
[
CREDENTIAL_SCRIPT_PATH,
"acquire",
"--lease-file",
path.join(tempDir, "lease.json"),
"--credential-env-file",
path.join(tempDir, "credentials.env"),
],
{
env: {
...credentialBrokerEnv(port),
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "75",
OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "250",
},
maxBuffer: 128 * 1024,
},
);
throw new Error("Expected credential acquire to fail.");
} catch (error) {
const execError = error as Error & { stderr?: string };
expect(execError.stderr).toContain("credential broker acquire timed out after 75ms");
expect(Date.now() - startedAt).toBeLessThan(500);
expect(requests).toBe(1);
} finally {
await closeServer(server);
}
});
it("caps credential acquire HTTP retries to the remaining acquire deadline", async () => {
let requests = 0;
const server = createServer((_request, response) => {
requests += 1;
if (requests === 1) {
response.writeHead(503, { "content-type": "application/json" });
response.end(
JSON.stringify({
status: "error",
code: "POOL_EXHAUSTED",
message: "credential pool exhausted",
retryAfterMs: 1,
}),
);
}
});
const { port } = await listenOnLoopback(server);
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-cap-"));
tempDirs.push(tempDir);
const startedAt = Date.now();
try {
await execFileAsync(
process.execPath,
[
CREDENTIAL_SCRIPT_PATH,
"acquire",
"--lease-file",
path.join(tempDir, "lease.json"),
"--credential-env-file",
path.join(tempDir, "credentials.env"),
],
{
env: {
...credentialBrokerEnv(port),
OPENCLAW_QA_CREDENTIAL_ACQUIRE_TIMEOUT_MS: "100",
OPENCLAW_QA_CREDENTIAL_HTTP_TIMEOUT_MS: "900",
},
maxBuffer: 128 * 1024,
},
);
throw new Error("Expected credential acquire to fail.");
} catch (error) {
const execError = error as Error & { stderr?: string };
expect(execError.stderr).toContain("credential broker acquire timed out after");
expect(Date.now() - startedAt).toBeLessThan(500);
expect(requests).toBe(2);
} finally {
await closeServer(server);
}
});
it("preserves empty broker responses for successful lease release", async () => {
const server = createServer((_request, response) => {
response.writeHead(204);
response.end();
});
const { port } = await listenOnLoopback(server);
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-credentials-"));
tempDirs.push(tempDir);
const leaseFile = path.join(tempDir, "lease.json");
await fs.writeFile(
leaseFile,
`${JSON.stringify({
kind: "telegram",
ownerId: "test-owner",
actorRole: "maintainer",
credentialId: "credential",
leaseToken: "lease",
})}\n`,
);
try {
await execFileAsync(
process.execPath,
[CREDENTIAL_SCRIPT_PATH, "release", "--lease-file", leaseFile],
{
env: credentialBrokerEnv(port),
},
);
await expect(fs.stat(leaseFile)).rejects.toMatchObject({ code: "ENOENT" });
} finally {
await closeServer(server);
}
});
it("generates final-only Telegram RTT delivery config for release packages", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-config-test-"));
tempDirs.push(tempDir);
const configPath = path.join(tempDir, "config.json");
await execFileAsync(process.execPath, [
CONFIG_SCRIPT_PATH,
configPath,
"12345",
"-100123",
"111:driver-token",
"222:sut-token",
"2026.5.16-beta.6",
]);
const config = JSON.parse(await fs.readFile(configPath, "utf8"));
expect(config.channels.telegram.replyToMode).toBe("first");
expect(config.channels.telegram.streaming).toEqual({ mode: "off" });
expect(config.messages.groupChat.visibleReplies).toBe("automatic");
});
it("extracts RTT values from evidence summaries", () => {
const summary = makeTelegramRttEvidenceSummary();
expect(extractRtt(summary)).toEqual({
canaryMs: 1234,
mentionReplyMs: 5000,
avgMs: 5333,
p50Ms: 5000,
p95Ms: 7000,
maxMs: 7000,
failedSamples: 0,
});
});
it("resolves the evidence summary path for Telegram RTT artifacts", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-summary-test-"));
tempDirs.push(tempDir);
await expect(resolveTelegramSummaryPath(tempDir)).resolves.toBe(
path.join(tempDir, QA_EVIDENCE_FILENAME),
);
});
it("builds RTT result JSON", async () => {
const summary = makeTelegramRttEvidenceSummary();
const result = buildRttResult({
artifacts: {
rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json",
rawReportPath: "runs/run/raw/telegram-qa-report.md",
rawSummaryPath: "runs/run/raw/qa-evidence.json",
resultPath: "runs/run/result.json",
},
finishedAt: new Date("2026-05-01T00:00:12.000Z"),
providerMode: "mock-openai",
rawSummary: summary,
runId: "run",
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@beta",
startedAt: new Date("2026-05-01T00:00:00.000Z"),
version: "2026.4.30-beta.1",
});
expect(result).toStrictEqual({
artifacts: {
rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json",
rawReportPath: "runs/run/raw/telegram-qa-report.md",
rawSummaryPath: "runs/run/raw/qa-evidence.json",
resultPath: "runs/run/result.json",
},
package: { spec: "openclaw@beta", version: "2026.4.30-beta.1" },
run: {
durationMs: 12_000,
finishedAt: "2026-05-01T00:00:12.000Z",
id: "run",
startedAt: "2026-05-01T00:00:00.000Z",
status: "pass",
},
mode: {
providerMode: "mock-openai",
scenarios: ["telegram-mentioned-message-reply"],
},
rtt: {
canaryMs: 1234,
mentionReplyMs: 5000,
avgMs: 5333,
p50Ms: 5000,
p95Ms: 7000,
maxMs: 7000,
failedSamples: 0,
},
});
});
it("marks failed scenario summaries as failed results", () => {
const result = buildRttResult({
artifacts: {
rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json",
rawReportPath: "runs/run/raw/telegram-qa-report.md",
rawSummaryPath: "runs/run/raw/qa-evidence.json",
resultPath: "runs/run/result.json",
},
finishedAt: new Date("2026-05-01T00:00:12.000Z"),
providerMode: "mock-openai",
rawSummary: makeTelegramRttEvidenceSummary({
canaryTiming: { rttMs: 5948 },
mentionStatus: "fail",
mentionTiming: undefined,
}),
runId: "run",
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@latest",
startedAt: new Date("2026-05-01T00:00:00.000Z"),
version: "2026.4.29",
});
expect(result.run.status).toBe("fail");
expect(result.rtt).toEqual({ canaryMs: 5948, mentionReplyMs: undefined });
});
it("marks incomplete RTT summaries as failed results", () => {
const baseParams = {
artifacts: {
rawObservedMessagesPath: "runs/run/raw/telegram-qa-observed-messages.json",
rawReportPath: "runs/run/raw/telegram-qa-report.md",
rawSummaryPath: "runs/run/raw/qa-evidence.json",
resultPath: "runs/run/result.json",
},
finishedAt: new Date("2026-05-01T00:00:12.000Z"),
providerMode: "mock-openai" as const,
runId: "run",
scenarios: ["telegram-mentioned-message-reply"],
spec: "openclaw@latest",
startedAt: new Date("2026-05-01T00:00:00.000Z"),
version: "2026.4.29",
};
const emptySummary = { ...makeTelegramRttEvidenceSummary(), entries: [] };
const canaryOnlySummary = makeTelegramRttEvidenceSummary();
canaryOnlySummary.entries = canaryOnlySummary.entries.slice(0, 1);
for (const rawSummary of [
emptySummary,
canaryOnlySummary,
makeTelegramRttEvidenceSummary({ mentionStatus: "skipped" }),
makeTelegramRttEvidenceSummary({ mentionTiming: undefined }),
]) {
expect(buildRttResult({ ...baseParams, rawSummary }).run.status).toBe("fail");
}
});
it("appends JSONL rows", async () => {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-rtt-test-"));
tempDirs.push(tempDir);
const jsonlPath = path.join(tempDir, "data/rtt.jsonl");
await appendJsonl(jsonlPath, { run: 1 });
await appendJsonl(jsonlPath, { run: 2 });
await expect(fs.readFile(jsonlPath, "utf8")).resolves.toBe('{"run":1}\n{"run":2}\n');
});
it("parses CLI options", () => {
const parsed = cliTesting.parseArgs([
"openclaw@latest",
"--package-tgz",
"/tmp/openclaw.tgz",
"--provider",
"live-frontier",
"--credential-source",
"convex",
"--credential-role",
"ci",
"--runs",
"3",
"--samples",
"5",
"--sample-timeout-ms",
"30000",
"--timeout-ms",
"240000",
"--harness-root",
"/tmp/openclaw",
"--output",
"/tmp/runs",
]);
expect(parsed.spec).toBe("openclaw@latest");
expect(parsed.options).toStrictEqual({
packageTgz: "/tmp/openclaw.tgz",
credentialRole: "ci",
credentialSource: "convex",
providerMode: "live-frontier",
runs: 3,
samples: 5,
sampleTimeoutMs: 30_000,
harnessRoot: "/tmp/openclaw",
output: "/tmp/runs",
scenarios: ["telegram-mentioned-message-reply"],
timeoutMs: 240_000,
});
});
it("rejects missing CLI path option values", () => {
for (const [flag, next] of [
["--package-tgz", "--runs"],
["--harness-root", "--output"],
["--output", "--samples"],
] as const) {
expect(() => cliTesting.parseArgs(["openclaw@latest", flag, next])).toThrow(
`${flag} requires a path.`,
);
}
});
});