mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
refactor(scripts): deslop tooling subdirectories and agent skill helpers (#159218)
Consolidate repeated tooling flows while preserving command contracts. Fix swallowed HTML translation errors and browser-realm error handling in the Google Live smoke. Owner tests and CLI parity passed on Testbox; final broad gate replay follows a fixture lint correction.
This commit is contained in:
Peter Steinberger
GitHub
parent
768b28c060
commit
f8a3418494
37 changed files with 908 additions and 1893 deletions
|
|
@ -330,12 +330,8 @@ function toolCallFamily(row) {
|
|||
}
|
||||
|
||||
function compactToolSummary(familyCounts, dropped) {
|
||||
const families = new Map();
|
||||
for (const [family, count] of familyCounts.entries()) {
|
||||
families.set(family, (families.get(family) || 0) + count);
|
||||
}
|
||||
const ordered = ["read", "write", "execute", "network", "other"]
|
||||
.map((family) => [family, families.get(family) || 0])
|
||||
.map((family) => [family, familyCounts.get(family) || 0])
|
||||
.filter(([, count]) => count > 0)
|
||||
.map(([family, count]) => `${count} ${family}`);
|
||||
const calls = ordered.length ? ordered.join(", ") : "0 tool";
|
||||
|
|
@ -503,10 +499,6 @@ function recentFiles(files, maxFiles) {
|
|||
.map((entry) => entry.file);
|
||||
}
|
||||
|
||||
function candidateFiles(roots, terms, sinceMs, options = {}) {
|
||||
return recentFiles(roots.flatMap((root) => walkJsonl(root, sinceMs)), Number(options["max-files"] || 400));
|
||||
}
|
||||
|
||||
function findSessions(options) {
|
||||
const sinceDays = Number(options["since-days"] || 14);
|
||||
const sinceMs = Date.now() - sinceDays * 24 * 60 * 60 * 1000;
|
||||
|
|
@ -516,14 +508,13 @@ function findSessions(options) {
|
|||
.split(/\s+/)
|
||||
.concat(query.match(/https?:\/\/\S+/g) || [])
|
||||
.filter(Boolean);
|
||||
const files = candidateFiles(roots, terms, sinceMs, options);
|
||||
const files = recentFiles(roots.flatMap((root) => walkJsonl(root, sinceMs)), Number(options["max-files"] || 400));
|
||||
const scanBytes = Number(options["scan-bytes"] || 60000);
|
||||
const results = files
|
||||
return files
|
||||
.map((file) => scoreScanRecord(sessionScanRecord(file, scanBytes), terms, options.cwd))
|
||||
.filter((result) => result.score > 0)
|
||||
.sort((a, b) => b.score - a.score || b.mtime.localeCompare(a.mtime))
|
||||
.slice(0, Number(options.limit || 10));
|
||||
return results;
|
||||
}
|
||||
|
||||
function sessionScanRecords(options) {
|
||||
|
|
@ -578,10 +569,6 @@ ${rows}
|
|||
`;
|
||||
}
|
||||
|
||||
function singlePreviewDocument(record) {
|
||||
return htmlDocument([record]);
|
||||
}
|
||||
|
||||
function readPrs(file) {
|
||||
const raw = fs.readFileSync(file, "utf8");
|
||||
const parsed = JSON.parse(raw);
|
||||
|
|
@ -599,39 +586,28 @@ function main() {
|
|||
console.log(JSON.stringify(findSessions(args), null, 2));
|
||||
return;
|
||||
}
|
||||
if (command === "render") {
|
||||
if (!args.session) throw new Error("--session is required");
|
||||
if (["render", "preview", "append-body"].includes(command)) {
|
||||
if (command === "append-body") {
|
||||
if (!args.body || !args.session) throw new Error("--body and --session are required");
|
||||
} else if (!args.session) throw new Error("--session is required");
|
||||
const rendered = renderSession(args.session, args);
|
||||
if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`);
|
||||
if (args.out) fs.writeFileSync(args.out, rendered.markdown);
|
||||
else process.stdout.write(rendered.markdown);
|
||||
return;
|
||||
}
|
||||
if (command === "preview") {
|
||||
if (!args.session) throw new Error("--session is required");
|
||||
const rendered = renderSession(args.session, args);
|
||||
if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`);
|
||||
const output = singlePreviewDocument({
|
||||
title: args.title || "Agent Transcript Preview",
|
||||
url: args.url || "",
|
||||
session: args.session,
|
||||
safe: rendered.safe,
|
||||
markdown: rendered.markdown,
|
||||
});
|
||||
let output = rendered.markdown;
|
||||
if (command === "preview") {
|
||||
output = htmlDocument([{
|
||||
title: args.title || "Agent Transcript Preview",
|
||||
url: args.url || "",
|
||||
session: args.session,
|
||||
safe: rendered.safe,
|
||||
markdown: rendered.markdown,
|
||||
}]);
|
||||
} else if (command === "append-body") {
|
||||
output = replaceSection(fs.readFileSync(args.body, "utf8"), rendered.markdown);
|
||||
}
|
||||
if (args.out) fs.writeFileSync(args.out, output);
|
||||
else process.stdout.write(output);
|
||||
return;
|
||||
}
|
||||
if (command === "append-body") {
|
||||
if (!args.body || !args.session) throw new Error("--body and --session are required");
|
||||
const rendered = renderSession(args.session, args);
|
||||
if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`);
|
||||
const body = fs.readFileSync(args.body, "utf8");
|
||||
const next = replaceSection(body, rendered.markdown);
|
||||
if (args.out) fs.writeFileSync(args.out, next);
|
||||
else process.stdout.write(next);
|
||||
return;
|
||||
}
|
||||
if (command === "html") {
|
||||
if (!args.prs) throw new Error("--prs is required");
|
||||
const records = [];
|
||||
|
|
|
|||
|
|
@ -129,65 +129,44 @@ export function parseArgs(argv) {
|
|||
shippedRefs: [],
|
||||
writeLedger: false,
|
||||
};
|
||||
const booleanOptions = new Map([
|
||||
["--help", "help"],
|
||||
["--check-github", "checkGithub"],
|
||||
["--json", "json"],
|
||||
["--no-github-snapshot", "noGithubSnapshot"],
|
||||
["--refresh-github-snapshot", "refreshGithubSnapshot"],
|
||||
["--write-ledger", "writeLedger"],
|
||||
]);
|
||||
const valueOptions = new Map([
|
||||
["--base", "base"],
|
||||
["--target", "target"],
|
||||
["--version", "version"],
|
||||
["--release-tag", "releaseTags"],
|
||||
["--release-provenance", "releaseProvenance"],
|
||||
["--shipped-ref", "shippedRefs"],
|
||||
["--github-snapshot", "githubSnapshotPath"],
|
||||
["--main-ref", "mainRef"],
|
||||
["--manifest", "manifestPath"],
|
||||
["--seed-ref", "seedRef"],
|
||||
]);
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--help") {
|
||||
options.help = true;
|
||||
const booleanKey = booleanOptions.get(arg);
|
||||
if (booleanKey) {
|
||||
options[booleanKey] = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
arg === "--check-github" ||
|
||||
arg === "--json" ||
|
||||
arg === "--no-github-snapshot" ||
|
||||
arg === "--refresh-github-snapshot" ||
|
||||
arg === "--write-ledger"
|
||||
) {
|
||||
options[
|
||||
arg === "--check-github"
|
||||
? "checkGithub"
|
||||
: arg === "--write-ledger"
|
||||
? "writeLedger"
|
||||
: arg === "--no-github-snapshot"
|
||||
? "noGithubSnapshot"
|
||||
: arg === "--refresh-github-snapshot"
|
||||
? "refreshGithubSnapshot"
|
||||
: "json"
|
||||
] = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
arg === "--base" ||
|
||||
arg === "--target" ||
|
||||
arg === "--version" ||
|
||||
arg === "--release-tag" ||
|
||||
arg === "--release-provenance" ||
|
||||
arg === "--shipped-ref" ||
|
||||
arg === "--github-snapshot" ||
|
||||
arg === "--main-ref" ||
|
||||
arg === "--manifest" ||
|
||||
arg === "--seed-ref"
|
||||
) {
|
||||
const valueKey = valueOptions.get(arg);
|
||||
if (valueKey) {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
fail(`missing value for ${arg}`);
|
||||
}
|
||||
if (arg === "--release-tag") {
|
||||
options.releaseTags.push(value);
|
||||
} else if (arg === "--release-provenance") {
|
||||
options.releaseProvenance.push(value);
|
||||
} else if (arg === "--shipped-ref") {
|
||||
options.shippedRefs.push(value);
|
||||
} else if (arg === "--manifest") {
|
||||
options.manifestPath = value;
|
||||
} else if (arg === "--github-snapshot") {
|
||||
options.githubSnapshotPath = value;
|
||||
} else if (arg === "--main-ref") {
|
||||
options.mainRef = value;
|
||||
} else if (arg === "--seed-ref") {
|
||||
options.seedRef = value;
|
||||
if (Array.isArray(options[valueKey])) {
|
||||
options[valueKey].push(value);
|
||||
} else {
|
||||
options[arg.slice(2)] = value;
|
||||
options[valueKey] = value;
|
||||
}
|
||||
index += 1;
|
||||
continue;
|
||||
|
|
@ -894,12 +873,8 @@ export function withoutExcludedContributionRecords(record, excludedReferences) {
|
|||
return filtered;
|
||||
}
|
||||
|
||||
function contributionRecordReferences(record) {
|
||||
return [...record.pullRequests.keys()];
|
||||
}
|
||||
|
||||
function contributionRecordMetadataReferences(record) {
|
||||
const references = contributionRecordReferences(record);
|
||||
const references = [...record.pullRequests.keys()];
|
||||
for (const entry of record.pullRequests.values()) {
|
||||
appendReferences(references, entry.references);
|
||||
}
|
||||
|
|
@ -1265,35 +1240,15 @@ function sourceCommits(base, target, mainRef, releaseProvenance = []) {
|
|||
const coauthorsByReference = new Map();
|
||||
const activeCommits = [];
|
||||
for (const commit of commits.values()) {
|
||||
if (commit.isRevert && isActive(commit.hash)) {
|
||||
const coauthorEmails = [...commit.body.matchAll(/^Co-authored-by:\s*.+?<([^>\s]+)>$/gim)].map(
|
||||
(match) => match[1],
|
||||
);
|
||||
activeCommits.push({
|
||||
authorEmail: commit.authorEmail,
|
||||
authorHandle: githubHandleFromNoreply(commit.authorEmail),
|
||||
authorName: commit.authorName,
|
||||
body: commit.body,
|
||||
closingReferences: [],
|
||||
committedAt: commit.committedAt,
|
||||
coauthors: coauthorEmails.map(githubHandleFromNoreply).filter(isEligibleHandle),
|
||||
coauthorEmails,
|
||||
hash: commit.hash,
|
||||
isRevert: true,
|
||||
pullRequests: [],
|
||||
references: [],
|
||||
subject: commit.subject,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (commit.isRevert) {
|
||||
continue;
|
||||
}
|
||||
const uniqueReferences = [...new Set(referencesIn(`${commit.subject}\n${commit.body}`))];
|
||||
const uniqueReferences = commit.isRevert
|
||||
? []
|
||||
: [...new Set(referencesIn(`${commit.subject}\n${commit.body}`))];
|
||||
if (!isActive(commit.hash)) {
|
||||
revertedCommitHashes.add(commit.hash);
|
||||
for (const number of uniqueReferences) {
|
||||
revertedReferences.add(number);
|
||||
if (!commit.isRevert) {
|
||||
revertedCommitHashes.add(commit.hash);
|
||||
for (const number of uniqueReferences) {
|
||||
revertedReferences.add(number);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
|
@ -1306,12 +1261,14 @@ function sourceCommits(base, target, mainRef, releaseProvenance = []) {
|
|||
authorHandle: githubHandleFromNoreply(commit.authorEmail),
|
||||
authorName: commit.authorName,
|
||||
body: commit.body,
|
||||
closingReferences: closingReferencesIn(`${commit.subject}\n${commit.body}`),
|
||||
closingReferences: commit.isRevert
|
||||
? []
|
||||
: closingReferencesIn(`${commit.subject}\n${commit.body}`),
|
||||
committedAt: commit.committedAt,
|
||||
coauthors,
|
||||
coauthorEmails,
|
||||
hash: commit.hash,
|
||||
isRevert: false,
|
||||
isRevert: commit.isRevert,
|
||||
pullRequests: [],
|
||||
references: uniqueReferences,
|
||||
subject: commit.subject,
|
||||
|
|
@ -2009,23 +1966,12 @@ function thanksFor(node, coauthorHandles) {
|
|||
if (node.author?.__typename === "User" && isEligibleHandle(node.author.login)) {
|
||||
handles.push(node.author.login);
|
||||
}
|
||||
for (const handle of coauthorHandles) {
|
||||
if (!handles.some((candidate) => candidate.toLowerCase() === handle.toLowerCase())) {
|
||||
handles.push(handle);
|
||||
}
|
||||
}
|
||||
appendUnique(handles, coauthorHandles);
|
||||
return handles;
|
||||
}
|
||||
|
||||
function addHandles(handles, additions) {
|
||||
for (const handle of additions) {
|
||||
if (!isEligibleHandle(handle)) {
|
||||
continue;
|
||||
}
|
||||
if (!handles.some((candidate) => candidate.toLowerCase() === handle.toLowerCase())) {
|
||||
handles.push(handle);
|
||||
}
|
||||
}
|
||||
appendUnique(handles, [...additions].filter(isEligibleHandle));
|
||||
return handles;
|
||||
}
|
||||
|
||||
|
|
@ -2033,10 +1979,6 @@ function titleReferences(entries) {
|
|||
return [...new Set(entries.flatMap((entry) => referencesIn(entry.title)))];
|
||||
}
|
||||
|
||||
function releaseTitle(title) {
|
||||
return title;
|
||||
}
|
||||
|
||||
function withSentenceEnding(value) {
|
||||
return /[.!?]$/.test(value) ? value : `${value}.`;
|
||||
}
|
||||
|
|
@ -2236,7 +2178,7 @@ export function ledgerFor(
|
|||
.filter(Boolean);
|
||||
return {
|
||||
number,
|
||||
title: releaseTitle(node.title.replace(/\s+/g, " ").trim()),
|
||||
title: node.title.replace(/\s+/g, " ").trim(),
|
||||
type: node.__typename,
|
||||
mergedAt: node.mergedAt,
|
||||
closingIssuesReferences: node.closingIssuesReferences,
|
||||
|
|
@ -2863,10 +2805,8 @@ function main() {
|
|||
);
|
||||
}
|
||||
}
|
||||
if (errors.length === 0) {
|
||||
if (options.writeLedger) {
|
||||
writeReleaseChangelog({ rootDir, version: options.version, section: candidateChangelog });
|
||||
}
|
||||
if (errors.length === 0 && options.writeLedger) {
|
||||
writeReleaseChangelog({ rootDir, version: options.version, section: candidateChangelog });
|
||||
}
|
||||
|
||||
const result = {
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ import {
|
|||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { setTimeout as defaultSleep } from "node:timers/promises";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { detectChangedScope } from "../../../../scripts/ci-changed-scope.mjs";
|
||||
import { isDirectRunUrl } from "../../../../scripts/lib/direct-run.mjs";
|
||||
|
|
@ -1986,29 +1986,16 @@ function defaultResumeGatewaySuspension(checkout, suspensionId, deployment) {
|
|||
}
|
||||
|
||||
async function stopManagedGateway(runCommand, checkout, deployment) {
|
||||
if (!deployment) {
|
||||
await runUpdateCommand(
|
||||
runCommand,
|
||||
"gateway.stop",
|
||||
process.execPath,
|
||||
["dist/index.js", "gateway", "stop"],
|
||||
checkout,
|
||||
{
|
||||
phase: "gateway stop",
|
||||
serviceState: "stopping",
|
||||
timeoutMs: COMMAND_TIMEOUT_MS.gatewayService,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
await runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.bootout",
|
||||
"/bin/launchctl",
|
||||
["bootout", `gui/${process.getuid()}/${deployment.label}`],
|
||||
deployment ? "launchd.bootout" : "gateway.stop",
|
||||
deployment ? "/bin/launchctl" : process.execPath,
|
||||
deployment
|
||||
? ["bootout", `gui/${process.getuid()}/${deployment.label}`]
|
||||
: ["dist/index.js", "gateway", "stop"],
|
||||
checkout,
|
||||
{
|
||||
phase: "Gateway LaunchAgent bootout",
|
||||
phase: deployment ? "Gateway LaunchAgent bootout" : "gateway stop",
|
||||
serviceState: "stopping",
|
||||
timeoutMs: COMMAND_TIMEOUT_MS.gatewayService,
|
||||
},
|
||||
|
|
@ -2105,12 +2092,7 @@ function isTrustedSourceControlBuild(checkout, buildState, currentHead) {
|
|||
) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
git(checkout, ["merge-base", "--is-ancestor", commit, currentHead]);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
return isAncestorCommit(checkout, commit, currentHead);
|
||||
}
|
||||
|
||||
function resolveGatewayControlDeployment(checkout, deployment, buildBefore, currentHead) {
|
||||
|
|
@ -2418,7 +2400,7 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options
|
|||
const serviceTarget = `${domain}/${deployment.label}`;
|
||||
const waitForProcess = options.waitForProcess ?? waitForManagedGatewayProcess;
|
||||
const now = options.now ?? Date.now;
|
||||
if (!options.startupTrace) {
|
||||
const start = async () => {
|
||||
await runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.enable",
|
||||
|
|
@ -2439,7 +2421,10 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options
|
|||
waitForProcess,
|
||||
options.sleep ?? defaultSleep,
|
||||
);
|
||||
return { processStartedAt: timestampAt(now) };
|
||||
return timestampAt(now);
|
||||
};
|
||||
if (!options.startupTrace) {
|
||||
return { processStartedAt: await start() };
|
||||
}
|
||||
|
||||
const readLaunchdEnvironment = options.readLaunchdEnvironment ?? readLaunchdEnvironmentVariable;
|
||||
|
|
@ -2461,27 +2446,7 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options
|
|||
},
|
||||
);
|
||||
try {
|
||||
await runUpdateCommand(
|
||||
runCommand,
|
||||
"launchd.enable",
|
||||
"/bin/launchctl",
|
||||
["enable", serviceTarget],
|
||||
checkout,
|
||||
{
|
||||
phase: "Gateway LaunchAgent enable",
|
||||
serviceState: "stopped",
|
||||
timeoutMs: COMMAND_TIMEOUT_MS.gatewayService,
|
||||
},
|
||||
);
|
||||
await bootstrapLaunchAgentAndWait(
|
||||
runCommand,
|
||||
checkout,
|
||||
deployment,
|
||||
domain,
|
||||
waitForProcess,
|
||||
options.sleep ?? defaultSleep,
|
||||
);
|
||||
processStartedAt = timestampAt(now);
|
||||
processStartedAt = await start();
|
||||
} catch (error) {
|
||||
restartError = error;
|
||||
}
|
||||
|
|
@ -2751,21 +2716,15 @@ function markGatewayMilestones(timing, observation, observedAt, deepRpcUpperBoun
|
|||
if (!observation) {
|
||||
return;
|
||||
}
|
||||
if (observation.listenerReady) {
|
||||
recordGatewayTimestamp(
|
||||
timing,
|
||||
"listenerReadyAt",
|
||||
deepRpcUpperBoundAt ?? observedAt,
|
||||
deepRpcUpperBoundAt ? "no-later-than" : "observed",
|
||||
);
|
||||
}
|
||||
if (observation.healthzReady) {
|
||||
recordGatewayTimestamp(
|
||||
timing,
|
||||
"healthzReadyAt",
|
||||
deepRpcUpperBoundAt ?? observedAt,
|
||||
deepRpcUpperBoundAt ? "no-later-than" : "observed",
|
||||
);
|
||||
for (const key of ["listenerReady", "healthzReady"]) {
|
||||
if (observation[key]) {
|
||||
recordGatewayTimestamp(
|
||||
timing,
|
||||
`${key}At`,
|
||||
deepRpcUpperBoundAt ?? observedAt,
|
||||
deepRpcUpperBoundAt ? "no-later-than" : "observed",
|
||||
);
|
||||
}
|
||||
}
|
||||
if (observation.readyzReady) {
|
||||
recordGatewayTimestamp(timing, "readyzReadyAt", observedAt);
|
||||
|
|
@ -2804,16 +2763,14 @@ async function readGatewayHealth(runCommand, checkout, deployment) {
|
|||
["health", "--verbose", "--json"],
|
||||
deployment,
|
||||
);
|
||||
let healthSummary;
|
||||
try {
|
||||
healthSummary = JSON.parse(healthOutput);
|
||||
return JSON.parse(healthOutput);
|
||||
} catch (error) {
|
||||
throw new UpdateInvariantError(
|
||||
"gateway_health_invalid",
|
||||
`Gateway health probe did not return JSON: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
return healthSummary;
|
||||
}
|
||||
await runUpdateCommand(
|
||||
runCommand,
|
||||
|
|
@ -2844,10 +2801,6 @@ async function verifyGateway(runCommand, checkout, expectedSha, deployment = nul
|
|||
};
|
||||
}
|
||||
|
||||
function defaultSleep(ms) {
|
||||
return delay(ms);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {(command: string, args: string[], checkout: string, options?: Record<string, unknown>) => void | Promise<void>} runCommand
|
||||
* @param {string} checkout
|
||||
|
|
@ -3255,7 +3208,7 @@ export function findExactMacTarget(processes, executable) {
|
|||
}
|
||||
|
||||
async function defaultVerifyMacTarget(checkout) {
|
||||
await delay(10_000);
|
||||
await defaultSleep(10_000);
|
||||
const executable = path.join(checkout, "dist/OpenClaw.app/Contents/MacOS/OpenClaw");
|
||||
const processes = execFileSync(
|
||||
"ps",
|
||||
|
|
|
|||
|
|
@ -14,8 +14,6 @@ import { execPlainGh } from "../../../../scripts/lib/plain-gh.mjs";
|
|||
const REPO = "openclaw/openclaw";
|
||||
const REPO_URL = `https://github.com/${REPO}`;
|
||||
|
||||
// ─── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function fail(message) {
|
||||
console.error(`error: ${message}`);
|
||||
process.exit(1);
|
||||
|
|
@ -23,7 +21,6 @@ function fail(message) {
|
|||
|
||||
function tmpFile(purpose) {
|
||||
const filePath = path.join(os.tmpdir(), `secretscan-${purpose}-${crypto.randomUUID()}`);
|
||||
// 预创建文件，限制权限为 owner-only
|
||||
fs.writeFileSync(filePath, "", { mode: 0o600 });
|
||||
return filePath;
|
||||
}
|
||||
|
|
@ -60,6 +57,19 @@ function ghGraphQL(query, options = {}) {
|
|||
return gh(["api", "graphql", "-f", `query=${query}`], options);
|
||||
}
|
||||
|
||||
function contentEditCount(nodeId, typeName) {
|
||||
const result = ghGraphQL(`{
|
||||
node(id: "${nodeId}") {
|
||||
... on ${typeName} {
|
||||
userContentEdits(first: 50) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}`);
|
||||
return result?.data?.node?.userContentEdits?.totalCount ?? 0;
|
||||
}
|
||||
|
||||
function isBodyLocationType(locationType) {
|
||||
return locationType === "issue_body" || locationType === "pull_request_body";
|
||||
}
|
||||
|
|
@ -251,8 +261,6 @@ function createDiscussionComment(discussionNodeId, body, replyToNodeId) {
|
|||
return result?.data?.addDiscussionComment?.comment;
|
||||
}
|
||||
|
||||
// ─── Commands ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* fetch-alert <number>
|
||||
* Fetch alert metadata + locations. Never exposes .secret.
|
||||
|
|
@ -270,7 +278,6 @@ function cmdFetchAlert(alertNumber) {
|
|||
"--paginate",
|
||||
"--slurp",
|
||||
]);
|
||||
// --paginate + --slurp 确保多页结果合并为一个 JSON 数组
|
||||
const flatLocations = Array.isArray(locations?.[0])
|
||||
? locations.flat()
|
||||
: Array.isArray(locations)
|
||||
|
|
@ -305,6 +312,7 @@ function cmdFetchContent(locationJson) {
|
|||
const location = JSON.parse(locationJson);
|
||||
const type = location.type;
|
||||
const details = location.details;
|
||||
let result;
|
||||
|
||||
if (type === "discussion_comment") {
|
||||
const commentUrl = details.discussion_comment_url;
|
||||
|
|
@ -332,30 +340,23 @@ function cmdFetchContent(locationJson) {
|
|||
const bodyFile = tmpFile("body.md");
|
||||
fs.writeFileSync(bodyFile, comment.body || "");
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
comment_node_id: comment.id,
|
||||
discussion_node_id: discussionId,
|
||||
reply_to_node_id: comment.replyTo?.id ?? null,
|
||||
discussion_number: Number(discussionNumber),
|
||||
discussion_comment_db_id: Number(discussionCommentDbId),
|
||||
author: comment.author?.login,
|
||||
html_url: comment.url || commentUrl,
|
||||
edit_history_count: comment.userContentEdits?.totalCount ?? 0,
|
||||
body_file: bodyFile,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
result = {
|
||||
type,
|
||||
comment_node_id: comment.id,
|
||||
discussion_node_id: discussionId,
|
||||
reply_to_node_id: comment.replyTo?.id ?? null,
|
||||
discussion_number: Number(discussionNumber),
|
||||
discussion_comment_db_id: Number(discussionCommentDbId),
|
||||
author: comment.author?.login,
|
||||
html_url: comment.url || commentUrl,
|
||||
edit_history_count: comment.userContentEdits?.totalCount ?? 0,
|
||||
body_file: bodyFile,
|
||||
};
|
||||
} else if (
|
||||
type === "issue_comment" ||
|
||||
type === "pull_request_comment" ||
|
||||
type === "pull_request_review_comment"
|
||||
) {
|
||||
// Extract comment ID from URL
|
||||
const commentUrl =
|
||||
details.issue_comment_url ||
|
||||
details.pull_request_comment_url ||
|
||||
|
|
@ -368,150 +369,62 @@ function cmdFetchContent(locationJson) {
|
|||
const bodyFile = tmpFile("body.md");
|
||||
fs.writeFileSync(bodyFile, comment.body || "");
|
||||
|
||||
// Fetch edit history
|
||||
const nodeId = comment.node_id;
|
||||
const typeName =
|
||||
type === "pull_request_review_comment" ? "PullRequestReviewComment" : "IssueComment";
|
||||
const gql = ghGraphQL(`{
|
||||
node(id: "${nodeId}") {
|
||||
... on ${typeName} {
|
||||
userContentEdits(first: 50) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}`);
|
||||
const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0;
|
||||
const editCount = contentEditCount(nodeId, typeName);
|
||||
|
||||
// Extract issue number from html_url
|
||||
const htmlUrl = comment.html_url || details.html_url || "";
|
||||
const issueMatch = htmlUrl.match(/\/(issues|pull)\/(\d+)/);
|
||||
const issueNumber = issueMatch ? issueMatch[2] : null;
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
comment_id: comment.id,
|
||||
node_id: nodeId,
|
||||
author: comment.user?.login,
|
||||
issue_number: issueNumber,
|
||||
html_url: htmlUrl,
|
||||
edit_history_count: editCount,
|
||||
body_file: bodyFile,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} else if (type === "issue_body") {
|
||||
const issueUrl = details.issue_body_url || details.issue_url;
|
||||
if (!issueUrl) {
|
||||
fail("No issue URL in location details");
|
||||
result = {
|
||||
type,
|
||||
comment_id: comment.id,
|
||||
node_id: nodeId,
|
||||
author: comment.user?.login,
|
||||
issue_number: issueNumber,
|
||||
html_url: htmlUrl,
|
||||
edit_history_count: editCount,
|
||||
body_file: bodyFile,
|
||||
};
|
||||
} else if (isBodyLocationType(type)) {
|
||||
const isPullRequest = type === "pull_request_body";
|
||||
const url = isPullRequest
|
||||
? details.pull_request_body_url || details.pull_request_url
|
||||
: details.issue_body_url || details.issue_url;
|
||||
if (!url) {
|
||||
fail(`No ${isPullRequest ? "PR" : "issue"} URL in location details`);
|
||||
}
|
||||
|
||||
const issue = gh(["api", issueUrl]);
|
||||
const content = gh(["api", url]);
|
||||
const bodyFile = tmpFile("body.md");
|
||||
fs.writeFileSync(bodyFile, issue.body || "");
|
||||
|
||||
const nodeId = issue.node_id;
|
||||
const number = issue.number;
|
||||
const gql = ghGraphQL(`{
|
||||
node(id: "${nodeId}") {
|
||||
... on Issue {
|
||||
userContentEdits(first: 50) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}`);
|
||||
const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0;
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
issue_number: number,
|
||||
node_id: nodeId,
|
||||
author: issue.user?.login,
|
||||
html_url: issue.html_url,
|
||||
edit_history_count: editCount,
|
||||
body_file: bodyFile,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
} else if (type === "pull_request_body") {
|
||||
const prUrl = details.pull_request_body_url || details.pull_request_url;
|
||||
if (!prUrl) {
|
||||
fail("No PR URL in location details");
|
||||
}
|
||||
|
||||
const pr = gh(["api", prUrl]);
|
||||
const bodyFile = tmpFile("body.md");
|
||||
fs.writeFileSync(bodyFile, pr.body || "");
|
||||
|
||||
const nodeId = pr.node_id;
|
||||
const number = pr.number;
|
||||
const gql = ghGraphQL(`{
|
||||
node(id: "${nodeId}") {
|
||||
... on PullRequest {
|
||||
userContentEdits(first: 50) {
|
||||
totalCount
|
||||
}
|
||||
}
|
||||
}
|
||||
}`);
|
||||
const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0;
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
pr_number: number,
|
||||
node_id: nodeId,
|
||||
author: pr.user?.login,
|
||||
merged: pr.merged,
|
||||
state: pr.state,
|
||||
html_url: pr.html_url,
|
||||
edit_history_count: editCount,
|
||||
body_file: bodyFile,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
fs.writeFileSync(bodyFile, content.body || "");
|
||||
const editCount = contentEditCount(content.node_id, isPullRequest ? "PullRequest" : "Issue");
|
||||
result = {
|
||||
type,
|
||||
[isPullRequest ? "pr_number" : "issue_number"]: content.number,
|
||||
node_id: content.node_id,
|
||||
author: content.user?.login,
|
||||
...(isPullRequest ? { merged: content.merged, state: content.state } : {}),
|
||||
html_url: content.html_url,
|
||||
edit_history_count: editCount,
|
||||
body_file: bodyFile,
|
||||
};
|
||||
} else if (type === "commit") {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
commit_sha: details.commit_sha,
|
||||
path: details.path,
|
||||
start_line: details.start_line,
|
||||
end_line: details.end_line,
|
||||
html_url: details.html_url || details.commit_url || details.blob_url || null,
|
||||
// No body file for commits
|
||||
body_file: null,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
result = {
|
||||
type,
|
||||
commit_sha: details.commit_sha,
|
||||
path: details.path,
|
||||
start_line: details.start_line,
|
||||
end_line: details.end_line,
|
||||
html_url: details.html_url || details.commit_url || details.blob_url || null,
|
||||
body_file: null,
|
||||
};
|
||||
} else {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
type,
|
||||
unsupported: true,
|
||||
details,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
result = { type, unsupported: true, details };
|
||||
}
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -720,11 +633,8 @@ function cmdNotify(target, author, locationType, secretTypes, replyToNodeId) {
|
|||
"**Please rotate these credentials immediately.**",
|
||||
"",
|
||||
"These secrets were publicly exposed and should be considered compromised.",
|
||||
]
|
||||
.filter((line) => line !== undefined)
|
||||
.join("\n");
|
||||
].join("\n");
|
||||
|
||||
// Discussion comments must be notified via GraphQL
|
||||
if (locationType === "discussion_comment") {
|
||||
const newComment = createDiscussionComment(target, body, replyToNodeId);
|
||||
console.log(
|
||||
|
|
@ -737,7 +647,6 @@ function cmdNotify(target, author, locationType, secretTypes, replyToNodeId) {
|
|||
return;
|
||||
}
|
||||
|
||||
// Issue/PR comments via REST
|
||||
const bodyFile = tmpFile("notify.md");
|
||||
fs.writeFileSync(bodyFile, body);
|
||||
|
||||
|
|
@ -807,7 +716,6 @@ function cmdListOpen() {
|
|||
"--slurp",
|
||||
]);
|
||||
|
||||
// --slurp 将分页结果合并为 [[page1], [page2], ...] 需要 flat
|
||||
const flat = Array.isArray(alerts?.[0]) ? alerts.flat() : Array.isArray(alerts) ? alerts : [];
|
||||
const rows = flat.map((a) => ({
|
||||
number: a.number,
|
||||
|
|
@ -888,29 +796,23 @@ function cmdSummary(jsonFile) {
|
|||
console.log(lines.join("\n"));
|
||||
}
|
||||
|
||||
// ─── Dispatch ───────────────────────────────────────────────────────────────
|
||||
|
||||
const args = [];
|
||||
|
||||
const commands = {
|
||||
"fetch-alert": () => cmdFetchAlert(args[0]),
|
||||
"fetch-content": () => cmdFetchContent(args[0]),
|
||||
"redact-body": () => cmdRedactBody(args[0], args[1], args[2]),
|
||||
"redact-body-if-needed": () => cmdRedactBodyIfNeeded(args[0], args[1], args[2], args[3], args[4]),
|
||||
"delete-comment": () => cmdDeleteComment(args[0]),
|
||||
"delete-discussion-comment": () => cmdDeleteDiscussionComment(args[0]),
|
||||
"recreate-comment": () => cmdRecreateComment(args[0], args[1]),
|
||||
"recreate-discussion-comment": () => cmdRecreateDiscussionComment(args[0], args[1], args[2]),
|
||||
notify: () => cmdNotify(args[0], args[1], args[2], args[3], args[4]),
|
||||
resolve: () => cmdResolve(args[0], args[1], args[2]),
|
||||
"list-open": () => cmdListOpen(),
|
||||
summary: () => cmdSummary(args[0]),
|
||||
"fetch-alert": cmdFetchAlert,
|
||||
"fetch-content": cmdFetchContent,
|
||||
"redact-body": cmdRedactBody,
|
||||
"redact-body-if-needed": cmdRedactBodyIfNeeded,
|
||||
"delete-comment": cmdDeleteComment,
|
||||
"delete-discussion-comment": cmdDeleteDiscussionComment,
|
||||
"recreate-comment": cmdRecreateComment,
|
||||
"recreate-discussion-comment": cmdRecreateDiscussionComment,
|
||||
notify: cmdNotify,
|
||||
resolve: cmdResolve,
|
||||
"list-open": cmdListOpen,
|
||||
summary: cmdSummary,
|
||||
};
|
||||
|
||||
function main(argv = process.argv.slice(2)) {
|
||||
const [command, ...commandArgs] = argv;
|
||||
args.length = 0;
|
||||
args.push(...commandArgs);
|
||||
|
||||
if (!command || !commands[command]) {
|
||||
console.error(
|
||||
|
|
@ -935,7 +837,7 @@ function main(argv = process.argv.slice(2)) {
|
|||
process.exit(1);
|
||||
}
|
||||
|
||||
commands[command]();
|
||||
commands[command](...commandArgs);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
|
|
|
|||
|
|
@ -31,7 +31,8 @@ if (commandPath && statusPath) {
|
|||
inFlight: queue?.inFlight.size ?? 0,
|
||||
};
|
||||
};
|
||||
const writeStatus = (value) => {
|
||||
const writeStatus = (seq, command) => {
|
||||
const value = { seq, command, status: "completed", ...queueState(wrappedKey) };
|
||||
const pending = `${statusPath}.${process.pid}.tmp`;
|
||||
fs.writeFileSync(pending, `${JSON.stringify(value)}\n`, { mode: 0o600 });
|
||||
fs.renameSync(pending, statusPath);
|
||||
|
|
@ -59,35 +60,20 @@ if (commandPath && statusPath) {
|
|||
if (!heldOnce) {
|
||||
heldOnce = true;
|
||||
if (waitSeq) {
|
||||
writeStatus({
|
||||
seq: waitSeq,
|
||||
command: "waitHeld",
|
||||
status: "completed",
|
||||
...queueState(wrappedKey),
|
||||
});
|
||||
writeStatus(waitSeq, "waitHeld");
|
||||
}
|
||||
await gate.promise;
|
||||
}
|
||||
return originalCallback(run);
|
||||
});
|
||||
writeStatus({
|
||||
seq: command.seq,
|
||||
command: "arm",
|
||||
status: "completed",
|
||||
...queueState(wrappedKey),
|
||||
});
|
||||
writeStatus(command.seq, "arm");
|
||||
return;
|
||||
}
|
||||
if (!wrappedKey) return;
|
||||
lastSeq = command.seq;
|
||||
if (command.command === "waitHeld") {
|
||||
if (heldOnce) {
|
||||
writeStatus({
|
||||
seq: command.seq,
|
||||
command: "waitHeld",
|
||||
status: "completed",
|
||||
...queueState(wrappedKey),
|
||||
});
|
||||
writeStatus(command.seq, "waitHeld");
|
||||
} else {
|
||||
waitSeq = command.seq;
|
||||
}
|
||||
|
|
@ -95,12 +81,7 @@ if (commandPath && statusPath) {
|
|||
}
|
||||
if (command.command === "release") {
|
||||
gate.resolve();
|
||||
writeStatus({
|
||||
seq: command.seq,
|
||||
command: "release",
|
||||
status: "completed",
|
||||
...queueState(wrappedKey),
|
||||
});
|
||||
writeStatus(command.seq, "release");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -13,56 +13,48 @@ function writeJson(response, status, body) {
|
|||
response.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
function writeEvents(response, events) {
|
||||
function startEvents(response) {
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-store",
|
||||
connection: "keep-alive",
|
||||
});
|
||||
}
|
||||
|
||||
function completionChunk(id, delta, finishReason = null) {
|
||||
return {
|
||||
id,
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
||||
};
|
||||
}
|
||||
|
||||
function writeEvents(response, events) {
|
||||
startEvents(response);
|
||||
for (const event of events) response.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
response.end("data: [DONE]\n\n");
|
||||
}
|
||||
|
||||
function writeInterleavedMonologue(response) {
|
||||
const id = "chatcmpl_interleaved_monologue";
|
||||
writeEvents(response, [
|
||||
{
|
||||
id: "chatcmpl_interleaved_monologue",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [
|
||||
{
|
||||
index: 0,
|
||||
delta: {
|
||||
role: "assistant",
|
||||
reasoning_details: [
|
||||
{ type: "response.output_text", text: "PRIVATE_MONOLOGUE" },
|
||||
{ type: "reasoning.text", text: "HIDDEN_REASONING" },
|
||||
{ type: "response.text", text: "PUBLIC_FINAL" },
|
||||
],
|
||||
},
|
||||
finish_reason: null,
|
||||
},
|
||||
completionChunk(id, {
|
||||
role: "assistant",
|
||||
reasoning_details: [
|
||||
{ type: "response.output_text", text: "PRIVATE_MONOLOGUE" },
|
||||
{ type: "reasoning.text", text: "HIDDEN_REASONING" },
|
||||
{ type: "response.text", text: "PUBLIC_FINAL" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl_interleaved_monologue",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
|
||||
},
|
||||
}),
|
||||
completionChunk(id, {}, "stop"),
|
||||
]);
|
||||
}
|
||||
|
||||
function writeIncompleteToolUse(response) {
|
||||
const id = "chatcmpl_incomplete_tool_use";
|
||||
writeEvents(response, [
|
||||
{
|
||||
id: "chatcmpl_incomplete_tool_use",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }],
|
||||
},
|
||||
{
|
||||
id: "chatcmpl_incomplete_tool_use",
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }],
|
||||
},
|
||||
completionChunk(id, { role: "assistant" }),
|
||||
completionChunk(id, {}, "tool_calls"),
|
||||
]);
|
||||
}
|
||||
|
||||
|
|
@ -78,18 +70,10 @@ async function writePreviewToolBoundary(response, body) {
|
|||
writeJson(response, 400, { error: { message: "Preview fixture requires streaming and exec" } });
|
||||
return;
|
||||
}
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-store",
|
||||
connection: "keep-alive",
|
||||
});
|
||||
startEvents(response);
|
||||
const writeChunk = (delta, finishReason = null) =>
|
||||
response.write(
|
||||
`data: ${JSON.stringify({
|
||||
id: `chatcmpl_preview_${toolResults}`,
|
||||
object: "chat.completion.chunk",
|
||||
choices: [{ index: 0, delta, finish_reason: finishReason }],
|
||||
})}\n\n`,
|
||||
`data: ${JSON.stringify(completionChunk(`chatcmpl_preview_${toolResults}`, delta, finishReason))}\n\n`,
|
||||
);
|
||||
writeChunk({ role: "assistant" });
|
||||
if (toolResults === 0) {
|
||||
|
|
@ -119,63 +103,82 @@ async function writePreviewToolBoundary(response, body) {
|
|||
response.end("data: [DONE]\n\n");
|
||||
}
|
||||
|
||||
async function writeStreamingThrottle(response) {
|
||||
const itemId = "msg_streaming_throttle_107179";
|
||||
const finalText = "STREAM_FINAL_107179";
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-store",
|
||||
connection: "keep-alive",
|
||||
});
|
||||
const previewEvents = [
|
||||
function messageStartEvents(item, deltas = [item.content[0].text]) {
|
||||
return [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: 0,
|
||||
item: {
|
||||
type: "message",
|
||||
id: itemId,
|
||||
role: "assistant",
|
||||
phase: "final_answer",
|
||||
status: "in_progress",
|
||||
content: [],
|
||||
},
|
||||
item: { ...item, status: "in_progress", content: [] },
|
||||
},
|
||||
...["QA streaming ", "preview in ", "progress"].map((delta) => ({
|
||||
...deltas.map((delta) => ({
|
||||
type: "response.output_text.delta",
|
||||
item_id: itemId,
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
delta,
|
||||
})),
|
||||
];
|
||||
for (const event of previewEvents) response.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
}
|
||||
|
||||
function messageDoneEvents(item) {
|
||||
return [
|
||||
{
|
||||
type: "response.output_text.done",
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
text: item.content[0].text,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item },
|
||||
];
|
||||
}
|
||||
|
||||
function functionCallEvents(item, outputIndex = 0) {
|
||||
return [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: outputIndex,
|
||||
item: { ...item, arguments: "" },
|
||||
},
|
||||
{
|
||||
type: "response.function_call_arguments.delta",
|
||||
item_id: item.id,
|
||||
output_index: outputIndex,
|
||||
delta: item.arguments,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: outputIndex, item },
|
||||
];
|
||||
}
|
||||
|
||||
function completedEvent(id, output) {
|
||||
return {
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id,
|
||||
status: "completed",
|
||||
output,
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function writeStreamingThrottle(response) {
|
||||
const item = {
|
||||
type: "message",
|
||||
id: itemId,
|
||||
id: "msg_streaming_throttle_107179",
|
||||
role: "assistant",
|
||||
phase: "final_answer",
|
||||
status: "completed",
|
||||
content: [{ type: "output_text", text: finalText, annotations: [] }],
|
||||
content: [{ type: "output_text", text: "STREAM_FINAL_107179", annotations: [] }],
|
||||
};
|
||||
startEvents(response);
|
||||
for (const event of messageStartEvents(item, ["QA streaming ", "preview in ", "progress"])) {
|
||||
response.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_500));
|
||||
for (const event of [
|
||||
{
|
||||
type: "response.output_text.done",
|
||||
item_id: itemId,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
text: finalText,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_streaming_throttle_107179",
|
||||
status: "completed",
|
||||
output: [item],
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
},
|
||||
...messageDoneEvents(item),
|
||||
completedEvent("resp_streaming_throttle_107179", [item]),
|
||||
]) {
|
||||
response.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
}
|
||||
|
|
@ -191,67 +194,27 @@ function responseEvents(text) {
|
|||
content: [{ type: "output_text", text, annotations: [] }],
|
||||
};
|
||||
return [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: 0,
|
||||
item: { ...item, status: "in_progress", content: [] },
|
||||
},
|
||||
{
|
||||
type: "response.output_text.delta",
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
delta: text,
|
||||
},
|
||||
{
|
||||
type: "response.output_text.done",
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
text,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_telegram_triage_fixture",
|
||||
status: "completed",
|
||||
output: [item],
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
},
|
||||
...messageStartEvents(item),
|
||||
...messageDoneEvents(item),
|
||||
completedEvent("resp_telegram_triage_fixture", [item]),
|
||||
];
|
||||
}
|
||||
|
||||
function functionCallResponse(name, argumentsText, suffix) {
|
||||
const item = {
|
||||
type: "function_call",
|
||||
id: `fc_${name}_${suffix}`,
|
||||
call_id: `call_${name}_${suffix}`,
|
||||
name,
|
||||
arguments: argumentsText,
|
||||
};
|
||||
return [...functionCallEvents(item), completedEvent(`resp_${name}_${suffix}`, [item])];
|
||||
}
|
||||
|
||||
function toolCallEvents(sequence) {
|
||||
const args = JSON.stringify({ args: { id: "session_status", args: {} } });
|
||||
const suffix = createHash("sha256").update(`${sequence}:${args}`).digest("hex").slice(0, 10);
|
||||
const item = {
|
||||
type: "function_call",
|
||||
id: `fc_tool_call_${suffix}`,
|
||||
call_id: `call_tool_call_${suffix}`,
|
||||
name: "tool_call",
|
||||
arguments: args,
|
||||
};
|
||||
return [
|
||||
{ type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } },
|
||||
{
|
||||
type: "response.function_call_arguments.delta",
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
delta: args,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: `resp_tool_call_${suffix}`,
|
||||
status: "completed",
|
||||
output: [item],
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
},
|
||||
];
|
||||
return functionCallResponse("tool_call", args, suffix);
|
||||
}
|
||||
|
||||
function namedToolCallEvents(name, args, sequence) {
|
||||
|
|
@ -260,32 +223,7 @@ function namedToolCallEvents(name, args, sequence) {
|
|||
.update(`${name}:${sequence}:${argumentsText}`)
|
||||
.digest("hex")
|
||||
.slice(0, 10);
|
||||
const item = {
|
||||
type: "function_call",
|
||||
id: `fc_${name}_${suffix}`,
|
||||
call_id: `call_${name}_${suffix}`,
|
||||
name,
|
||||
arguments: argumentsText,
|
||||
};
|
||||
return [
|
||||
{ type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } },
|
||||
{
|
||||
type: "response.function_call_arguments.delta",
|
||||
item_id: item.id,
|
||||
output_index: 0,
|
||||
delta: argumentsText,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: `resp_${name}_${suffix}`,
|
||||
status: "completed",
|
||||
output: [item],
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
},
|
||||
];
|
||||
return functionCallResponse(name, argumentsText, suffix);
|
||||
}
|
||||
|
||||
function hasTool(body, name) {
|
||||
|
|
@ -300,56 +238,18 @@ function draftThenExecEvents() {
|
|||
status: "completed",
|
||||
content: [{ type: "output_text", text: "GOOD_DRAFT_115041", annotations: [] }],
|
||||
};
|
||||
const argumentsText = JSON.stringify({ command: "printf tool-ok" });
|
||||
const call = {
|
||||
type: "function_call",
|
||||
id: "fc_exec_115041",
|
||||
call_id: "call_exec_115041",
|
||||
name: "exec",
|
||||
arguments: argumentsText,
|
||||
arguments: JSON.stringify({ command: "printf tool-ok" }),
|
||||
};
|
||||
return [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: 0,
|
||||
item: { ...message, status: "in_progress", content: [] },
|
||||
},
|
||||
{
|
||||
type: "response.output_text.delta",
|
||||
item_id: message.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
delta: "GOOD_DRAFT_115041",
|
||||
},
|
||||
{
|
||||
type: "response.output_text.done",
|
||||
item_id: message.id,
|
||||
output_index: 0,
|
||||
content_index: 0,
|
||||
text: "GOOD_DRAFT_115041",
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 0, item: message },
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
output_index: 1,
|
||||
item: { ...call, arguments: "" },
|
||||
},
|
||||
{
|
||||
type: "response.function_call_arguments.delta",
|
||||
item_id: call.id,
|
||||
output_index: 1,
|
||||
delta: argumentsText,
|
||||
},
|
||||
{ type: "response.output_item.done", output_index: 1, item: call },
|
||||
{
|
||||
type: "response.completed",
|
||||
response: {
|
||||
id: "resp_good_draft_115041",
|
||||
status: "completed",
|
||||
output: [message, call],
|
||||
usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 },
|
||||
},
|
||||
},
|
||||
...messageStartEvents(message),
|
||||
...messageDoneEvents(message),
|
||||
...functionCallEvents(call, 1),
|
||||
completedEvent("resp_good_draft_115041", [message, call]),
|
||||
];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -324,7 +324,6 @@ class TdClient:
|
|||
self.lib.td_execute(json.dumps({"@type": "setLogVerbosityLevel", "new_verbosity_level": 0}).encode())
|
||||
self.client = self.lib.td_json_client_create()
|
||||
self.extra = 0
|
||||
self.pending = {}
|
||||
self.users = {}
|
||||
self.updates = []
|
||||
atexit.register(self.destroy)
|
||||
|
|
@ -361,7 +360,6 @@ class TdClient:
|
|||
extra = str(self.extra)
|
||||
payload = dict(payload)
|
||||
payload["@extra"] = extra
|
||||
self.pending[extra] = payload["@type"]
|
||||
self.lib.td_json_client_send(self.client, json.dumps(payload).encode())
|
||||
return extra
|
||||
|
||||
|
|
@ -399,17 +397,13 @@ class TdClient:
|
|||
tdlib_method=payload["@type"],
|
||||
)
|
||||
return item
|
||||
self.handle_update(item)
|
||||
self.updates.append(item)
|
||||
raise DriverError(
|
||||
f"Timed out waiting for {payload['@type']}",
|
||||
tdlib_method=payload["@type"],
|
||||
tdlib_timed_out=True,
|
||||
)
|
||||
|
||||
def handle_update(self, item):
|
||||
self.updates.append(item)
|
||||
return item
|
||||
|
||||
def next_update(self, timeout=1.0):
|
||||
if self.updates:
|
||||
return self.updates.pop(0)
|
||||
|
|
@ -915,10 +909,7 @@ def print_result(payload, as_json=False, output=""):
|
|||
output_path = Path(output).expanduser()
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
if as_json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
|
||||
|
||||
def command_configure(args):
|
||||
|
|
@ -1719,59 +1710,51 @@ def main():
|
|||
parser = argparse.ArgumentParser(description="Telegram real-user E2E driver backed by TDLib.")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
configure = sub.add_parser("configure")
|
||||
def command_parser(name, func, common=True):
|
||||
command = sub.add_parser(name)
|
||||
if common:
|
||||
add_common(command)
|
||||
command.set_defaults(func=func)
|
||||
return command
|
||||
|
||||
configure = command_parser("configure", command_configure, common=False)
|
||||
configure.add_argument("--api-id")
|
||||
configure.add_argument("--api-hash")
|
||||
configure.add_argument("--tdlib-path")
|
||||
configure.add_argument("--chat")
|
||||
configure.add_argument("--sut-username")
|
||||
configure.add_argument("--sut-id")
|
||||
configure.set_defaults(func=command_configure)
|
||||
|
||||
doctor = sub.add_parser("doctor")
|
||||
doctor = command_parser("doctor", command_doctor, common=False)
|
||||
doctor.add_argument("--json", action="store_true")
|
||||
doctor.add_argument("--output", default="")
|
||||
doctor.set_defaults(func=command_doctor)
|
||||
|
||||
login = sub.add_parser("login")
|
||||
add_common(login)
|
||||
login = command_parser("login", command_login)
|
||||
login.add_argument("--qr", action="store_true", default=True)
|
||||
login.add_argument("--phone", default="")
|
||||
login.add_argument("--code", default="")
|
||||
login.add_argument("--password", default="")
|
||||
login.add_argument("--first-name", default="")
|
||||
login.add_argument("--last-name", default="")
|
||||
login.set_defaults(func=command_login)
|
||||
|
||||
status = sub.add_parser("status")
|
||||
add_common(status)
|
||||
status = command_parser("status", command_status)
|
||||
status.add_argument("--check-chat", default="")
|
||||
status.add_argument("--require-chat", default="")
|
||||
status.set_defaults(func=command_status)
|
||||
|
||||
resolve_chat = sub.add_parser("resolve-chat")
|
||||
add_common(resolve_chat)
|
||||
resolve_chat = command_parser("resolve-chat", command_resolve_chat)
|
||||
resolve_chat.add_argument("--chat", required=True)
|
||||
resolve_chat.set_defaults(func=command_resolve_chat)
|
||||
|
||||
for name in ("prepare-group", "cleanup-group"):
|
||||
group = sub.add_parser(name)
|
||||
add_common(group)
|
||||
group = command_parser(name, command_test_group)
|
||||
group.add_argument("--chat", default="")
|
||||
group.set_defaults(func=command_test_group)
|
||||
|
||||
for name in ("prepare-private-forum", "cleanup-private-forum"):
|
||||
private_forum = sub.add_parser(name)
|
||||
add_common(private_forum)
|
||||
private_forum.set_defaults(func=command_private_forum)
|
||||
command_parser(name, command_private_forum)
|
||||
|
||||
confirm_qr = sub.add_parser("confirm-qr")
|
||||
add_common(confirm_qr)
|
||||
confirm_qr = command_parser("confirm-qr", command_confirm_qr)
|
||||
confirm_qr.add_argument("--link", required=True)
|
||||
confirm_qr.set_defaults(func=command_confirm_qr)
|
||||
|
||||
send = sub.add_parser("send")
|
||||
add_common(send)
|
||||
send = command_parser("send", command_send)
|
||||
send.add_argument("--chat", default="")
|
||||
send.add_argument("--text")
|
||||
send.add_argument("--photo", action="append", default=[])
|
||||
|
|
@ -1779,20 +1762,16 @@ def main():
|
|||
send.add_argument("--reply-to")
|
||||
send.add_argument("--thread-id", type=int, default=0)
|
||||
send.add_argument("--forum-topic-id", type=int)
|
||||
send.set_defaults(func=command_send)
|
||||
|
||||
wait = sub.add_parser("wait")
|
||||
add_common(wait)
|
||||
wait = command_parser("wait", command_wait)
|
||||
wait.add_argument("--chat", default="")
|
||||
wait.add_argument("--expect", action="append", default=[])
|
||||
wait.add_argument("--from-bot", default="")
|
||||
wait.add_argument("--reply-to")
|
||||
wait.add_argument("--thread-id", type=int, default=0)
|
||||
wait.add_argument("--after-message-id", type=int, default=0)
|
||||
wait.set_defaults(func=command_wait)
|
||||
|
||||
probe = sub.add_parser("probe")
|
||||
add_common(probe)
|
||||
probe = command_parser("probe", command_probe)
|
||||
probe.add_argument("--chat", default="")
|
||||
probe.add_argument("--text", default="@{sut} Reply exactly: USER-E2E-{run}")
|
||||
probe.add_argument("--photo", action="append", default=[])
|
||||
|
|
@ -1803,24 +1782,18 @@ def main():
|
|||
probe.add_argument("--thread-id", type=int, default=0)
|
||||
probe.add_argument("--require-reply", action="store_true", default=True)
|
||||
probe.add_argument("--any-sut-reply", dest="require_reply", action="store_false")
|
||||
probe.set_defaults(func=command_probe)
|
||||
|
||||
transcript = sub.add_parser("transcript")
|
||||
add_common(transcript)
|
||||
transcript = command_parser("transcript", command_transcript)
|
||||
transcript.add_argument("--chat", default="")
|
||||
transcript.add_argument("--limit", type=int, default=20)
|
||||
transcript.set_defaults(func=command_transcript)
|
||||
|
||||
chats = sub.add_parser("chats")
|
||||
add_common(chats)
|
||||
chats = command_parser("chats", command_chats)
|
||||
chats.add_argument("--limit", type=int, default=50)
|
||||
chats.set_defaults(func=command_chats)
|
||||
|
||||
serve = sub.add_parser("serve")
|
||||
serve = command_parser("serve", command_serve, common=False)
|
||||
serve.add_argument("--chat", default="")
|
||||
serve.add_argument("--observe-chat", action="append", default=[])
|
||||
serve.add_argument("--timeout-ms", type=int, default=120000)
|
||||
serve.set_defaults(func=command_serve)
|
||||
|
||||
args = parser.parse_args()
|
||||
if args.command == "send" and not args.text and not args.photo:
|
||||
|
|
|
|||
|
|
@ -51,10 +51,6 @@ def message_text(message):
|
|||
return content_text(message.get("content") or {})
|
||||
|
||||
|
||||
def content_kind(message):
|
||||
return (message.get("content") or {}).get("@type", "")
|
||||
|
||||
|
||||
class EventRecorder:
|
||||
def __init__(self, client, chat_id, record_path, sut_user_id=None):
|
||||
self.client = client
|
||||
|
|
@ -156,7 +152,7 @@ class EventRecorder:
|
|||
"senderId": sender,
|
||||
"isSut": self.sut_user_id is not None and sender == self.sut_user_id,
|
||||
"isOutgoing": bool(message.get("is_outgoing")),
|
||||
"contentType": content_kind(message),
|
||||
"contentType": content.get("@type", ""),
|
||||
"textLen": len(text),
|
||||
"text": text,
|
||||
"richMessageIsFull": rich_message.get("is_full") if isinstance(rich_message, dict) else None,
|
||||
|
|
@ -214,13 +210,14 @@ class EventRecorder:
|
|||
# Ack and status reactions arrive here, on the *user's own* message.
|
||||
# A bot reacting to its own message produces no update for the user,
|
||||
# so probe this by reacting to a message the QA user sent.
|
||||
reactions = (
|
||||
((update.get("interaction_info") or {}).get("reactions") or {}).get("reactions") or []
|
||||
)
|
||||
reactions = [
|
||||
reaction
|
||||
for reaction in ((update.get("interaction_info") or {}).get("reactions") or {}).get("reactions") or []
|
||||
if isinstance(reaction, dict)
|
||||
]
|
||||
emojis = "".join(
|
||||
(reaction.get("type") or {}).get("emoji", "")
|
||||
for reaction in reactions
|
||||
if isinstance(reaction, dict)
|
||||
)
|
||||
yield (
|
||||
"reaction",
|
||||
|
|
@ -232,11 +229,8 @@ class EventRecorder:
|
|||
"reactionCount": sum(
|
||||
int(reaction.get("total_count") or 0)
|
||||
for reaction in reactions
|
||||
if isinstance(reaction, dict)
|
||||
),
|
||||
"reactionTypes": [
|
||||
reaction.get("type") for reaction in reactions if isinstance(reaction, dict)
|
||||
],
|
||||
"reactionTypes": [reaction.get("type") for reaction in reactions],
|
||||
},
|
||||
)
|
||||
elif kind == "updateChatAction":
|
||||
|
|
@ -306,23 +300,12 @@ class EventRecorder:
|
|||
"elapsedMs": e["elapsedMs"],
|
||||
"kind": e["kind"],
|
||||
"messageId": e["messageId"],
|
||||
"botApiMessageId": e.get("botApiMessageId"),
|
||||
"textLen": e.get("textLen"),
|
||||
"contentType": e.get("contentType"),
|
||||
"senderId": e.get("senderId"),
|
||||
"isSut": e.get("isSut"),
|
||||
"isOutgoing": e.get("isOutgoing"),
|
||||
"replyToMessageId": e.get("replyToMessageId"),
|
||||
"quoteText": e.get("quoteText"),
|
||||
"topicType": e.get("topicType"),
|
||||
"topicId": e.get("topicId"),
|
||||
"reactionText": e.get("reactionText"),
|
||||
"reactionCount": e.get("reactionCount"),
|
||||
"actionType": e.get("actionType"),
|
||||
"status": e.get("status"),
|
||||
"buttonText": e.get("buttonText"),
|
||||
"durationMs": e.get("durationMs"),
|
||||
"error": e.get("error"),
|
||||
**{key: e.get(key) for key in (
|
||||
"botApiMessageId", "textLen", "contentType", "senderId", "isSut",
|
||||
"isOutgoing", "replyToMessageId", "quoteText", "topicType", "topicId",
|
||||
"reactionText", "reactionCount", "actionType", "status", "buttonText",
|
||||
"durationMs", "error",
|
||||
)},
|
||||
}
|
||||
for e in self.events
|
||||
],
|
||||
|
|
|
|||
|
|
@ -215,24 +215,21 @@ function isTooLargeError(error: unknown): boolean {
|
|||
return (error as NodeJS.ErrnoException | undefined)?.code === "ETOOBIG";
|
||||
}
|
||||
|
||||
async function readDiscordResponseText(params: {
|
||||
type DiscordResponseRead = {
|
||||
response: Response;
|
||||
label: string;
|
||||
signal: AbortSignal;
|
||||
maxBytes: number;
|
||||
}): Promise<string> {
|
||||
};
|
||||
|
||||
async function readDiscordResponseText(params: DiscordResponseRead): Promise<string> {
|
||||
return await readBoundedResponseText(params.response, params.label, params.maxBytes, {
|
||||
createTooLargeError: createDiscordResponseTooLargeError,
|
||||
signal: params.signal,
|
||||
});
|
||||
}
|
||||
|
||||
async function readDiscordResponseJson(params: {
|
||||
response: Response;
|
||||
label: string;
|
||||
signal: AbortSignal;
|
||||
maxBytes: number;
|
||||
}): Promise<unknown> {
|
||||
async function readDiscordResponseJson(params: DiscordResponseRead): Promise<unknown> {
|
||||
const text = await readDiscordResponseText(params);
|
||||
if (!text) {
|
||||
return {};
|
||||
|
|
@ -361,79 +358,32 @@ function usage(): string {
|
|||
|
||||
function parseArgs(argv = process.argv.slice(2)): Args {
|
||||
validateCliArgs(argv);
|
||||
const channelId =
|
||||
resolveArg("--channel", argv) || process.env.OPENCLAW_DISCORD_SMOKE_CHANNEL_ID || "";
|
||||
const driverModeRaw =
|
||||
resolveArg("--driver", argv) || process.env.OPENCLAW_DISCORD_SMOKE_DRIVER || "token";
|
||||
const driverMode = parseDriverMode(driverModeRaw);
|
||||
const driverToken =
|
||||
resolveArg("--token", argv) || process.env.OPENCLAW_DISCORD_SMOKE_DRIVER_TOKEN || "";
|
||||
const driverTokenPrefix =
|
||||
resolveArg("--token-prefix", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_DRIVER_TOKEN_PREFIX ||
|
||||
"Bot";
|
||||
const botToken =
|
||||
resolveArg("--bot-token", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_BOT_TOKEN ||
|
||||
process.env.DISCORD_BOT_TOKEN ||
|
||||
"";
|
||||
const botTokenPrefix =
|
||||
resolveArg("--bot-token-prefix", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_BOT_TOKEN_PREFIX ||
|
||||
"Bot";
|
||||
const targetAgent =
|
||||
resolveArg("--agent", argv) || process.env.OPENCLAW_DISCORD_SMOKE_AGENT || "codex";
|
||||
const mentionUserId =
|
||||
resolveArg("--mention", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_MENTION_USER_ID ||
|
||||
undefined;
|
||||
const instruction =
|
||||
resolveArg("--instruction", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_INSTRUCTION ||
|
||||
undefined;
|
||||
const timeoutMs = parseNumber(
|
||||
resolveArg("--timeout-ms", argv) || process.env.OPENCLAW_DISCORD_SMOKE_TIMEOUT_MS,
|
||||
240_000,
|
||||
"--timeout-ms",
|
||||
);
|
||||
const pollMs = parseNumber(
|
||||
resolveArg("--poll-ms", argv) || process.env.OPENCLAW_DISCORD_SMOKE_POLL_MS,
|
||||
1_500,
|
||||
"--poll-ms",
|
||||
);
|
||||
const stateDir = path.resolve(resolveArg("--state-dir", argv) || resolveStateDir());
|
||||
const openclawBin =
|
||||
resolveArg("--openclaw-bin", argv) ||
|
||||
process.env.OPENCLAW_DISCORD_SMOKE_OPENCLAW_BIN ||
|
||||
"openclaw";
|
||||
const json = hasFlag("--json", argv);
|
||||
|
||||
if (!channelId) {
|
||||
throw new Error(usage());
|
||||
}
|
||||
if (driverMode === "token" && !driverToken) {
|
||||
throw new Error(usage());
|
||||
}
|
||||
if (driverMode === "webhook" && !botToken) {
|
||||
throw new Error(usage());
|
||||
}
|
||||
|
||||
return {
|
||||
channelId,
|
||||
driverMode,
|
||||
driverToken,
|
||||
driverTokenPrefix,
|
||||
botToken,
|
||||
botTokenPrefix,
|
||||
targetAgent,
|
||||
timeoutMs,
|
||||
pollMs,
|
||||
mentionUserId,
|
||||
instruction,
|
||||
stateDir,
|
||||
openclawBin,
|
||||
json,
|
||||
const option = (flag: string, env: string, fallback = "") =>
|
||||
resolveArg(flag, argv) || process.env[`OPENCLAW_DISCORD_SMOKE_${env}`] || fallback;
|
||||
const args: Args = {
|
||||
channelId: option("--channel", "CHANNEL_ID"),
|
||||
driverMode: parseDriverMode(option("--driver", "DRIVER", "token")),
|
||||
driverToken: option("--token", "DRIVER_TOKEN"),
|
||||
driverTokenPrefix: option("--token-prefix", "DRIVER_TOKEN_PREFIX", "Bot"),
|
||||
botToken: option("--bot-token", "BOT_TOKEN", process.env.DISCORD_BOT_TOKEN || ""),
|
||||
botTokenPrefix: option("--bot-token-prefix", "BOT_TOKEN_PREFIX", "Bot"),
|
||||
targetAgent: option("--agent", "AGENT", "codex"),
|
||||
mentionUserId: option("--mention", "MENTION_USER_ID") || undefined,
|
||||
instruction: option("--instruction", "INSTRUCTION") || undefined,
|
||||
timeoutMs: parseNumber(option("--timeout-ms", "TIMEOUT_MS"), 240_000, "--timeout-ms"),
|
||||
pollMs: parseNumber(option("--poll-ms", "POLL_MS"), 1_500, "--poll-ms"),
|
||||
stateDir: path.resolve(resolveArg("--state-dir", argv) || resolveStateDir()),
|
||||
openclawBin: option("--openclaw-bin", "OPENCLAW_BIN", "openclaw"),
|
||||
json: hasFlag("--json", argv),
|
||||
};
|
||||
if (
|
||||
!args.channelId ||
|
||||
(args.driverMode === "token" && !args.driverToken) ||
|
||||
(args.driverMode === "webhook" && !args.botToken)
|
||||
) {
|
||||
throw new Error(usage());
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
async function openclawCliJson<T>(params: {
|
||||
|
|
@ -575,25 +525,30 @@ async function requestDiscordJson<T>(params: {
|
|||
timeoutError,
|
||||
onTimeout: () => controller.abort(),
|
||||
});
|
||||
|
||||
if (response.status === 429) {
|
||||
const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError);
|
||||
const body = (await withTimeout({
|
||||
operation: readDiscordResponseJson({
|
||||
const readBody = <R>(reader: (input: DiscordResponseRead) => Promise<R>): Promise<R> => {
|
||||
const timeoutMs = remainingTimeoutMs(deadlineMs, timeoutError);
|
||||
return withTimeout({
|
||||
operation: reader({
|
||||
response,
|
||||
label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`,
|
||||
signal: controller.signal,
|
||||
maxBytes: responseBodyMaxBytes,
|
||||
}).catch((error: unknown) => {
|
||||
}),
|
||||
timeoutMs,
|
||||
timeoutError,
|
||||
onTimeout: () => controller.abort(),
|
||||
});
|
||||
};
|
||||
|
||||
if (response.status === 429) {
|
||||
const body = (await readBody((input) =>
|
||||
readDiscordResponseJson(input).catch((error: unknown) => {
|
||||
if (isTooLargeError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return {};
|
||||
}),
|
||||
timeoutMs: bodyTimeoutMs,
|
||||
timeoutError,
|
||||
onTimeout: () => controller.abort(),
|
||||
})) as { retry_after?: number };
|
||||
)) as { retry_after?: number };
|
||||
const waitSeconds = typeof body.retry_after === "number" ? body.retry_after : 1;
|
||||
const waitMs = Math.ceil(waitSeconds * 1000);
|
||||
const remainingMs = remainingTimeoutMs(deadlineMs, timeoutError);
|
||||
|
|
@ -607,18 +562,7 @@ async function requestDiscordJson<T>(params: {
|
|||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError);
|
||||
const text = await withTimeout({
|
||||
operation: readDiscordResponseText({
|
||||
response,
|
||||
label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`,
|
||||
signal: controller.signal,
|
||||
maxBytes: responseBodyMaxBytes,
|
||||
}),
|
||||
timeoutMs: bodyTimeoutMs,
|
||||
timeoutError,
|
||||
onTimeout: () => controller.abort(),
|
||||
});
|
||||
const text = await readBody(readDiscordResponseText);
|
||||
throw new Error(
|
||||
redactForDevToolLog(
|
||||
`${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)} failed: ${response.status} ${response.statusText}${text ? ` :: ${text}` : ""}`,
|
||||
|
|
@ -630,18 +574,7 @@ async function requestDiscordJson<T>(params: {
|
|||
return undefined as T;
|
||||
}
|
||||
|
||||
const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError);
|
||||
return (await withTimeout({
|
||||
operation: readDiscordResponseJson({
|
||||
response,
|
||||
label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`,
|
||||
signal: controller.signal,
|
||||
maxBytes: responseBodyMaxBytes,
|
||||
}),
|
||||
timeoutMs: bodyTimeoutMs,
|
||||
timeoutError,
|
||||
onTimeout: () => controller.abort(),
|
||||
})) as T;
|
||||
return (await readBody(readDiscordResponseJson)) as T;
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
|
|
@ -706,22 +639,24 @@ function toRecentMessageRow(message: DiscordMessage) {
|
|||
};
|
||||
}
|
||||
|
||||
async function loadParentRecentMessages(params: {
|
||||
async function loadMessages(params: {
|
||||
args: Args;
|
||||
readAuthHeader: string;
|
||||
target: string;
|
||||
limit: number;
|
||||
timeoutMs?: number;
|
||||
}): Promise<DiscordMessage[]> {
|
||||
if (params.args.driverMode === "openclaw") {
|
||||
return await readMessagesWithOpenclaw({
|
||||
openclawBin: params.args.openclawBin,
|
||||
target: params.args.channelId,
|
||||
limit: 20,
|
||||
target: params.target,
|
||||
limit: params.limit,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
}
|
||||
return await discordApi<DiscordMessage[]>({
|
||||
method: "GET",
|
||||
path: `/channels/${encodeURIComponent(params.args.channelId)}/messages?limit=20`,
|
||||
path: `/channels/${encodeURIComponent(params.target)}/messages?limit=${params.limit}`,
|
||||
authHeader: params.readAuthHeader,
|
||||
timeoutMs: params.timeoutMs,
|
||||
});
|
||||
|
|
@ -941,6 +876,31 @@ async function run(argv = process.argv.slice(2)): Promise<SuccessResult | Failur
|
|||
};
|
||||
}
|
||||
|
||||
const diagnostics = async (candidates: ThreadBindingRecord[]) => {
|
||||
let parentRecent: DiscordMessage[] = [];
|
||||
try {
|
||||
parentRecent = await loadMessages({
|
||||
args,
|
||||
readAuthHeader,
|
||||
target: args.channelId,
|
||||
limit: 20,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
});
|
||||
} catch {
|
||||
// Best effort diagnostics only.
|
||||
}
|
||||
return {
|
||||
bindingCandidates: candidates.map((entry) => ({
|
||||
threadId: entry.threadId || "",
|
||||
targetSessionKey: maskIdentifier(entry.targetSessionKey),
|
||||
targetKind: entry.targetKind,
|
||||
agentId: entry.agentId,
|
||||
boundAt: entry.boundAt,
|
||||
})),
|
||||
parentChannelRecent: parentRecent.map(toRecentMessageRow),
|
||||
};
|
||||
};
|
||||
|
||||
let winningBinding: ThreadBindingRecord | undefined;
|
||||
let latestCandidates: ThreadBindingRecord[] = [];
|
||||
|
||||
|
|
@ -965,31 +925,12 @@ async function run(argv = process.argv.slice(2)): Promise<SuccessResult | Failur
|
|||
}
|
||||
|
||||
if (!winningBinding?.threadId || !winningBinding?.targetSessionKey) {
|
||||
let parentRecent: DiscordMessage[] = [];
|
||||
try {
|
||||
parentRecent = await loadParentRecentMessages({
|
||||
args,
|
||||
readAuthHeader,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
});
|
||||
} catch {
|
||||
// Best effort diagnostics only.
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
stage: "wait-binding",
|
||||
smokeId,
|
||||
error: `Timed out waiting for new ACP thread binding (state: ${redactHomePath(args.stateDir)}).`,
|
||||
diagnostics: {
|
||||
bindingCandidates: latestCandidates.slice(0, 6).map((entry) => ({
|
||||
threadId: entry.threadId || "",
|
||||
targetSessionKey: maskIdentifier(entry.targetSessionKey),
|
||||
targetKind: entry.targetKind,
|
||||
agentId: entry.agentId,
|
||||
boundAt: entry.boundAt,
|
||||
})),
|
||||
parentChannelRecent: parentRecent.map(toRecentMessageRow),
|
||||
},
|
||||
diagnostics: await diagnostics(latestCandidates.slice(0, 6)),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -997,20 +938,13 @@ async function run(argv = process.argv.slice(2)): Promise<SuccessResult | Failur
|
|||
let ackMessage: DiscordMessage | undefined;
|
||||
while (Date.now() < deadline && !ackMessage) {
|
||||
try {
|
||||
const threadMessages =
|
||||
args.driverMode === "openclaw"
|
||||
? await readMessagesWithOpenclaw({
|
||||
openclawBin: args.openclawBin,
|
||||
target: threadId,
|
||||
limit: 50,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
})
|
||||
: await discordApi<DiscordMessage[]>({
|
||||
method: "GET",
|
||||
path: `/channels/${encodeURIComponent(threadId)}/messages?limit=50`,
|
||||
authHeader: readAuthHeader,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
});
|
||||
const threadMessages = await loadMessages({
|
||||
args,
|
||||
readAuthHeader,
|
||||
target: threadId,
|
||||
limit: 50,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
});
|
||||
ackMessage = threadMessages.find((message) => {
|
||||
const content = message.content || "";
|
||||
if (!content.includes(ackToken)) {
|
||||
|
|
@ -1028,34 +962,12 @@ async function run(argv = process.argv.slice(2)): Promise<SuccessResult | Failur
|
|||
}
|
||||
|
||||
if (!ackMessage) {
|
||||
let parentRecent: DiscordMessage[] = [];
|
||||
try {
|
||||
parentRecent = await loadParentRecentMessages({
|
||||
args,
|
||||
readAuthHeader,
|
||||
timeoutMs: remainingTimeoutMs(deadline),
|
||||
});
|
||||
} catch {
|
||||
// Best effort diagnostics only.
|
||||
}
|
||||
|
||||
return {
|
||||
ok: false,
|
||||
stage: "wait-ack",
|
||||
smokeId,
|
||||
error: `Thread bound (${threadId}) but timed out waiting for ACK token "${ackToken}" from OpenClaw.`,
|
||||
diagnostics: {
|
||||
bindingCandidates: [
|
||||
{
|
||||
threadId: winningBinding.threadId || "",
|
||||
targetSessionKey: maskIdentifier(winningBinding.targetSessionKey),
|
||||
targetKind: winningBinding.targetKind,
|
||||
agentId: winningBinding.agentId,
|
||||
boundAt: winningBinding.boundAt,
|
||||
},
|
||||
],
|
||||
parentChannelRecent: parentRecent.map(toRecentMessageRow),
|
||||
},
|
||||
diagnostics: await diagnostics([winningBinding]),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,11 @@ import {
|
|||
MIN_CLIENT_PROTOCOL_VERSION,
|
||||
PROTOCOL_VERSION,
|
||||
} from "../../packages/gateway-protocol/src/version.js";
|
||||
import { createArgReader, createGatewayWsClient, resolveGatewayUrl } from "./gateway-ws-client.ts";
|
||||
import {
|
||||
createArgReader,
|
||||
createGatewayWsClient,
|
||||
resolveGatewayUrl,
|
||||
} from "../lib/gateway-ws-client.ts";
|
||||
|
||||
function writeStdoutLine(message: string): void {
|
||||
process.stdout.write(`${message}\n`);
|
||||
|
|
@ -15,11 +19,6 @@ function writeStderrLine(message: string): void {
|
|||
process.stderr.write(`${message}\n`);
|
||||
}
|
||||
|
||||
function writeUsage(): void {
|
||||
writeStderrLine(usage());
|
||||
}
|
||||
|
||||
type GatewaySmokeClient = ReturnType<typeof createGatewayWsClient>;
|
||||
type GatewaySmokeCliOptions = {
|
||||
help: boolean;
|
||||
token?: string;
|
||||
|
|
@ -131,18 +130,6 @@ function connectHelloScopes(response: unknown): string[] | null {
|
|||
return payload.auth.scopes;
|
||||
}
|
||||
|
||||
function hasConnectHelloPayload(response: unknown): boolean {
|
||||
return connectHelloScopes(response) !== null;
|
||||
}
|
||||
|
||||
function hasUnpairedOperatorScopes(response: unknown): boolean {
|
||||
const scopes = connectHelloScopes(response);
|
||||
if (!scopes) {
|
||||
return false;
|
||||
}
|
||||
return scopes.length > 0;
|
||||
}
|
||||
|
||||
export async function runGatewaySmoke(
|
||||
input: { token: string; urlRaw: string },
|
||||
deps: GatewaySmokeDeps = {},
|
||||
|
|
@ -151,13 +138,7 @@ export async function runGatewaySmoke(
|
|||
const createClient = deps.createClient ?? createGatewayWsClient;
|
||||
const stderr = deps.stderr ?? writeStderrLine;
|
||||
const stdout = deps.stdout ?? writeStdoutLine;
|
||||
const client: GatewaySmokeClient = createClient({
|
||||
url: url.toString(),
|
||||
onEvent: (evt) => {
|
||||
// Ignore noisy connect handshakes.
|
||||
void evt;
|
||||
},
|
||||
});
|
||||
const client = createClient({ url: url.toString() });
|
||||
const { request, waitOpen, close } = client;
|
||||
|
||||
try {
|
||||
|
|
@ -187,11 +168,12 @@ export async function runGatewaySmoke(
|
|||
stderr(`connect failed: ${String(connectRes.error)}`);
|
||||
return 2;
|
||||
}
|
||||
if (!hasConnectHelloPayload(connectRes)) {
|
||||
const scopes = connectHelloScopes(connectRes);
|
||||
if (scopes === null) {
|
||||
stderr("connect failed: missing hello-ok payload");
|
||||
return 2;
|
||||
}
|
||||
if (hasUnpairedOperatorScopes(connectRes)) {
|
||||
if (scopes.length > 0) {
|
||||
stderr("connect failed: unpaired iOS smoke unexpectedly received operator scopes");
|
||||
return 2;
|
||||
}
|
||||
|
|
@ -225,7 +207,7 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
|
|||
if (cli.help) {
|
||||
writeStdoutLine(usage());
|
||||
} else if (!cli.urlRaw || !cli.token) {
|
||||
writeUsage();
|
||||
writeStderrLine(usage());
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
process.exitCode = await runGatewaySmoke({ token: cli.token, urlRaw: cli.urlRaw });
|
||||
|
|
|
|||
|
|
@ -1,2 +0,0 @@
|
|||
// Gateway Ws Client script supports OpenClaw repository automation.
|
||||
export * from "../lib/gateway-ws-client.ts";
|
||||
|
|
@ -101,14 +101,13 @@ if (!urlRaw || !token) {
|
|||
}
|
||||
|
||||
const waitSeconds = parseWaitSeconds(getArg("--wait-seconds"));
|
||||
const { createGatewayWsClient, resolveGatewayUrl } = await import("./gateway-ws-client.ts");
|
||||
const { createGatewayWsClient, resolveGatewayUrl } = await import("../lib/gateway-ws-client.ts");
|
||||
const url = resolveGatewayUrl(urlRaw);
|
||||
|
||||
const isoNow = () => new Date().toISOString();
|
||||
const isoMinusMs = (ms: number) => new Date(Date.now() - ms).toISOString();
|
||||
|
||||
type TestCase = {
|
||||
id: string;
|
||||
command: string;
|
||||
params?: unknown;
|
||||
timeoutMs?: number;
|
||||
|
|
@ -273,47 +272,39 @@ async function main() {
|
|||
}
|
||||
|
||||
const tests: TestCase[] = [
|
||||
{ id: "device.info", command: "device.info" },
|
||||
{ id: "device.status", command: "device.status" },
|
||||
{ command: "device.info" },
|
||||
{ command: "device.status" },
|
||||
{
|
||||
id: "system.notify",
|
||||
command: "system.notify",
|
||||
params: { title: "OpenClaw E2E", body: `ios-node-e2e @ ${isoNow()}`, delivery: "system" },
|
||||
},
|
||||
{
|
||||
id: "contacts.search",
|
||||
command: "contacts.search",
|
||||
params: { query: null, limit: 5 },
|
||||
},
|
||||
{
|
||||
id: "calendar.events",
|
||||
command: "calendar.events",
|
||||
params: { startISO: isoMinusMs(6 * 60 * 60 * 1000), endISO: isoNow(), limit: 10 },
|
||||
},
|
||||
{
|
||||
id: "reminders.list",
|
||||
command: "reminders.list",
|
||||
params: { status: "incomplete", limit: 10 },
|
||||
},
|
||||
{
|
||||
id: "motion.pedometer",
|
||||
command: "motion.pedometer",
|
||||
params: { startISO: isoMinusMs(60 * 60 * 1000), endISO: isoNow() },
|
||||
},
|
||||
{
|
||||
id: "photos.latest",
|
||||
command: "photos.latest",
|
||||
params: { limit: 1, maxWidth: 512, quality: 0.7 },
|
||||
},
|
||||
{
|
||||
id: "camera.snap",
|
||||
command: "camera.snap",
|
||||
params: { facing: "back", maxWidth: 768, quality: 0.7, format: "jpeg" },
|
||||
dangerous: true,
|
||||
timeoutMs: 20_000,
|
||||
},
|
||||
{
|
||||
id: "screen.record",
|
||||
command: "screen.record",
|
||||
params: { durationMs: 2_000, fps: 15, includeAudio: false },
|
||||
dangerous: true,
|
||||
|
|
@ -342,7 +333,7 @@ async function main() {
|
|||
},
|
||||
(t.timeoutMs ?? 12_000) + 2_000,
|
||||
).catch((err: unknown) => {
|
||||
results.push({ id: t.id, ok: false, error: formatErr(err) });
|
||||
results.push({ id: t.command, ok: false, error: formatErr(err) });
|
||||
return null;
|
||||
});
|
||||
|
||||
|
|
@ -351,18 +342,18 @@ async function main() {
|
|||
}
|
||||
|
||||
if (!invokeRes.ok) {
|
||||
results.push({ id: t.id, ok: false, error: invokeRes.error });
|
||||
results.push({ id: t.command, ok: false, error: invokeRes.error });
|
||||
continue;
|
||||
}
|
||||
|
||||
const commandPayload = commandPayloadFromInvokePayload(invokeRes.payload);
|
||||
const payloadError = payloadShapeError(t.command, commandPayload);
|
||||
if (payloadError) {
|
||||
results.push({ id: t.id, ok: false, error: payloadError, payload: invokeRes.payload });
|
||||
results.push({ id: t.command, ok: false, error: payloadError, payload: invokeRes.payload });
|
||||
continue;
|
||||
}
|
||||
|
||||
results.push({ id: t.id, ok: true, payload: invokeRes.payload });
|
||||
results.push({ id: t.command, ok: true, payload: invokeRes.payload });
|
||||
}
|
||||
|
||||
if (jsonOut) {
|
||||
|
|
@ -377,7 +368,6 @@ async function main() {
|
|||
results,
|
||||
});
|
||||
} else {
|
||||
const pad = (s: string, n: number) => (s.length >= n ? s : s + " ".repeat(n - s.length));
|
||||
const rows = results.map((r) => ({
|
||||
cmd: r.id,
|
||||
ok: r.ok ? "ok" : "fail",
|
||||
|
|
@ -388,7 +378,7 @@ async function main() {
|
|||
writeStdoutLine(`dangerous: ${dangerous ? "on" : "off"}`);
|
||||
writeStdoutLine();
|
||||
for (const r of rows) {
|
||||
writeStdoutLine(`${pad(r.cmd, width)} ${pad(r.ok, 4)} ${r.note}`);
|
||||
writeStdoutLine(`${r.cmd.padEnd(width)} ${r.ok.padEnd(4)} ${r.note}`);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import {
|
|||
previewForDevToolLog,
|
||||
redactJsonValueForDevToolLog,
|
||||
} from "../lib/dev-tooling-safety.ts";
|
||||
import { toErrorObject as toLintErrorObject } from "../lib/error-format.mts";
|
||||
|
||||
const OPENAI_REALTIME_MODEL =
|
||||
process.env.OPENCLAW_REALTIME_OPENAI_MODEL?.trim() || "gpt-realtime-2.1";
|
||||
|
|
@ -971,7 +970,18 @@ async function smokeGoogleLiveBrowserWs(browser: Browser, apiKey: string): Promi
|
|||
}
|
||||
})().catch((error: unknown) => {
|
||||
window.clearTimeout(timeout);
|
||||
reject(toLintErrorObject(error, "Non-Error rejection"));
|
||||
// This callback is serialized into the browser without module imports.
|
||||
if (error instanceof Error) {
|
||||
reject(error);
|
||||
} else if (typeof error === "string") {
|
||||
reject(new Error(error));
|
||||
} else {
|
||||
const failure = new Error("Non-Error rejection", { cause: error });
|
||||
if ((typeof error === "object" && error !== null) || typeof error === "function") {
|
||||
Object.assign(failure, error);
|
||||
}
|
||||
reject(failure);
|
||||
}
|
||||
});
|
||||
});
|
||||
ws.addEventListener("error", () => {
|
||||
|
|
|
|||
|
|
@ -178,31 +178,19 @@ time_phase "Preinstall previous" preinstall_previous_version
|
|||
time_phase "Run official installer one-liner" run_official_installer
|
||||
time_phase "Verify installed version" verify_installed_version
|
||||
|
||||
set_image_model() {
|
||||
set_profile_model() {
|
||||
local profile="$1"
|
||||
shift
|
||||
local command="$2"
|
||||
local label="$3"
|
||||
shift 3
|
||||
local candidate
|
||||
for candidate in "$@"; do
|
||||
if openclaw --profile "$profile" models set-image "$candidate" >/dev/null 2>&1; then
|
||||
if openclaw --profile "$profile" models "$command" "$candidate" >/dev/null 2>&1; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "ERROR: could not set an image model (tried: $*)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
set_agent_model() {
|
||||
local profile="$1"
|
||||
local candidate
|
||||
shift
|
||||
for candidate in "$@"; do
|
||||
if openclaw --profile "$profile" models set "$candidate" >/dev/null 2>&1; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
echo "ERROR: could not set agent model (tried: $*)" >&2
|
||||
echo "ERROR: could not set $label model (tried: $*)" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
|
|
@ -419,14 +407,9 @@ run_agent_turn_logged_or_skip_profile() {
|
|||
}
|
||||
|
||||
run_agent_turn_bg() {
|
||||
local label="$1"
|
||||
local profile="$2"
|
||||
local session_id="$3"
|
||||
local prompt="$4"
|
||||
local out_json="$5"
|
||||
(
|
||||
set -euo pipefail
|
||||
run_agent_turn_logged "$label" "$profile" "$session_id" "$prompt" "$out_json"
|
||||
run_agent_turn_logged "$@"
|
||||
) &
|
||||
RUN_AGENT_TURN_BG_PID="$!"
|
||||
}
|
||||
|
|
@ -795,56 +778,24 @@ run_profile() {
|
|||
CURRENT_AGENT_MODEL_PROVIDER="$agent_model_provider"
|
||||
|
||||
phase_mark_start "Onboard ($profile)"
|
||||
if [[ "$agent_model_provider" == "openai" ]]; then
|
||||
openclaw --profile "$profile" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--auth-choice openai-api-key \
|
||||
--openai-api-key "$OPENAI_API_KEY" \
|
||||
--gateway-port "$port" \
|
||||
--gateway-bind loopback \
|
||||
--gateway-auth token \
|
||||
--workspace "$workspace" \
|
||||
--skip-health
|
||||
elif [[ -n "$ANTHROPIC_API_KEY" ]]; then
|
||||
openclaw --profile "$profile" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--auth-choice apiKey \
|
||||
--anthropic-api-key "$ANTHROPIC_API_KEY" \
|
||||
--gateway-port "$port" \
|
||||
--gateway-bind loopback \
|
||||
--gateway-auth token \
|
||||
--workspace "$workspace" \
|
||||
--skip-health
|
||||
elif [[ -n "$ANTHROPIC_API_TOKEN" ]]; then
|
||||
openclaw --profile "$profile" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--auth-choice token \
|
||||
--token-provider anthropic \
|
||||
--token "$ANTHROPIC_API_TOKEN" \
|
||||
--gateway-port "$port" \
|
||||
--gateway-bind loopback \
|
||||
--gateway-auth token \
|
||||
--workspace "$workspace" \
|
||||
--skip-health
|
||||
else
|
||||
openclaw --profile "$profile" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
--auth-choice apiKey \
|
||||
--anthropic-api-key "$ANTHROPIC_API_KEY" \
|
||||
--gateway-port "$port" \
|
||||
--gateway-bind loopback \
|
||||
--gateway-auth token \
|
||||
--workspace "$workspace" \
|
||||
--skip-health
|
||||
local auth_args=()
|
||||
if [[ "$agent_model_provider" == "openai" ]]; then
|
||||
auth_args=(--auth-choice openai-api-key --openai-api-key "$OPENAI_API_KEY")
|
||||
elif [[ -z "$ANTHROPIC_API_KEY" && -n "$ANTHROPIC_API_TOKEN" ]]; then
|
||||
auth_args=(--auth-choice token --token-provider anthropic --token "$ANTHROPIC_API_TOKEN")
|
||||
else
|
||||
auth_args=(--auth-choice apiKey --anthropic-api-key "$ANTHROPIC_API_KEY")
|
||||
fi
|
||||
openclaw --profile "$profile" onboard \
|
||||
--non-interactive \
|
||||
--accept-risk \
|
||||
--flow quickstart \
|
||||
"${auth_args[@]}" \
|
||||
--gateway-port "$port" \
|
||||
--gateway-bind loopback \
|
||||
--gateway-auth token \
|
||||
--workspace "$workspace" \
|
||||
--skip-health
|
||||
phase_mark_passed "Onboard ($profile)"
|
||||
|
||||
phase_mark_start "Verify workspace identity files ($profile)"
|
||||
|
|
@ -862,18 +813,18 @@ run_profile() {
|
|||
local agent_model
|
||||
local image_model
|
||||
if [[ "$agent_model_provider" == "openai" ]]; then
|
||||
agent_model="$(set_agent_model "$profile" \
|
||||
agent_model="$(set_profile_model "$profile" set agent \
|
||||
"$OPENAI_AGENT_MODEL" \
|
||||
"openai/gpt-5.5" \
|
||||
"openai/gpt-5.4-mini")"
|
||||
openclaw --profile "$profile" config set models.providers.openai "{\"baseUrl\":\"https://api.openai.com/v1\",\"models\":[],\"timeoutSeconds\":${OPENAI_PROVIDER_TIMEOUT_SECONDS},\"agentRuntime\":{\"id\":\"openclaw\"}}" --strict-json >/dev/null
|
||||
image_model="$(set_image_model "$profile" \
|
||||
image_model="$(set_profile_model "$profile" set-image "an image" \
|
||||
"openai/gpt-5.4-image-2")"
|
||||
else
|
||||
agent_model="$(set_agent_model "$profile" \
|
||||
agent_model="$(set_profile_model "$profile" set agent \
|
||||
"anthropic/claude-opus-4-6" \
|
||||
"claude-opus-4-6")"
|
||||
image_model="$(set_image_model "$profile" \
|
||||
image_model="$(set_profile_model "$profile" set-image "an image" \
|
||||
"anthropic/claude-opus-4-6" \
|
||||
"claude-opus-4-6")"
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -50,12 +50,6 @@ require_cmd() {
|
|||
fi
|
||||
}
|
||||
|
||||
run_docker_build() {
|
||||
# Dockerfile uses BuildKit-only syntax (RUN --mount=type=cache). Force
|
||||
# BuildKit so hosts defaulting to the legacy builder do not fail.
|
||||
docker_build_exec "$@"
|
||||
}
|
||||
|
||||
run_docker_pull() {
|
||||
local image="$1"
|
||||
openclaw_host_timeout_cmd "$DOCKER_PULL_TIMEOUT" docker pull "$image"
|
||||
|
|
@ -598,53 +592,35 @@ COMPOSE_ARGS=()
|
|||
write_extra_compose() {
|
||||
local home_volume="$1"
|
||||
shift
|
||||
local mount
|
||||
local gateway_home_mount
|
||||
local gateway_config_mount
|
||||
local gateway_workspace_mount
|
||||
local gateway_auth_profile_secret_mount
|
||||
|
||||
cat >"$EXTRA_COMPOSE_FILE" <<'YAML'
|
||||
services:
|
||||
openclaw-gateway:
|
||||
volumes:
|
||||
YAML
|
||||
local mount service
|
||||
local home_mounts=()
|
||||
|
||||
if [[ -n "$home_volume" ]]; then
|
||||
gateway_home_mount="${home_volume}:/home/node"
|
||||
gateway_config_mount="${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw"
|
||||
gateway_workspace_mount="${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace"
|
||||
gateway_auth_profile_secret_mount="${OPENCLAW_AUTH_PROFILE_SECRET_DIR}:/home/node/.config/openclaw"
|
||||
validate_mount_spec "$gateway_home_mount"
|
||||
validate_mount_spec "$gateway_config_mount"
|
||||
validate_mount_spec "$gateway_workspace_mount"
|
||||
validate_mount_spec "$gateway_auth_profile_secret_mount"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_home_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_config_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_workspace_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_auth_profile_secret_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
home_mounts=(
|
||||
"${home_volume}:/home/node"
|
||||
"${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw"
|
||||
"${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace"
|
||||
"${OPENCLAW_AUTH_PROFILE_SECRET_DIR}:/home/node/.config/openclaw"
|
||||
)
|
||||
fi
|
||||
|
||||
for mount in "$@"; do
|
||||
validate_mount_spec "$mount"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
done
|
||||
|
||||
cat >>"$EXTRA_COMPOSE_FILE" <<'YAML'
|
||||
openclaw-cli:
|
||||
volumes:
|
||||
YAML
|
||||
|
||||
if [[ -n "$home_volume" ]]; then
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_home_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_config_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_workspace_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$gateway_auth_profile_secret_mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
fi
|
||||
|
||||
for mount in "$@"; do
|
||||
validate_mount_spec "$mount"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
printf 'services:\n' >"$EXTRA_COMPOSE_FILE"
|
||||
for service in openclaw-gateway openclaw-cli; do
|
||||
printf ' %s:\n volumes:\n' "$service" >>"$EXTRA_COMPOSE_FILE"
|
||||
if [[ -n "$home_volume" ]]; then
|
||||
if [[ "$service" == "openclaw-gateway" ]]; then
|
||||
for mount in "${home_mounts[@]}"; do
|
||||
validate_mount_spec "$mount"
|
||||
done
|
||||
fi
|
||||
for mount in "${home_mounts[@]}"; do
|
||||
printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
done
|
||||
fi
|
||||
for mount in "$@"; do
|
||||
validate_mount_spec "$mount"
|
||||
printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE"
|
||||
done
|
||||
done
|
||||
|
||||
if [[ -n "$home_volume" && "$home_volume" != *"/"* ]]; then
|
||||
|
|
@ -783,7 +759,7 @@ elif [[ "$IMAGE_NAME" == "openclaw:local" ]]; then
|
|||
if [[ "$BUILD_GIT_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]]; then
|
||||
PROVENANCE_BUILD_ARGS+=(--build-arg "GIT_COMMIT=${BUILD_GIT_COMMIT}")
|
||||
fi
|
||||
run_docker_build \
|
||||
docker_build_exec \
|
||||
"${PROVENANCE_BUILD_ARGS[@]}" \
|
||||
--build-arg "OPENCLAW_IMAGE_APT_PACKAGES=${OPENCLAW_IMAGE_APT_PACKAGES}" \
|
||||
--build-arg "OPENCLAW_IMAGE_PIP_PACKAGES=${OPENCLAW_IMAGE_PIP_PACKAGES}" \
|
||||
|
|
@ -898,7 +874,7 @@ if [[ -n "$SANDBOX_ENABLED" ]]; then
|
|||
|
||||
if [[ -n "$SANDBOX_ENABLED" && -z "$OFFLINE_MODE" && -f "$sandbox_dockerfile" ]]; then
|
||||
echo "Building sandbox image: $DEFAULT_SANDBOX_IMAGE"
|
||||
run_docker_build \
|
||||
docker_build_exec \
|
||||
-t "$DEFAULT_SANDBOX_IMAGE" \
|
||||
-f "$sandbox_dockerfile" \
|
||||
"$ROOT_DIR"
|
||||
|
|
@ -945,21 +921,14 @@ fi
|
|||
if [[ -n "$SANDBOX_ENABLED" ]]; then
|
||||
# Enable sandbox in OpenClaw config.
|
||||
sandbox_config_ok=true
|
||||
if ! run_runtime_cli current no-deps \
|
||||
config set agents.defaults.sandbox.mode "non-main" >/dev/null; then
|
||||
echo "WARNING: Failed to set agents.defaults.sandbox.mode" >&2
|
||||
sandbox_config_ok=false
|
||||
fi
|
||||
if ! run_runtime_cli current no-deps \
|
||||
config set agents.defaults.sandbox.scope "agent" >/dev/null; then
|
||||
echo "WARNING: Failed to set agents.defaults.sandbox.scope" >&2
|
||||
sandbox_config_ok=false
|
||||
fi
|
||||
if ! run_runtime_cli current no-deps \
|
||||
config set agents.defaults.sandbox.workspaceAccess "none" >/dev/null; then
|
||||
echo "WARNING: Failed to set agents.defaults.sandbox.workspaceAccess" >&2
|
||||
sandbox_config_ok=false
|
||||
fi
|
||||
for sandbox_setting in mode:non-main scope:agent workspaceAccess:none; do
|
||||
sandbox_path="agents.defaults.sandbox.${sandbox_setting%%:*}"
|
||||
if ! run_runtime_cli current no-deps \
|
||||
config set "$sandbox_path" "${sandbox_setting#*:}" >/dev/null; then
|
||||
echo "WARNING: Failed to set $sandbox_path" >&2
|
||||
sandbox_config_ok=false
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ "$sandbox_config_ok" == true ]]; then
|
||||
echo "Sandbox enabled: mode=non-main, scope=agent, workspaceAccess=none"
|
||||
|
|
|
|||
|
|
@ -349,19 +349,11 @@ load_artifact() {
|
|||
}
|
||||
trap cleanup_load EXIT
|
||||
|
||||
EXPECTED_IMAGES_JSON="$(
|
||||
printf '%s\0' "${image_refs[@]}" |
|
||||
node -e '
|
||||
const fs = require("node:fs");
|
||||
const refs = fs.readFileSync(0).toString("utf8").split("\0").filter(Boolean);
|
||||
process.stdout.write(JSON.stringify(refs));
|
||||
'
|
||||
)" node - "$manifest_path" > "$validated_path" <<'NODE'
|
||||
node - "$manifest_path" "${image_refs[@]}" > "$validated_path" <<'NODE'
|
||||
const fs = require("node:fs");
|
||||
|
||||
const [manifestPath] = process.argv.slice(2);
|
||||
const [manifestPath, ...expectedRefs] = process.argv.slice(2);
|
||||
const value = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||
const expectedRefs = JSON.parse(process.env.EXPECTED_IMAGES_JSON);
|
||||
const fail = (message) => {
|
||||
throw new Error(`invalid shared Docker image artifact: ${message}`);
|
||||
};
|
||||
|
|
|
|||
|
|
@ -111,34 +111,7 @@ func validateDocBodyFencedLiterals(source, translated string) error {
|
|||
if sourceStructure.fenceCount != translatedStructure.fenceCount {
|
||||
return fmt.Errorf("code fence mismatch: source=%d translated=%d", sourceStructure.fenceCount, translatedStructure.fenceCount)
|
||||
}
|
||||
if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) {
|
||||
return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes)
|
||||
}
|
||||
if !slices.Equal(sourceStructure.listMarkerPrefixes, translatedStructure.listMarkerPrefixes) {
|
||||
return fmt.Errorf("list marker structure mismatch: source=%q translated=%q", sourceStructure.listMarkerPrefixes, translatedStructure.listMarkerPrefixes)
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.inlineCodeSpans, translatedStructure.inlineCodeSpans) {
|
||||
return fmt.Errorf("inline code mismatch: source=%d translated=%d", len(sourceStructure.inlineCodeSpans), len(translatedStructure.inlineCodeSpans))
|
||||
}
|
||||
if !slices.Equal(sourceStructure.fencedPlaceholders, translatedStructure.fencedPlaceholders) {
|
||||
return fmt.Errorf("fenced placeholder mismatch: source=%d translated=%d", len(sourceStructure.fencedPlaceholders), len(translatedStructure.fencedPlaceholders))
|
||||
}
|
||||
if !slices.Equal(sourceStructure.fencedProtocolTokens, translatedStructure.fencedProtocolTokens) {
|
||||
return fmt.Errorf("fenced protocol marker mismatch: source=%d translated=%d", len(sourceStructure.fencedProtocolTokens), len(translatedStructure.fencedProtocolTokens))
|
||||
}
|
||||
if !slices.Equal(sourceStructure.fencedDirectiveTokens, translatedStructure.fencedDirectiveTokens) {
|
||||
return fmt.Errorf("fenced directive mismatch: source=%d translated=%d", len(sourceStructure.fencedDirectiveTokens), len(translatedStructure.fencedDirectiveTokens))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) {
|
||||
return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) {
|
||||
return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels))
|
||||
}
|
||||
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
|
||||
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
|
||||
}
|
||||
return nil
|
||||
return validateDocChunkLiterals(sourceStructure, translatedStructure)
|
||||
}
|
||||
|
||||
func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) {
|
||||
|
|
@ -161,14 +134,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
|
|||
log.Printf("docs-i18n: rejected raw chunk %s input=%q output=%q err=%v", chunkID, normalizedSource, translated, err)
|
||||
}
|
||||
if err == nil {
|
||||
translated = sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
|
||||
translated = reapplyCommonIndent(translated, commonIndent)
|
||||
translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders)
|
||||
translated = normalizeMaskedListMarkerSpacing(source, translated, listPlaceholders)
|
||||
translated = escapeUnexpectedListItemBodyMarkers(source, translated, listPlaceholders)
|
||||
translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
|
||||
translated = unwrapUnexpectedInlineCodeSpans(source, translated)
|
||||
translated = normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent, listPlaceholders)
|
||||
if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil {
|
||||
log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated))
|
||||
return translated, nil
|
||||
|
|
@ -182,7 +148,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
|
|||
} else if os.Getenv("OPENCLAW_DOCS_I18N_LOG_REJECTED_BODY") == "1" {
|
||||
log.Printf("docs-i18n: chunk leaf-fallback failed %s err=%v", chunkID, fallbackErr)
|
||||
}
|
||||
if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok {
|
||||
if plan, ok := planSingletonDocChunkWithMode(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget(), true); ok {
|
||||
logDocChunkPlanSplit(chunkID, plan, source)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
|
||||
}
|
||||
|
|
@ -192,7 +158,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun
|
|||
logDocChunkSplit(chunkID, len(blocks), err)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
|
||||
}
|
||||
if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok {
|
||||
if plan, ok := splitDocChunkBlocksMidpoint(blocks, "retry-midpoint"); ok {
|
||||
logDocChunkSplit(chunkID, len(blocks), err)
|
||||
return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang)
|
||||
}
|
||||
|
|
@ -217,6 +183,15 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk
|
|||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
translated = normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent, listPlaceholders)
|
||||
if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
|
||||
return "", validationErr
|
||||
}
|
||||
log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated))
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
func normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent string, listPlaceholders map[string]string) string {
|
||||
translated = sanitizeDocChunkProtocolWrappers(source, translated)
|
||||
translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated)
|
||||
translated = reapplyCommonIndent(translated, commonIndent)
|
||||
|
|
@ -225,11 +200,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk
|
|||
translated = escapeUnexpectedListItemBodyMarkers(source, translated, listPlaceholders)
|
||||
translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders)
|
||||
translated = unwrapUnexpectedInlineCodeSpans(source, translated)
|
||||
if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil {
|
||||
return "", validationErr
|
||||
}
|
||||
log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated))
|
||||
return translated, nil
|
||||
return translated
|
||||
}
|
||||
|
||||
func splitDocBodyIntoBlocks(body string) []string {
|
||||
|
|
@ -324,6 +295,21 @@ func validateDocChunkTranslation(source, translated string) error {
|
|||
if !slices.Equal(sourceStructure.headingLevels, translatedStructure.headingLevels) {
|
||||
return fmt.Errorf("heading structure mismatch: source=%v translated=%v", sourceStructure.headingLevels, translatedStructure.headingLevels)
|
||||
}
|
||||
if err := validateDocChunkLiterals(sourceStructure, translatedStructure); err != nil {
|
||||
return err
|
||||
}
|
||||
if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) {
|
||||
return fmt.Errorf("component tag set mismatch")
|
||||
}
|
||||
for _, key := range sortedKeys(sourceStructure.tagCounts) {
|
||||
if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] {
|
||||
return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateDocChunkLiterals(sourceStructure, translatedStructure docChunkStructure) error {
|
||||
if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) {
|
||||
return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes)
|
||||
}
|
||||
|
|
@ -351,14 +337,6 @@ func validateDocChunkTranslation(source, translated string) error {
|
|||
if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) {
|
||||
return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues))
|
||||
}
|
||||
if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) {
|
||||
return fmt.Errorf("component tag set mismatch")
|
||||
}
|
||||
for _, key := range sortedKeys(sourceStructure.tagCounts) {
|
||||
if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] {
|
||||
return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
@ -410,29 +388,9 @@ func sanitizeDocChunkProtocolWrappers(source, translated string) string {
|
|||
}
|
||||
|
||||
func preserveDocChunkBoundaryWhitespace(source, translated string) string {
|
||||
prefixEnd := 0
|
||||
for prefixEnd < len(source) && isDocChunkBoundaryWhitespace(source[prefixEnd]) {
|
||||
prefixEnd++
|
||||
}
|
||||
suffixStart := len(source)
|
||||
for suffixStart > prefixEnd && isDocChunkBoundaryWhitespace(source[suffixStart-1]) {
|
||||
suffixStart--
|
||||
}
|
||||
|
||||
translatedStart := 0
|
||||
for translatedStart < len(translated) && isDocChunkBoundaryWhitespace(translated[translatedStart]) {
|
||||
translatedStart++
|
||||
}
|
||||
translatedEnd := len(translated)
|
||||
for translatedEnd > translatedStart && isDocChunkBoundaryWhitespace(translated[translatedEnd-1]) {
|
||||
translatedEnd--
|
||||
}
|
||||
|
||||
return source[:prefixEnd] + translated[translatedStart:translatedEnd] + source[suffixStart:]
|
||||
}
|
||||
|
||||
func isDocChunkBoundaryWhitespace(value byte) bool {
|
||||
return value == ' ' || value == '\t' || value == '\r' || value == '\n'
|
||||
prefix, _, suffix := splitWhitespace(source)
|
||||
_, core, _ := splitWhitespace(translated)
|
||||
return prefix + core + suffix
|
||||
}
|
||||
|
||||
func stripBodyOnlyWrapper(source, text string) (string, bool) {
|
||||
|
|
@ -506,13 +464,14 @@ func logDocChunkPlanSplit(chunkID string, plan docChunkSplitPlan, source string)
|
|||
|
||||
func summarizeDocChunkStructure(text string) docChunkStructure {
|
||||
counts := map[string]int{}
|
||||
fenceCount := 0
|
||||
lines := strings.Split(text, "\n")
|
||||
fenceDelimiter := ""
|
||||
for _, line := range lines {
|
||||
var toggled bool
|
||||
fenceDelimiter, toggled = updateFenceDelimiter(fenceDelimiter, line)
|
||||
if toggled {
|
||||
counts["__fence_toggle__"]++
|
||||
fenceCount++
|
||||
}
|
||||
for _, match := range docsComponentTagRE.FindAllStringSubmatch(line, -1) {
|
||||
if len(match) < 3 {
|
||||
|
|
@ -532,8 +491,8 @@ func summarizeDocChunkStructure(text string) docChunkStructure {
|
|||
}
|
||||
fencedPlaceholders, fencedProtocolTokens, fencedDirectiveTokens := extractMarkdownFencedLiteralValues(text)
|
||||
return docChunkStructure{
|
||||
fenceCount: counts["__fence_toggle__"],
|
||||
tagCounts: countsWithoutFence(counts),
|
||||
fenceCount: fenceCount,
|
||||
tagCounts: counts,
|
||||
headingLevels: extractMarkdownHeadingLevels(text),
|
||||
listShapes: extractMarkdownListShapes(text),
|
||||
listMarkerPrefixes: extractMarkdownListMarkerPrefixes(text),
|
||||
|
|
@ -675,17 +634,6 @@ func parseDocsMarkdown(source []byte) ast.Node {
|
|||
return goldmark.New(goldmark.WithExtensions(extension.GFM, extension.Footnote)).Parser().Parse(textpkg.NewReader(source))
|
||||
}
|
||||
|
||||
func countsWithoutFence(counts map[string]int) map[string]int {
|
||||
filtered := map[string]int{}
|
||||
for key, value := range counts {
|
||||
if key == "__fence_toggle__" {
|
||||
continue
|
||||
}
|
||||
filtered[key] = value
|
||||
}
|
||||
return filtered
|
||||
}
|
||||
|
||||
func sortedKeys(counts map[string]int) []string {
|
||||
keys := make([]string, 0, len(counts))
|
||||
for key := range counts {
|
||||
|
|
@ -842,7 +790,7 @@ func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSpl
|
|||
normalizedSource, _ := stripCommonIndent(source)
|
||||
estimatedPromptCost := estimateDocPromptCost(normalizedSource)
|
||||
if len(blocks) > 1 && promptBudget > 0 && estimatedPromptCost > promptBudget {
|
||||
return splitDocChunkBlocksMidpoint(blocks, estimatedPromptCost, promptBudget)
|
||||
return splitDocChunkBlocksMidpoint(blocks, fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget))
|
||||
}
|
||||
if len(blocks) == 1 {
|
||||
return planSingletonDocChunk(blocks[0], maxBytes, promptBudget)
|
||||
|
|
@ -850,32 +798,12 @@ func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSpl
|
|||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
|
||||
func splitDocChunkBlocksMidpoint(blocks []string, estimatedPromptCost, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
func splitDocChunkBlocksMidpoint(blocks []string, reason string) (docChunkSplitPlan, bool) {
|
||||
if len(blocks) <= 1 {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
mid := len(blocks) / 2
|
||||
if mid <= 0 || mid >= len(blocks) {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: [][]string{blocks[:mid], blocks[mid:]},
|
||||
reason: fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget),
|
||||
}, true
|
||||
}
|
||||
|
||||
func splitDocChunkBlocksMidpointSimple(blocks []string) (docChunkSplitPlan, bool) {
|
||||
if len(blocks) <= 1 {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
mid := len(blocks) / 2
|
||||
if mid <= 0 || mid >= len(blocks) {
|
||||
return docChunkSplitPlan{}, false
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: [][]string{blocks[:mid], blocks[mid:]},
|
||||
reason: "retry-midpoint",
|
||||
}, true
|
||||
return docChunkSplitPlan{groups: [][]string{blocks[:mid], blocks[mid:]}, reason: reason}, true
|
||||
}
|
||||
|
||||
func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
|
|
@ -890,43 +818,31 @@ func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSp
|
|||
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, false)
|
||||
}
|
||||
|
||||
func planSingletonDocChunkRetry(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) {
|
||||
return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, true)
|
||||
}
|
||||
|
||||
func planSingletonDocChunkWithMode(block string, maxBytes, promptBudget int, force bool) (docChunkSplitPlan, bool) {
|
||||
prefix := "singleton-"
|
||||
if force {
|
||||
prefix = "singleton-retry-"
|
||||
}
|
||||
if sections := splitDocBlockSections(block); len(sections) > 1 {
|
||||
if groups := wrapDocChunkSections(sections); len(groups) > 1 {
|
||||
reason := "singleton-structural"
|
||||
if force {
|
||||
reason = "singleton-retry-structural"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
reason: prefix + "structural",
|
||||
}, true
|
||||
}
|
||||
}
|
||||
|
||||
if groups, ok := splitPureFencedDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
|
||||
reason := "singleton-fence"
|
||||
if force {
|
||||
reason = "singleton-retry-fence"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
reason: prefix + "fence",
|
||||
}, true
|
||||
}
|
||||
|
||||
if groups, ok := splitPlainDocSectionWithMode(block, maxBytes, promptBudget, force); ok {
|
||||
reason := "singleton-lines"
|
||||
if force {
|
||||
reason = "singleton-retry-lines"
|
||||
}
|
||||
return docChunkSplitPlan{
|
||||
groups: groups,
|
||||
reason: reason,
|
||||
reason: prefix + "lines",
|
||||
}, true
|
||||
}
|
||||
|
||||
|
|
@ -946,9 +862,6 @@ func wrapDocChunkSections(sections []string) [][]string {
|
|||
|
||||
func splitDocBlockSections(block string) []string {
|
||||
lines := strings.SplitAfter(block, "\n")
|
||||
if len(lines) == 0 {
|
||||
return nil
|
||||
}
|
||||
sections := make([]string, 0, len(lines))
|
||||
var current strings.Builder
|
||||
fenceDelimiter := ""
|
||||
|
|
@ -1051,9 +964,6 @@ func splitPlainDocSectionMidpoint(lines []string) ([][]string, bool) {
|
|||
return nil, false
|
||||
}
|
||||
mid := len(lines) / 2
|
||||
if mid <= 0 || mid >= len(lines) {
|
||||
return nil, false
|
||||
}
|
||||
left := strings.Join(lines[:mid], "")
|
||||
right := strings.Join(lines[mid:], "")
|
||||
if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" {
|
||||
|
|
@ -1126,7 +1036,7 @@ func stripCommonIndent(text string) (string, string) {
|
|||
if strings.TrimSpace(trimmed) == "" {
|
||||
continue
|
||||
}
|
||||
indent := leadingIndent(trimmed)
|
||||
indent := leadingWhitespace(trimmed)
|
||||
if common == "" {
|
||||
common = indent
|
||||
} else {
|
||||
|
|
@ -1173,22 +1083,8 @@ func reapplyCommonIndent(text, indent string) string {
|
|||
return out.String()
|
||||
}
|
||||
|
||||
func leadingIndent(line string) string {
|
||||
index := 0
|
||||
for index < len(line) {
|
||||
if line[index] != ' ' && line[index] != '\t' {
|
||||
break
|
||||
}
|
||||
index++
|
||||
}
|
||||
return line[:index]
|
||||
}
|
||||
|
||||
func commonIndentPrefix(a, b string) string {
|
||||
limit := len(a)
|
||||
if len(b) < limit {
|
||||
limit = len(b)
|
||||
}
|
||||
limit := min(len(a), len(b))
|
||||
index := 0
|
||||
for index < limit && a[index] == b[index] {
|
||||
index++
|
||||
|
|
|
|||
|
|
@ -121,25 +121,11 @@ func findTaggedBodyEnd(text string, bodyStart int) int {
|
|||
if bodyStart < 0 || bodyStart > len(text) {
|
||||
return -1
|
||||
}
|
||||
search := text[bodyStart:]
|
||||
candidate := -1
|
||||
offset := 0
|
||||
for {
|
||||
index := strings.Index(search[offset:], bodyTagEnd)
|
||||
if index == -1 {
|
||||
return candidate
|
||||
}
|
||||
index += offset
|
||||
absolute := bodyStart + index
|
||||
suffix := strings.TrimSpace(text[absolute+len(bodyTagEnd):])
|
||||
if suffix == "" {
|
||||
candidate = absolute
|
||||
}
|
||||
offset = index + len(bodyTagEnd)
|
||||
if offset >= len(search) {
|
||||
return candidate
|
||||
}
|
||||
end := strings.LastIndex(text[bodyStart:], bodyTagEnd)
|
||||
if end < 0 || strings.TrimSpace(text[bodyStart+end+len(bodyTagEnd):]) != "" {
|
||||
return -1
|
||||
}
|
||||
return bodyStart + end
|
||||
}
|
||||
|
||||
func trimTagNewlines(value string) string {
|
||||
|
|
@ -164,7 +150,7 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string)
|
|||
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
storedHash := extractSourceHash(frontData)
|
||||
storedHash := extractI18NString(frontData, "source_hash")
|
||||
if storedHash == "" {
|
||||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
|
|
@ -179,7 +165,7 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string)
|
|||
return docOutputNeedsTranslation, nil
|
||||
}
|
||||
|
||||
postprocessVersion := extractPostprocessVersion(frontData)
|
||||
postprocessVersion := extractI18NString(frontData, "postprocess_version")
|
||||
if strings.EqualFold(postprocessVersion, localizedLinkPostprocessVersion) {
|
||||
return docOutputReady, nil
|
||||
}
|
||||
|
|
@ -187,49 +173,17 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string)
|
|||
}
|
||||
|
||||
func extractI18NVersion(frontData map[string]any, field string) int {
|
||||
xi, ok := extractXI18N(frontData)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
value, ok := xi[field].(int)
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
xi, _ := frontData["x-i18n"].(map[string]any)
|
||||
value, _ := xi[field].(int)
|
||||
return value
|
||||
}
|
||||
|
||||
func extractSourceHash(frontData map[string]any) string {
|
||||
xi, ok := extractXI18N(frontData)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
value, ok := xi["source_hash"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
func extractI18NString(frontData map[string]any, field string) string {
|
||||
xi, _ := frontData["x-i18n"].(map[string]any)
|
||||
value, _ := xi[field].(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractPostprocessVersion(frontData map[string]any) string {
|
||||
xi, ok := extractXI18N(frontData)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
value, ok := xi["postprocess_version"].(string)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func extractXI18N(frontData map[string]any) (map[string]any, bool) {
|
||||
xi, ok := frontData["x-i18n"].(map[string]any)
|
||||
if ok {
|
||||
return xi, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
func logDocChunkPlan(relPath string, blocks []string, groups [][]string) {
|
||||
totalBytes := 0
|
||||
for _, block := range blocks {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ package main
|
|||
import (
|
||||
"context"
|
||||
"io"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/yuin/goldmark"
|
||||
|
|
@ -10,15 +11,8 @@ import (
|
|||
"github.com/yuin/goldmark/extension"
|
||||
"github.com/yuin/goldmark/text"
|
||||
"golang.org/x/net/html"
|
||||
"sort"
|
||||
)
|
||||
|
||||
type htmlReplacement struct {
|
||||
Start int
|
||||
Stop int
|
||||
Value string
|
||||
}
|
||||
|
||||
func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, srcLang, tgtLang string) (string, error) {
|
||||
source := []byte(body)
|
||||
r := text.NewReader(source)
|
||||
|
|
@ -27,9 +21,9 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s
|
|||
)
|
||||
doc := md.Parser().Parse(r)
|
||||
|
||||
replacements := make([]htmlReplacement, 0, 8)
|
||||
replacements := make([]Segment, 0, 8)
|
||||
|
||||
_ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
|
||||
if !entering {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
|
|
@ -37,7 +31,7 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s
|
|||
if !ok {
|
||||
return ast.WalkContinue, nil
|
||||
}
|
||||
start, stop, ok := htmlBlockSpan(block, source)
|
||||
start, stop, ok := htmlBlockSpan(block)
|
||||
if !ok {
|
||||
return ast.WalkSkipChildren, nil
|
||||
}
|
||||
|
|
@ -46,53 +40,27 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s
|
|||
if err != nil {
|
||||
return ast.WalkStop, err
|
||||
}
|
||||
replacements = append(replacements, htmlReplacement{Start: start, Stop: stop, Value: translated})
|
||||
replacements = append(replacements, Segment{Start: start, Stop: stop, Translated: translated})
|
||||
return ast.WalkSkipChildren, nil
|
||||
})
|
||||
|
||||
if len(replacements) == 0 {
|
||||
return body, nil
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return applyHTMLReplacements(body, replacements), nil
|
||||
sort.Slice(replacements, func(i, j int) bool {
|
||||
return replacements[i].Start < replacements[j].Start
|
||||
})
|
||||
return applyTranslations(body, replacements), nil
|
||||
}
|
||||
|
||||
func htmlBlockSpan(block *ast.HTMLBlock, source []byte) (int, int, bool) {
|
||||
func htmlBlockSpan(block *ast.HTMLBlock) (int, int, bool) {
|
||||
lines := block.Lines()
|
||||
if lines.Len() == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
start := lines.At(0).Start
|
||||
stop := lines.At(lines.Len() - 1).Stop
|
||||
if start >= stop {
|
||||
return 0, 0, false
|
||||
}
|
||||
return start, stop, true
|
||||
}
|
||||
|
||||
func applyHTMLReplacements(body string, replacements []htmlReplacement) string {
|
||||
if len(replacements) == 0 {
|
||||
return body
|
||||
}
|
||||
sortHTMLReplacements(replacements)
|
||||
var out strings.Builder
|
||||
last := 0
|
||||
for _, rep := range replacements {
|
||||
if rep.Start < last {
|
||||
continue
|
||||
}
|
||||
out.WriteString(body[last:rep.Start])
|
||||
out.WriteString(rep.Value)
|
||||
last = rep.Stop
|
||||
}
|
||||
out.WriteString(body[last:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func sortHTMLReplacements(replacements []htmlReplacement) {
|
||||
sort.Slice(replacements, func(i, j int) bool {
|
||||
return replacements[i].Start < replacements[j].Start
|
||||
})
|
||||
return start, stop, start < stop
|
||||
}
|
||||
|
||||
func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText, srcLang, tgtLang string) (string, error) {
|
||||
|
|
@ -123,8 +91,6 @@ func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText
|
|||
if isSkipTag(strings.ToLower(tok.Data)) && skipDepth > 0 {
|
||||
skipDepth--
|
||||
}
|
||||
case html.SelfClosingTagToken:
|
||||
out.WriteString(raw)
|
||||
case html.TextToken:
|
||||
if shouldTranslateHTMLText(skipDepth, raw) {
|
||||
translated, err := translator.Translate(ctx, raw, srcLang, tgtLang)
|
||||
|
|
@ -144,10 +110,7 @@ func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText
|
|||
}
|
||||
|
||||
func shouldTranslateHTMLText(skipDepth int, text string) bool {
|
||||
if strings.TrimSpace(text) == "" {
|
||||
return false
|
||||
}
|
||||
return skipDepth == 0
|
||||
return skipDepth == 0 && strings.TrimSpace(text) != ""
|
||||
}
|
||||
|
||||
func isSkipTag(tag string) bool {
|
||||
|
|
|
|||
|
|
@ -230,64 +230,22 @@ func (ri *routeIndex) localizeBodyLinks(body string) string {
|
|||
masked = maskMatches(masked, fencedTildeCodeBlock, state.Next, &placeholders, mapping)
|
||||
masked = maskMatches(masked, inlineCodeRe, state.Next, &placeholders, mapping)
|
||||
|
||||
masked = rewriteMarkdownLinkTargets(masked, ri)
|
||||
masked = rewriteHrefTargets(masked, ri)
|
||||
masked = rewriteCapturedTargets(masked, markdownLinkTargetRe, ri, true)
|
||||
masked = rewriteCapturedTargets(masked, hrefDoubleQuotedValueRe, ri, false)
|
||||
masked = rewriteCapturedTargets(masked, hrefSingleQuotedValueRe, ri, false)
|
||||
|
||||
return unmaskMarkdown(masked, placeholders, mapping)
|
||||
}
|
||||
|
||||
func rewriteMarkdownLinkTargets(text string, ri *routeIndex) string {
|
||||
matches := markdownLinkTargetRe.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
fullStart, targetStart, targetEnd := span[0], span[2], span[3]
|
||||
if fullStart < pos {
|
||||
continue
|
||||
func rewriteCapturedTargets(text string, re *regexp.Regexp, ri *routeIndex, skipImages bool) string {
|
||||
return re.ReplaceAllStringFunc(text, func(match string) string {
|
||||
if skipImages && strings.HasPrefix(match, "!") {
|
||||
return match
|
||||
}
|
||||
|
||||
out.WriteString(text[pos:targetStart])
|
||||
target := text[targetStart:targetEnd]
|
||||
if text[fullStart] == '!' {
|
||||
out.WriteString(target)
|
||||
} else {
|
||||
out.WriteString(ri.localizeURL(target))
|
||||
}
|
||||
pos = targetEnd
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
}
|
||||
|
||||
func rewriteHrefTargets(text string, ri *routeIndex) string {
|
||||
text = rewriteCapturedTargets(text, hrefDoubleQuotedValueRe, 2, ri)
|
||||
text = rewriteCapturedTargets(text, hrefSingleQuotedValueRe, 2, ri)
|
||||
return text
|
||||
}
|
||||
|
||||
func rewriteCapturedTargets(text string, re *regexp.Regexp, groupIndex int, ri *routeIndex) string {
|
||||
matches := re.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
start, end := span[groupIndex], span[groupIndex+1]
|
||||
if start < pos || start < 0 || end < 0 {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:start])
|
||||
out.WriteString(ri.localizeURL(text[start:end]))
|
||||
pos = end
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
span := re.FindStringSubmatchIndex(match)
|
||||
start, end := span[2], span[3]
|
||||
return match[:start] + ri.localizeURL(match[start:end]) + match[end:]
|
||||
})
|
||||
}
|
||||
|
||||
func (ri *routeIndex) localizeURL(raw string) string {
|
||||
|
|
|
|||
|
|
@ -138,28 +138,18 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
|
|||
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
|
||||
switch cfg.mode {
|
||||
case "doc":
|
||||
var outputs []string
|
||||
if parallel > 1 {
|
||||
proc, skip, outputs, err := runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial, parallel, glossary, cfg.thinking, newTranslator)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
translationErr = err
|
||||
}
|
||||
processed, skipped, outputs, translationErr = runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial, parallel, glossary, cfg.thinking, newTranslator)
|
||||
} else {
|
||||
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer translator.Close()
|
||||
proc, skip, outputs, err := runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial)
|
||||
processed += proc
|
||||
skipped += skip
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
if err != nil {
|
||||
translationErr = err
|
||||
}
|
||||
processed, skipped, outputs, translationErr = runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial)
|
||||
}
|
||||
localizedFiles = append(localizedFiles, outputs...)
|
||||
case "segment":
|
||||
if parallel > 1 {
|
||||
return fmt.Errorf("parallel processing is only supported in doc mode")
|
||||
|
|
@ -305,7 +295,7 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg
|
|||
log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
} else if result.err != nil {
|
||||
log.Printf("docs-i18n: [w* %d/%d] failed %s (%s): %v", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond), result.err)
|
||||
} else if result.err == nil {
|
||||
} else {
|
||||
processed++
|
||||
outputs = append(outputs, result.output)
|
||||
log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
|
||||
|
|
|
|||
|
|
@ -65,6 +65,22 @@ func (errorTranslator) TranslateRaw(context.Context, string, string, string) (st
|
|||
|
||||
func (errorTranslator) Close() {}
|
||||
|
||||
func TestProcessFileRejectsFailedHTMLTranslation(t *testing.T) {
|
||||
docsRoot := t.TempDir()
|
||||
sourcePath := filepath.Join(docsRoot, "page.md")
|
||||
if err := os.WriteFile(sourcePath, []byte("<div>Translate this text.</div>\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tm := &TranslationMemory{entries: map[string]TMEntry{}}
|
||||
_, _, err := processFile(context.Background(), errorTranslator{}, tm, docsRoot, sourcePath, "en", "de")
|
||||
if err == nil || err.Error() != "codex exec failed: exit status 1" {
|
||||
t.Fatalf("expected HTML translation failure, got %v", err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(docsRoot, "de", "page.md")); !os.IsNotExist(err) {
|
||||
t.Fatalf("failed translation wrote an output page: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type partialFailTranslator struct{}
|
||||
|
||||
func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
|
||||
|
|
|
|||
|
|
@ -219,10 +219,7 @@ func unwrapUnexpectedInlineCodeSpans(source, translated string) string {
|
|||
if rangeOverlapsAny(span, fenced) {
|
||||
continue
|
||||
}
|
||||
runLength := 0
|
||||
for span[0]+runLength < span[1] && translated[span[0]+runLength] == '`' {
|
||||
runLength++
|
||||
}
|
||||
runLength := backtickRunLength(translated, span[0], span[1])
|
||||
if runLength == 0 || span[1]-runLength < span[0]+runLength {
|
||||
continue
|
||||
}
|
||||
|
|
@ -237,46 +234,29 @@ func extractMarkdownFencedLiteralValues(body string) ([]string, []string, []stri
|
|||
allSquareTokens := []string{}
|
||||
state := markdownLiteralFenceState{}
|
||||
lines := []string{}
|
||||
flush := func() {
|
||||
flush := func(info string) {
|
||||
for _, line := range lines {
|
||||
if state.info != "mermaid" {
|
||||
if info != "mermaid" {
|
||||
allSquareTokens = append(allSquareTokens, extractSquareBracketValues(line)...)
|
||||
}
|
||||
}
|
||||
for _, line := range lines {
|
||||
linePlaceholders := extractAngleBracketValues(line)
|
||||
placeholders = append(placeholders, linePlaceholders...)
|
||||
if state.info != "mermaid" {
|
||||
directiveTokens = append(directiveTokens, extractDoubleBracketValues(line)...)
|
||||
}
|
||||
placeholders = append(placeholders, extractAngleBracketValues(line)...)
|
||||
}
|
||||
lines = lines[:0]
|
||||
}
|
||||
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if state.delimiter == "" {
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
previous := state
|
||||
if !state.consumeLine(line) {
|
||||
if previous.delimiter != "" {
|
||||
flush(previous.info)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !continuesMarkdownLiteralFenceContainer(line, state) {
|
||||
flush()
|
||||
state = markdownLiteralFenceState{}
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isMarkdownLiteralFenceClosing(line, state) {
|
||||
flush()
|
||||
state = markdownLiteralFenceState{}
|
||||
continue
|
||||
}
|
||||
lines = append(lines, strings.TrimSpace(stripMarkdownQuotePrefix(line, state.quoteDepth)))
|
||||
}
|
||||
if state.delimiter != "" {
|
||||
flush()
|
||||
flush(state.info)
|
||||
}
|
||||
closingNames := map[string]struct{}{}
|
||||
for _, token := range allSquareTokens {
|
||||
|
|
@ -296,22 +276,7 @@ func extractMarkdownFencedLiteralValues(body string) ([]string, []string, []stri
|
|||
func markdownLiteralFencesBalanced(body string) bool {
|
||||
state := markdownLiteralFenceState{}
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if state.delimiter == "" {
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !continuesMarkdownLiteralFenceContainer(line, state) {
|
||||
state = markdownLiteralFenceState{}
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isMarkdownLiteralFenceClosing(line, state) {
|
||||
state = markdownLiteralFenceState{}
|
||||
}
|
||||
state.consumeLine(line)
|
||||
}
|
||||
return state.delimiter == ""
|
||||
}
|
||||
|
|
@ -323,6 +288,19 @@ type markdownLiteralFenceState struct {
|
|||
containerIndent int
|
||||
}
|
||||
|
||||
// consumeLine reports literal content, excluding opening and closing fence lines.
|
||||
func (state *markdownLiteralFenceState) consumeLine(line string) bool {
|
||||
if state.delimiter != "" && continuesMarkdownLiteralFenceContainer(line, *state) {
|
||||
if !isMarkdownLiteralFenceClosing(line, *state) {
|
||||
return true
|
||||
}
|
||||
*state = markdownLiteralFenceState{}
|
||||
return false
|
||||
}
|
||||
*state, _ = parseMarkdownLiteralFenceOpening(line)
|
||||
return false
|
||||
}
|
||||
|
||||
func parseMarkdownLiteralFenceOpening(line string) (markdownLiteralFenceState, bool) {
|
||||
leadingIndent := len(line) - len(strings.TrimLeft(line, " \t"))
|
||||
remaining := strings.TrimLeft(line, " \t")
|
||||
|
|
@ -540,7 +518,7 @@ func isTranslatableBracketLabelContext(line string, start, end int, candidate st
|
|||
}
|
||||
|
||||
func isASCIIIdentifierByte(value byte) bool {
|
||||
return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') || (value >= '0' && value <= '9') || value == '_'
|
||||
return isASCIIAlphaNumeric(value) || value == '_'
|
||||
}
|
||||
|
||||
func extractDoubleBracketValues(line string) []string {
|
||||
|
|
@ -662,10 +640,7 @@ func extractFallbackBacktickValues(body string) []string {
|
|||
if rangeOverlapsAny(span, fenced) {
|
||||
continue
|
||||
}
|
||||
runLength := 0
|
||||
for span[0]+runLength < span[1] && body[span[0]+runLength] == '`' {
|
||||
runLength++
|
||||
}
|
||||
runLength := backtickRunLength(body, span[0], span[1])
|
||||
if runLength == 0 || span[1]-runLength < span[0]+runLength {
|
||||
continue
|
||||
}
|
||||
|
|
@ -753,10 +728,7 @@ func normalizeDocComponentsForMarkdownParse(body string) string {
|
|||
}
|
||||
|
||||
func isLikelyFencedBacktickRange(body string, span [2]int) bool {
|
||||
runLength := 0
|
||||
for span[0]+runLength < span[1] && body[span[0]+runLength] == '`' {
|
||||
runLength++
|
||||
}
|
||||
runLength := backtickRunLength(body, span[0], span[1])
|
||||
if runLength < 3 {
|
||||
return false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -55,24 +55,9 @@ func maskMarkdownFencedLiterals(text string, nextPlaceholder func() string, plac
|
|||
state := markdownLiteralFenceState{}
|
||||
lines := strings.SplitAfter(text, "\n")
|
||||
for index, line := range lines {
|
||||
if state.delimiter == "" {
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
}
|
||||
continue
|
||||
if state.consumeLine(line) {
|
||||
lines[index] = maskMatches(line, literalRE, nextPlaceholder, placeholders, mapping)
|
||||
}
|
||||
if !continuesMarkdownLiteralFenceContainer(line, state) {
|
||||
state = markdownLiteralFenceState{}
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
state = opening
|
||||
}
|
||||
continue
|
||||
}
|
||||
if isMarkdownLiteralFenceClosing(line, state) {
|
||||
state = markdownLiteralFenceState{}
|
||||
continue
|
||||
}
|
||||
lines[index] = maskMatches(line, literalRE, nextPlaceholder, placeholders, mapping)
|
||||
}
|
||||
return strings.Join(lines, "")
|
||||
}
|
||||
|
|
@ -96,23 +81,9 @@ func markdownListMarkerRanges(text string) [][2]int {
|
|||
fenceState := markdownLiteralFenceState{}
|
||||
offset := 0
|
||||
for _, line := range strings.SplitAfter(text, "\n") {
|
||||
insideFence := false
|
||||
if fenceState.delimiter != "" {
|
||||
if continuesMarkdownLiteralFenceContainer(line, fenceState) {
|
||||
insideFence = true
|
||||
if isMarkdownLiteralFenceClosing(line, fenceState) {
|
||||
fenceState = markdownLiteralFenceState{}
|
||||
}
|
||||
} else {
|
||||
fenceState = markdownLiteralFenceState{}
|
||||
}
|
||||
}
|
||||
if !insideFence {
|
||||
if opening, ok := parseMarkdownLiteralFenceOpening(line); ok {
|
||||
fenceState = opening
|
||||
insideFence = true
|
||||
}
|
||||
}
|
||||
insideFence := fenceState.delimiter != "" && continuesMarkdownLiteralFenceContainer(line, fenceState)
|
||||
fenceState.consumeLine(line)
|
||||
insideFence = insideFence || fenceState.delimiter != ""
|
||||
if !insideFence {
|
||||
if match := listMarkerRe.FindStringSubmatchIndex(line); len(match) >= 6 {
|
||||
listRanges = append(listRanges, [2]int{offset + match[0], offset + match[1]})
|
||||
|
|
@ -206,10 +177,7 @@ func markdownWhitespaceRunStart(text string, position int) int {
|
|||
}
|
||||
|
||||
func escapeUnexpectedListItemBodyMarkers(source, translated string, listPlaceholders map[string]string) string {
|
||||
type insertion struct {
|
||||
position int
|
||||
}
|
||||
insertions := make([]insertion, 0)
|
||||
insertions := make([]int, 0)
|
||||
for placeholder := range listPlaceholders {
|
||||
sourcePosition := strings.Index(source, placeholder)
|
||||
translatedPosition := strings.Index(translated, placeholder)
|
||||
|
|
@ -228,11 +196,11 @@ func escapeUnexpectedListItemBodyMarkers(source, translated string, listPlacehol
|
|||
if markerEnd-markerStart > 1 {
|
||||
insertAt = markerEnd - 1
|
||||
}
|
||||
insertions = append(insertions, insertion{position: translatedPosition + len(placeholder) + insertAt})
|
||||
insertions = append(insertions, translatedPosition+len(placeholder)+insertAt)
|
||||
}
|
||||
sort.Slice(insertions, func(i, j int) bool { return insertions[i].position > insertions[j].position })
|
||||
for _, item := range insertions {
|
||||
translated = translated[:item.position] + `\` + translated[item.position:]
|
||||
sort.Slice(insertions, func(i, j int) bool { return insertions[i] > insertions[j] })
|
||||
for _, position := range insertions {
|
||||
translated = translated[:position] + `\` + translated[position:]
|
||||
}
|
||||
return translated
|
||||
}
|
||||
|
|
@ -340,7 +308,7 @@ func hasClockMeridiemSuffix(text string, span [2]int) bool {
|
|||
if suffix != "am" && suffix != "pm" {
|
||||
return false
|
||||
}
|
||||
return span[1]+2 == len(text) || !isCompositeNumericWordByte(text[span[1]+2])
|
||||
return span[1]+2 == len(text) || !isASCIIAlphaNumeric(text[span[1]+2])
|
||||
}
|
||||
|
||||
func hasCompositeNumericLeadingContinuation(text string, position int) bool {
|
||||
|
|
@ -352,9 +320,9 @@ func hasCompositeNumericLeadingContinuation(text string, position int) bool {
|
|||
for position > 0 && text[position-1] == '_' {
|
||||
position--
|
||||
}
|
||||
return position > 0 && isCompositeNumericWordByte(text[position-1])
|
||||
return position > 0 && isASCIIAlphaNumeric(text[position-1])
|
||||
}
|
||||
return value == '.' || isCompositeNumericWordByte(value)
|
||||
return value == '.' || isASCIIAlphaNumeric(value)
|
||||
}
|
||||
|
||||
func hasCompositeNumericContinuation(text string, position int) bool {
|
||||
|
|
@ -366,16 +334,12 @@ func hasCompositeNumericContinuation(text string, position int) bool {
|
|||
for position < len(text) && text[position] == '_' {
|
||||
position++
|
||||
}
|
||||
return position < len(text) && isCompositeNumericWordByte(text[position])
|
||||
return position < len(text) && isASCIIAlphaNumeric(text[position])
|
||||
}
|
||||
if isCompositeNumericWordByte(value) {
|
||||
if isASCIIAlphaNumeric(value) {
|
||||
return true
|
||||
}
|
||||
return value == '.' && position+1 < len(text) && isCompositeNumericWordByte(text[position+1])
|
||||
}
|
||||
|
||||
func isCompositeNumericWordByte(value byte) bool {
|
||||
return value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z'
|
||||
return value == '.' && position+1 < len(text) && isASCIIAlphaNumeric(text[position+1])
|
||||
}
|
||||
|
||||
func markdownLiteralFenceByteRanges(text string) [][2]int {
|
||||
|
|
@ -449,53 +413,20 @@ func maskByteRanges(text string, ranges [][2]int, nextPlaceholder func() string,
|
|||
}
|
||||
|
||||
func maskMatches(text string, re *regexp.Regexp, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
|
||||
matches := re.FindAllStringIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
}
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
start, end := span[0], span[1]
|
||||
if start < pos {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:start])
|
||||
return re.ReplaceAllStringFunc(text, func(match string) string {
|
||||
placeholder := nextPlaceholder()
|
||||
mapping[placeholder] = text[start:end]
|
||||
mapping[placeholder] = match
|
||||
*placeholders = append(*placeholders, placeholder)
|
||||
out.WriteString(placeholder)
|
||||
pos = end
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
return placeholder
|
||||
})
|
||||
}
|
||||
|
||||
func maskLinkURLs(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string {
|
||||
matches := linkURLRe.FindAllStringSubmatchIndex(text, -1)
|
||||
if len(matches) == 0 {
|
||||
return text
|
||||
ranges := make([][2]int, 0)
|
||||
for _, span := range linkURLRe.FindAllStringSubmatchIndex(text, -1) {
|
||||
ranges = append(ranges, [2]int{span[2], span[3]})
|
||||
}
|
||||
var out strings.Builder
|
||||
pos := 0
|
||||
for _, span := range matches {
|
||||
fullStart := span[0]
|
||||
urlStart, urlEnd := span[2], span[3]
|
||||
if urlStart < 0 || urlEnd < 0 {
|
||||
continue
|
||||
}
|
||||
if fullStart < pos {
|
||||
continue
|
||||
}
|
||||
out.WriteString(text[pos:urlStart])
|
||||
placeholder := nextPlaceholder()
|
||||
mapping[placeholder] = text[urlStart:urlEnd]
|
||||
*placeholders = append(*placeholders, placeholder)
|
||||
out.WriteString(placeholder)
|
||||
pos = urlEnd
|
||||
}
|
||||
out.WriteString(text[pos:])
|
||||
return out.String()
|
||||
return maskByteRanges(text, ranges, nextPlaceholder, placeholders, mapping)
|
||||
}
|
||||
|
||||
func unmaskMarkdown(text string, placeholders []string, mapping map[string]string) string {
|
||||
|
|
|
|||
|
|
@ -139,31 +139,22 @@ func translateFrontMatter(ctx context.Context, translator docsTranslator, tm *Tr
|
|||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
if summary, ok := data["summary"].(string); ok {
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=summary bytes=%d", relPath, len(summary))
|
||||
for _, field := range []string{"summary", "title"} {
|
||||
text, ok := data[field].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:summary", summary, srcLang, tgtLang)
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=%s bytes=%d", relPath, field, len(text))
|
||||
}
|
||||
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:"+field, text, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter done %s field=summary out_bytes=%d", relPath, len(translated))
|
||||
log.Printf("docs-i18n: frontmatter done %s field=%s out_bytes=%d", relPath, field, len(translated))
|
||||
}
|
||||
data["summary"] = translated
|
||||
}
|
||||
if title, ok := data["title"].(string); ok {
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter start %s field=title bytes=%d", relPath, len(title))
|
||||
}
|
||||
translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if docsI18nVerboseLogs() {
|
||||
log.Printf("docs-i18n: frontmatter done %s field=title out_bytes=%d", relPath, len(translated))
|
||||
}
|
||||
data["title"] = translated
|
||||
data[field] = translated
|
||||
}
|
||||
if readWhen, ok := data["read_when"].([]any); ok {
|
||||
translated := make([]any, 0, len(readWhen))
|
||||
|
|
@ -218,11 +209,9 @@ func translateSnippet(ctx context.Context, translator docsTranslator, tm *Transl
|
|||
log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, err)
|
||||
return textValue, nil
|
||||
}
|
||||
shouldCache := true
|
||||
if validationErr := validateFrontmatterScalarTranslation(textValue, translated); validationErr != nil {
|
||||
log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, validationErr)
|
||||
translated = textValue
|
||||
shouldCache = false
|
||||
return textValue, nil
|
||||
}
|
||||
sourcePath := segmentID
|
||||
if path, _, ok := strings.Cut(segmentID, ":frontmatter:"); ok {
|
||||
|
|
@ -239,9 +228,7 @@ func translateSnippet(ctx context.Context, translator docsTranslator, tm *Transl
|
|||
TgtLang: tgtLang,
|
||||
UpdatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if shouldCache {
|
||||
tm.Put(entry)
|
||||
}
|
||||
tm.Put(entry)
|
||||
return translated, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -30,14 +30,11 @@ func cacheNamespace() string {
|
|||
|
||||
func cacheKey(namespace, srcLang, tgtLang, segmentID, textHash string) string {
|
||||
raw := fmt.Sprintf("%s|%s|%s|%s|%s", namespace, srcLang, tgtLang, segmentID, textHash)
|
||||
hash := sha256.Sum256([]byte(raw))
|
||||
return hex.EncodeToString(hash[:])
|
||||
return hashBytes([]byte(raw))
|
||||
}
|
||||
|
||||
func hashText(text string) string {
|
||||
normalized := normalizeText(text)
|
||||
hash := sha256.Sum256([]byte(normalized))
|
||||
return hex.EncodeToString(hash[:])
|
||||
return hashBytes([]byte(normalizeText(text)))
|
||||
}
|
||||
|
||||
func hashBytes(data []byte) string {
|
||||
|
|
|
|||
|
|
@ -733,11 +733,7 @@ async function isPrivilegedTargetAuthor(github, context, target, labelSet, isMai
|
|||
if (labelSet.has(maintainerAuthorLabel) || privilegedAuthorAssociations.has(authorAssociation)) {
|
||||
return true;
|
||||
}
|
||||
if (await isPrivilegedActor(github, context, authorLogin, isMaintainer)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
return isPrivilegedActor(github, context, authorLogin, isMaintainer);
|
||||
}
|
||||
|
||||
async function countMaintainerMentions(body, authorLogin, isMaintainer, owner) {
|
||||
|
|
@ -772,15 +768,6 @@ async function countMaintainerMentions(body, authorLogin, isMaintainer, owner) {
|
|||
return count;
|
||||
}
|
||||
|
||||
async function listPullRequestFiles(github, context, pullRequest) {
|
||||
return github.paginate(github.rest.pulls.listFiles, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: pullRequest.number,
|
||||
per_page: 100,
|
||||
});
|
||||
}
|
||||
|
||||
async function addMissingLabels(github, context, core, issueNumber, labels, labelSet) {
|
||||
const missingLabels = labels.filter((label) => !labelSet.has(label));
|
||||
if (missingLabels.length === 0) {
|
||||
|
|
@ -803,7 +790,12 @@ function isClawSweeperOwnedLabel(label) {
|
|||
}
|
||||
|
||||
async function applyPullRequestCandidateLabels(github, context, core, pullRequest, labelSet) {
|
||||
const files = await listPullRequestFiles(github, context, pullRequest);
|
||||
const files = await github.paginate(github.rest.pulls.listFiles, {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: pullRequest.number,
|
||||
per_page: 100,
|
||||
});
|
||||
const candidateLabelsToApply = classifyPullRequestCandidateLabels(
|
||||
{
|
||||
...pullRequest,
|
||||
|
|
@ -851,10 +843,6 @@ function isClawSweeperProofSufficientLabelEvent(context) {
|
|||
);
|
||||
}
|
||||
|
||||
function isGitHubAppPullRequestAuthor(pullRequest) {
|
||||
return isAutomationUser(pullRequest.user);
|
||||
}
|
||||
|
||||
function candidateActionRuleForLabelSet(labelSet, preferredLabel = "") {
|
||||
const preferredRule = candidateActionRules.find(
|
||||
(rule) => rule.label === preferredLabel && labelSet.has(rule.label),
|
||||
|
|
@ -891,23 +879,29 @@ async function applyPullRequestCandidateAction({
|
|||
return false;
|
||||
}
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
await applyResponseRule(github, context, pullRequest.number, rule);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function applyResponseRule(github, context, issueNumber, rule) {
|
||||
const target = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
body: rule.message,
|
||||
});
|
||||
|
||||
issue_number: issueNumber,
|
||||
};
|
||||
if (rule.message) {
|
||||
await github.rest.issues.createComment({ ...target, body: rule.message });
|
||||
}
|
||||
if (rule.close) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
...target,
|
||||
state: "closed",
|
||||
...(rule.stateReason ? { state_reason: rule.stateReason } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
return true;
|
||||
if (rule.lock) {
|
||||
await github.rest.issues.lock({ ...target, lock_reason: rule.lockReason ?? "resolved" });
|
||||
}
|
||||
}
|
||||
|
||||
async function removeLabels(github, context, issueNumber, labels, labelSet) {
|
||||
|
|
@ -1067,35 +1061,21 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
const title = issue.title ?? "";
|
||||
const body = issue.body ?? "";
|
||||
const haystack = `${title}\n${body}`.toLowerCase();
|
||||
const hasMoltbookLabel = labelSet.has("r: moltbook");
|
||||
const hasTestflightLabel = labelSet.has("r: testflight");
|
||||
const hasSecurityLabel = labelSet.has("security");
|
||||
if (title.toLowerCase().includes("security") && !hasSecurityLabel) {
|
||||
for (const [label, matches] of [
|
||||
["security", title.toLowerCase().includes("security")],
|
||||
["r: testflight", title.toLowerCase().includes("testflight")],
|
||||
["r: moltbook", haystack.includes("moltbook")],
|
||||
]) {
|
||||
if (!matches || labelSet.has(label)) {
|
||||
continue;
|
||||
}
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: ["security"],
|
||||
labels: [label],
|
||||
});
|
||||
labelSet.add("security");
|
||||
}
|
||||
if (title.toLowerCase().includes("testflight") && !hasTestflightLabel) {
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: ["r: testflight"],
|
||||
});
|
||||
labelSet.add("r: testflight");
|
||||
}
|
||||
if (haystack.includes("moltbook") && !hasMoltbookLabel) {
|
||||
await github.rest.issues.addLabels({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
labels: ["r: moltbook"],
|
||||
});
|
||||
labelSet.add("r: moltbook");
|
||||
labelSet.add(label);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1116,7 +1096,7 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
return;
|
||||
}
|
||||
|
||||
if (isGitHubAppPullRequestAuthor(pullRequest)) {
|
||||
if (isAutomationUser(pullRequest.user)) {
|
||||
await removeLabels(github, context, pullRequest.number, [activePrLimitLabel], labelSet);
|
||||
core.info(`Skipping active PR limit for GitHub App-authored PR #${pullRequest.number}.`);
|
||||
}
|
||||
|
|
@ -1131,42 +1111,22 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
}
|
||||
|
||||
if (labelSet.has(dirtyLabel)) {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
body: noisyPrMessage,
|
||||
});
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
state: "closed",
|
||||
await applyResponseRule(github, context, pullRequest.number, {
|
||||
message: noisyPrMessage,
|
||||
close: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (labelSet.has(spamLabel)) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
state: "closed",
|
||||
});
|
||||
await github.rest.issues.lock({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
lock_reason: "spam",
|
||||
await applyResponseRule(github, context, pullRequest.number, {
|
||||
close: true,
|
||||
lock: true,
|
||||
lockReason: "spam",
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (labelSet.has(invalidLabel)) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: pullRequest.number,
|
||||
state: "closed",
|
||||
});
|
||||
await applyResponseRule(github, context, pullRequest.number, { close: true });
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1184,29 +1144,19 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
}
|
||||
|
||||
if (issue && labelSet.has(spamLabel)) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
state: "closed",
|
||||
state_reason: "not_planned",
|
||||
});
|
||||
await github.rest.issues.lock({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
lock_reason: "spam",
|
||||
await applyResponseRule(github, context, issue.number, {
|
||||
close: true,
|
||||
stateReason: "not_planned",
|
||||
lock: true,
|
||||
lockReason: "spam",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (issue && labelSet.has(invalidLabel)) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issue.number,
|
||||
state: "closed",
|
||||
state_reason: "not_planned",
|
||||
await applyResponseRule(github, context, issue.number, {
|
||||
close: true,
|
||||
stateReason: "not_planned",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
|
@ -1214,10 +1164,7 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
if (pullRequest && labelSet.has(activePrLimitOverrideLabel)) {
|
||||
labelSet.delete(activePrLimitLabel);
|
||||
}
|
||||
if (
|
||||
pullRequest &&
|
||||
(isAutomationPullRequest(pullRequest) || isGitHubAppPullRequestAuthor(pullRequest))
|
||||
) {
|
||||
if (pullRequest && (isAutomationPullRequest(pullRequest) || isAutomationUser(pullRequest.user))) {
|
||||
await removeLabels(github, context, pullRequest.number, [activePrLimitLabel], labelSet);
|
||||
}
|
||||
|
||||
|
|
@ -1226,30 +1173,5 @@ export async function runBarnacleAutoResponse({ github, context, core = console
|
|||
return;
|
||||
}
|
||||
|
||||
const issueNumber = target.number;
|
||||
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
body: rule.message,
|
||||
});
|
||||
|
||||
if (rule.close) {
|
||||
await github.rest.issues.update({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
state: "closed",
|
||||
});
|
||||
}
|
||||
|
||||
if (rule.lock) {
|
||||
await github.rest.issues.lock({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: issueNumber,
|
||||
lock_reason: rule.lockReason ?? "resolved",
|
||||
});
|
||||
}
|
||||
await applyResponseRule(github, context, target.number, rule);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -395,19 +395,8 @@ function decodeContentFile(payload) {
|
|||
return Buffer.from(payload.content, payload.encoding ?? "base64").toString("utf8");
|
||||
}
|
||||
|
||||
async function readJsonFileAtRef(api, { owner, repo, path, ref }) {
|
||||
if (!ref) {
|
||||
return null;
|
||||
}
|
||||
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
|
||||
const payload = await api
|
||||
.request(`/repos/${owner}/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`)
|
||||
.catch((error) => {
|
||||
if (error?.status === 404) {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
async function readJsonFileAtRef(api, options) {
|
||||
const payload = await readContentFileMetadataAtRef(api, options);
|
||||
const text = decodeContentFile(payload);
|
||||
return text ? JSON.parse(text) : null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,10 +114,6 @@ export async function readBoundedGitHubApiJson(
|
|||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
async function cancelGitHubApiResponseBody(response) {
|
||||
await response.body?.cancel?.().catch(() => undefined);
|
||||
}
|
||||
|
||||
function normalizeLineEndings(text = "") {
|
||||
return text.replace(/\r\n?/g, "\n");
|
||||
}
|
||||
|
|
@ -172,10 +168,6 @@ function maskHtmlComments(text) {
|
|||
.join("\n");
|
||||
}
|
||||
|
||||
function stripHtmlComments(text) {
|
||||
return maskHtmlComments(text);
|
||||
}
|
||||
|
||||
function isAutomationUser(user = {}, fallbackLogin = "") {
|
||||
const login = user?.login ?? fallbackLogin;
|
||||
return user?.type === "Bot" || /\[bot\]$/i.test(login) || login.startsWith("app/");
|
||||
|
|
@ -252,7 +244,7 @@ export async function isMaintainerTeamMember({
|
|||
);
|
||||
return body?.state === "active";
|
||||
} finally {
|
||||
await cancelGitHubApiResponseBody(response);
|
||||
await response.body?.cancel?.().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -336,10 +328,6 @@ function legacyProofFieldLineValue(line) {
|
|||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function isAnyLegacyProofFieldLine(line) {
|
||||
return legacyProofFieldLineValue(line) !== null;
|
||||
}
|
||||
|
||||
function extractFieldValue(section, field) {
|
||||
const lines = maskHtmlComments(normalizeLineEndings(section)).split("\n");
|
||||
let fenceMarker = "";
|
||||
|
|
@ -361,7 +349,7 @@ function extractFieldValue(section, field) {
|
|||
const lineLocal = lines[next];
|
||||
if (
|
||||
!fenceMarker &&
|
||||
(markdownHeadingLevel(lineLocal) > 0 || isAnyLegacyProofFieldLine(lineLocal))
|
||||
(markdownHeadingLevel(lineLocal) > 0 || legacyProofFieldLineValue(lineLocal) !== null)
|
||||
) {
|
||||
break;
|
||||
}
|
||||
|
|
@ -374,7 +362,7 @@ function extractFieldValue(section, field) {
|
|||
}
|
||||
|
||||
function stripMarkdownFenceMarkers(value) {
|
||||
return stripHtmlComments(normalizeLineEndings(value))
|
||||
return maskHtmlComments(normalizeLineEndings(value))
|
||||
.split("\n")
|
||||
.filter((line) => !/^ {0,3}(?:`{3,}|~{3,})(?:.*)?$/.test(line))
|
||||
.join("\n")
|
||||
|
|
|
|||
|
|
@ -3,10 +3,7 @@
|
|||
if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then
|
||||
exec /bin/bash "$0" "$@"
|
||||
fi
|
||||
# ============================================================================
|
||||
# KIND CLUSTER BOOTSTRAP SCRIPT
|
||||
# ============================================================================
|
||||
#
|
||||
# Bootstrap a local Kind cluster.
|
||||
# Usage:
|
||||
# ./scripts/k8s/create-kind.sh # Create with auto-detected engine
|
||||
# ./scripts/k8s/create-kind.sh --name mycluster
|
||||
|
|
@ -14,16 +11,13 @@ fi
|
|||
#
|
||||
# After creation, deploy with:
|
||||
# export <AI_PROVIDER>_API_KEY="..." && ./scripts/k8s/deploy.sh
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Defaults
|
||||
CLUSTER_NAME="openclaw"
|
||||
CONTAINER_CMD=""
|
||||
DELETE=false
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
BLUE='\033[0;34m'
|
||||
YELLOW='\033[0;33m'
|
||||
|
|
@ -53,9 +47,6 @@ EOF
|
|||
exit 0
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument parsing
|
||||
# ---------------------------------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--name)
|
||||
|
|
@ -70,39 +61,18 @@ while [[ $# -gt 0 ]]; do
|
|||
esac
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Container engine detection
|
||||
# ---------------------------------------------------------------------------
|
||||
provider_installed() {
|
||||
command -v "$1" &>/dev/null
|
||||
}
|
||||
|
||||
provider_responsive() {
|
||||
case "$1" in
|
||||
docker)
|
||||
docker info &>/dev/null
|
||||
;;
|
||||
podman)
|
||||
podman info &>/dev/null
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
detect_provider() {
|
||||
local candidate
|
||||
|
||||
for candidate in podman docker; do
|
||||
if provider_installed "$candidate" && provider_responsive "$candidate"; then
|
||||
if command -v "$candidate" &>/dev/null && "$candidate" info &>/dev/null; then
|
||||
echo "$candidate"
|
||||
return 0
|
||||
fi
|
||||
done
|
||||
|
||||
for candidate in podman docker; do
|
||||
if provider_installed "$candidate"; then
|
||||
if command -v "$candidate" &>/dev/null; then
|
||||
case "$candidate" in
|
||||
podman)
|
||||
fail "Podman is installed but not responding, and no responsive Docker daemon was found. Ensure the podman machine is running (podman machine start) or start Docker."
|
||||
|
|
@ -120,9 +90,6 @@ detect_provider() {
|
|||
CONTAINER_CMD=$(detect_provider)
|
||||
info "Auto-detected container engine: $CONTAINER_CMD"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prerequisites
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! command -v kind &>/dev/null; then
|
||||
fail "kind is not installed. Install it from https://kind.sigs.k8s.io/"
|
||||
fi
|
||||
|
|
@ -131,8 +98,7 @@ if ! command -v kubectl &>/dev/null; then
|
|||
fail "kubectl is not installed. Install it before creating or managing a Kind cluster."
|
||||
fi
|
||||
|
||||
# Verify the container engine is responsive
|
||||
if ! provider_responsive "$CONTAINER_CMD"; then
|
||||
if ! "$CONTAINER_CMD" info &>/dev/null; then
|
||||
if [[ "$CONTAINER_CMD" == "docker" ]]; then
|
||||
fail "Docker daemon is not running. Start it and try again."
|
||||
elif [[ "$CONTAINER_CMD" == "podman" ]]; then
|
||||
|
|
@ -140,9 +106,6 @@ if ! provider_responsive "$CONTAINER_CMD"; then
|
|||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Delete mode
|
||||
# ---------------------------------------------------------------------------
|
||||
if $DELETE; then
|
||||
info "Deleting Kind cluster '$CLUSTER_NAME'..."
|
||||
if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then
|
||||
|
|
@ -154,9 +117,6 @@ if $DELETE; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Check if cluster already exists
|
||||
# ---------------------------------------------------------------------------
|
||||
if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then
|
||||
warn "Cluster '$CLUSTER_NAME' already exists."
|
||||
info "To recreate it, run: $0 --name \"$CLUSTER_NAME\" --delete && $0 --name \"$CLUSTER_NAME\""
|
||||
|
|
@ -165,9 +125,6 @@ if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | g
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Create cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
info "Creating Kind cluster '$CLUSTER_NAME' (provider: $CONTAINER_CMD)..."
|
||||
|
||||
KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind create cluster \
|
||||
|
|
@ -191,16 +148,10 @@ KINDCFG
|
|||
|
||||
success "Kind cluster '$CLUSTER_NAME' created."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Wait for readiness
|
||||
# ---------------------------------------------------------------------------
|
||||
info "Waiting for cluster to be ready..."
|
||||
kubectl --context "kind-$CLUSTER_NAME" wait --for=condition=Ready nodes --all --timeout=120s >/dev/null
|
||||
success "All nodes are Ready."
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summary
|
||||
# ---------------------------------------------------------------------------
|
||||
echo ""
|
||||
echo "---------------------------------------------------------------"
|
||||
echo " Kind cluster '$CLUSTER_NAME' is ready"
|
||||
|
|
|
|||
|
|
@ -24,15 +24,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
MANIFESTS="$SCRIPT_DIR/manifests"
|
||||
NS="${OPENCLAW_NAMESPACE:-openclaw}"
|
||||
|
||||
# Check prerequisites
|
||||
for cmd in kubectl openssl; do
|
||||
command -v "$cmd" &>/dev/null || { echo "Missing: $cmd" >&2; exit 1; }
|
||||
done
|
||||
kubectl cluster-info &>/dev/null || { echo "Cannot connect to cluster. Check kubeconfig." >&2; exit 1; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# -h / --help
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then
|
||||
cat <<'HELP'
|
||||
Usage: ./scripts/k8s/deploy.sh [OPTION]
|
||||
|
|
@ -61,17 +57,8 @@ MODE="deploy"
|
|||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--create-secret)
|
||||
MODE="create-secret"
|
||||
;;
|
||||
--delete)
|
||||
MODE="delete"
|
||||
;;
|
||||
--delete-resources)
|
||||
MODE="delete-resources"
|
||||
;;
|
||||
--delete-namespace)
|
||||
MODE="delete-namespace"
|
||||
--create-secret | --delete | --delete-resources | --delete-namespace)
|
||||
MODE="${1#--}"
|
||||
;;
|
||||
--show-token)
|
||||
SHOW_TOKEN=true
|
||||
|
|
@ -85,9 +72,6 @@ while [[ $# -gt 0 ]]; do
|
|||
shift
|
||||
done
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# --delete / --delete-namespace
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "delete" && "$NS" != "openclaw" ]]; then
|
||||
MODE="delete-resources"
|
||||
fi
|
||||
|
|
@ -99,9 +83,6 @@ if [[ "$MODE" == "delete" || "$MODE" == "delete-namespace" ]]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# --delete-resources
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "delete-resources" ]]; then
|
||||
echo "Deleting OpenClaw resources from namespace '$NS'..."
|
||||
kubectl delete -k "$MANIFESTS" -n "$NS" --ignore-not-found
|
||||
|
|
@ -110,9 +91,6 @@ if [[ "$MODE" == "delete-resources" ]]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Create and apply Secret to the cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
_apply_secret() {
|
||||
local TMP_DIR
|
||||
local EXISTING_SECRET=false
|
||||
|
|
@ -189,9 +167,6 @@ _apply_secret() {
|
|||
fi
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# --create-secret
|
||||
# ---------------------------------------------------------------------------
|
||||
if [[ "$MODE" == "create-secret" ]]; then
|
||||
HAS_KEY=false
|
||||
for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY; do
|
||||
|
|
@ -215,9 +190,6 @@ if [[ "$MODE" == "create-secret" ]]; then
|
|||
exit 0
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Check that the secret exists in the cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! kubectl get secret openclaw-secrets -n "$NS" &>/dev/null; then
|
||||
HAS_KEY=false
|
||||
for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY; do
|
||||
|
|
@ -238,9 +210,6 @@ if ! kubectl get secret openclaw-secrets -n "$NS" &>/dev/null; then
|
|||
fi
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Deploy
|
||||
# ---------------------------------------------------------------------------
|
||||
echo "Deploying to namespace '$NS'..."
|
||||
kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
kubectl apply -k "$MANIFESTS" -n "$NS"
|
||||
|
|
|
|||
|
|
@ -467,9 +467,6 @@ function laneLine(label, lane) {
|
|||
}
|
||||
return pieces.join("");
|
||||
}
|
||||
function publicSummary(manifest) {
|
||||
return manifest.summary ?? "Mantis captured QA evidence for this scenario.";
|
||||
}
|
||||
function overallStatus(manifest) {
|
||||
const outcome = manifest.comparison?.outcome;
|
||||
if (outcome === "blocked" || outcome === "fail" || outcome === "pass") {
|
||||
|
|
@ -478,13 +475,6 @@ function overallStatus(manifest) {
|
|||
const pass = manifest.comparison?.pass;
|
||||
return typeof pass === "boolean" ? String(pass) : "";
|
||||
}
|
||||
/**
|
||||
* @param {EvidenceManifest} manifest
|
||||
* @param {{ requestSource?: string }} [options]
|
||||
*/
|
||||
export function shouldPublishPrComment() {
|
||||
return true;
|
||||
}
|
||||
/** @param {RenderEvidenceCommentOptions} options */
|
||||
export function renderEvidenceComment({
|
||||
artifactUrl: actionsArtifactUrl,
|
||||
|
|
@ -511,7 +501,7 @@ export function renderEvidenceComment({
|
|||
marker,
|
||||
`## ${manifest.title}`,
|
||||
"",
|
||||
`Summary: ${publicSummary(manifest)}`,
|
||||
`Summary: ${manifest.summary ?? "Mantis captured QA evidence for this scenario."}`,
|
||||
"",
|
||||
`- Scenario: \`${manifest.scenario}\``,
|
||||
];
|
||||
|
|
@ -542,8 +532,7 @@ export function renderEvidenceComment({
|
|||
lines.push(`- Overall: \`${overall}\``);
|
||||
}
|
||||
lines.push("");
|
||||
const pairedSections = pairs.map((pair) => renderPairTable({ pair, rawBase }));
|
||||
lines.push(...pairedSections);
|
||||
lines.push(...pairs.map((pair) => renderPairTable({ pair, rawBase })));
|
||||
const singleTables = renderSingleImageTables({
|
||||
artifacts: manifest.artifacts,
|
||||
pairedKeys,
|
||||
|
|
@ -552,23 +541,14 @@ export function renderEvidenceComment({
|
|||
if (singleTables) {
|
||||
lines.push(singleTables);
|
||||
}
|
||||
const motionClips = renderLinkList({
|
||||
artifacts: manifest.artifacts,
|
||||
kind: "motionClip",
|
||||
rawBase,
|
||||
title: "Motion-trimmed clips",
|
||||
});
|
||||
if (motionClips) {
|
||||
lines.push(motionClips);
|
||||
}
|
||||
const fullVideos = renderLinkList({
|
||||
artifacts: manifest.artifacts,
|
||||
kind: "fullVideo",
|
||||
rawBase,
|
||||
title: "Full videos",
|
||||
});
|
||||
if (fullVideos) {
|
||||
lines.push(fullVideos);
|
||||
for (const [kind, title] of [
|
||||
["motionClip", "Motion-trimmed clips"],
|
||||
["fullVideo", "Full videos"],
|
||||
]) {
|
||||
const links = renderLinkList({ artifacts: manifest.artifacts, kind, rawBase, title });
|
||||
if (links) {
|
||||
lines.push(links);
|
||||
}
|
||||
}
|
||||
lines.push(`Raw QA files: ${treeUrl ?? rawBase}`);
|
||||
return `${lines.join("\n").replace(/\n{3,}/gu, "\n\n")}\n`;
|
||||
|
|
@ -769,10 +749,6 @@ export async function publishEvidence(rawArgs = process.argv.slice(2)) {
|
|||
runUrl: args.run_url,
|
||||
treeUrl: published.treeUrl,
|
||||
});
|
||||
if (!shouldPublishPrComment(manifest, { requestSource: args.request_source })) {
|
||||
console.log("Skipped Mantis QA evidence PR comment because the run did not capture proof.");
|
||||
return;
|
||||
}
|
||||
upsertPrComment({
|
||||
body,
|
||||
createMissing: args.create_missing !== "false",
|
||||
|
|
|
|||
|
|
@ -63,9 +63,6 @@ export function shouldPrintHelp(argv: readonly string[]): boolean {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parses CPU profile file paths and --limit.
|
||||
*/
|
||||
export function parseArgs(argv: readonly string[]): { files: string[]; limit: number } {
|
||||
const files: string[] = [];
|
||||
let limit = DEFAULT_LIMIT;
|
||||
|
|
@ -150,7 +147,7 @@ function summarizeProfile(file: string, limit: number): void {
|
|||
const profile: unknown = JSON.parse(fs.readFileSync(file, "utf8"));
|
||||
validateProfile(profile, file);
|
||||
const nodes = new Map(profile.nodes.map((node) => [node.id, node]));
|
||||
const samples = Array.isArray(profile.samples) ? profile.samples : [];
|
||||
const samples = profile.samples;
|
||||
const deltas = Array.isArray(profile.timeDeltas) ? profile.timeDeltas : [];
|
||||
const byFrame = new Map<string, number>();
|
||||
const byModule = new Map<string, number>();
|
||||
|
|
@ -178,17 +175,16 @@ function summarizeProfile(file: string, limit: number): void {
|
|||
const durationMs = (profile.endTime - profile.startTime) / 1000;
|
||||
console.log(`\n${file}`);
|
||||
console.log(`duration_ms: ${durationMs.toFixed(1)} samples: ${samples.length}`);
|
||||
console.log("top_frames:");
|
||||
for (const [key, micros] of [...byFrame.entries()]
|
||||
.toSorted((left, right) => right[1] - left[1])
|
||||
.slice(0, limit)) {
|
||||
console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`);
|
||||
}
|
||||
console.log("top_modules:");
|
||||
for (const [key, micros] of [...byModule.entries()]
|
||||
.toSorted((left, right) => right[1] - left[1])
|
||||
.slice(0, limit)) {
|
||||
console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`);
|
||||
for (const [label, totals] of [
|
||||
["top_frames", byFrame],
|
||||
["top_modules", byModule],
|
||||
] as const) {
|
||||
console.log(`${label}:`);
|
||||
for (const [key, micros] of [...totals.entries()]
|
||||
.toSorted((left, right) => right[1] - left[1])
|
||||
.slice(0, limit)) {
|
||||
console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,8 +46,6 @@ require_cmd() {
|
|||
fi
|
||||
}
|
||||
|
||||
is_root() { [[ "$(id -u)" -eq 0 ]]; }
|
||||
|
||||
run_podman_pull() {
|
||||
local image="$1"
|
||||
openclaw_host_timeout_cmd "$PODMAN_PULL_TIMEOUT" podman pull "$image"
|
||||
|
|
@ -181,7 +179,7 @@ if [[ "$INSTALL_QUADLET" == true ]]; then
|
|||
fi
|
||||
|
||||
require_cmd podman
|
||||
if is_root; then
|
||||
if [[ "$(id -u)" -eq 0 ]]; then
|
||||
echo "Run scripts/podman/setup.sh as your normal user so Podman stays rootless." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import { createServer, type Server } from "node:http";
|
|||
import type { Duplex } from "node:stream";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { WebSocket, WebSocketServer } from "ws";
|
||||
import { createGatewayWsClient } from "../../scripts/dev/gateway-ws-client.js";
|
||||
import { createGatewayWsClient } from "../../scripts/lib/gateway-ws-client.js";
|
||||
|
||||
let server: Server | undefined;
|
||||
let wss: WebSocketServer | undefined;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import path from "node:path";
|
||||
import { runInNewContext } from "node:vm";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
import type { RealtimeVoiceBridgeCreateRequest } from "../../src/talk/provider-types.js";
|
||||
|
||||
|
|
@ -7,6 +8,7 @@ const backend = vi.hoisted(() => ({ onFirstAudio: () => {} }));
|
|||
const browser = vi.hoisted(() => ({
|
||||
close: vi.fn(async () => {}),
|
||||
contextClose: vi.fn(async () => {}),
|
||||
googleEvaluate: vi.fn(),
|
||||
evaluate: vi.fn(async () => ({
|
||||
answerHasAudio: true,
|
||||
remoteDescriptionApplied: true,
|
||||
|
|
@ -25,6 +27,7 @@ vi.mock("playwright", () => ({
|
|||
chromium: {
|
||||
launch: async () => ({
|
||||
close: browser.close,
|
||||
newPage: async () => ({ evaluate: browser.googleEvaluate, close: async () => {} }),
|
||||
newContext: async () => ({
|
||||
close: browser.contextClose,
|
||||
newPage: async () => ({ evaluate: browser.evaluate }),
|
||||
|
|
@ -33,6 +36,24 @@ vi.mock("playwright", () => ({
|
|||
},
|
||||
}));
|
||||
|
||||
vi.mock("../../extensions/google/realtime-voice-provider.ts", () => ({
|
||||
buildGoogleRealtimeVoiceProvider: () => ({
|
||||
createBrowserSession: async () => ({
|
||||
transport: "provider-websocket",
|
||||
protocol: "google-live-bidi",
|
||||
clientSecret: "synthetic-google-session",
|
||||
websocketUrl: "wss://google.example.test/live",
|
||||
initialMessage: { setup: {} },
|
||||
}),
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("vite", () => ({
|
||||
createServer: async () => {
|
||||
throw new Error("Synthetic relay smoke unavailable");
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../../extensions/openai/realtime-voice-provider.ts", () => ({
|
||||
buildOpenAIRealtimeVoiceProvider: () => ({
|
||||
createBridge: (options: RealtimeVoiceBridgeCreateRequest) => {
|
||||
|
|
@ -83,6 +104,58 @@ afterEach(() => {
|
|||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("reports malformed Google Live frames from the serialized browser callback", async () => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
vi.stubEnv("OPENAI_API_KEY", "");
|
||||
vi.stubEnv("GEMINI_API_KEY", "synthetic-google-key");
|
||||
const output = vi.spyOn(console, "log").mockImplementation(() => {});
|
||||
process.argv = [process.execPath, path.resolve("scripts/dev/realtime-talk-live-smoke.ts")];
|
||||
process.exitCode = 0;
|
||||
vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] });
|
||||
let started!: () => void;
|
||||
const callbackStarted = new Promise<void>((resolve) => {
|
||||
started = resolve;
|
||||
});
|
||||
browser.googleEvaluate.mockImplementation((callback, payload) => {
|
||||
if (typeof callback === "string") {
|
||||
return undefined;
|
||||
}
|
||||
const result = runInNewContext(`(${callback.toString()})(payload)`, {
|
||||
payload,
|
||||
URL,
|
||||
__name: (value: unknown) => value,
|
||||
window: { setTimeout, clearTimeout },
|
||||
WebSocket: class {
|
||||
addEventListener(event: string, listener: (message: { data: string }) => void) {
|
||||
if (event === "message") {
|
||||
queueMicrotask(() => listener({ data: "not-json" }));
|
||||
}
|
||||
}
|
||||
close() {}
|
||||
},
|
||||
});
|
||||
started();
|
||||
return result;
|
||||
});
|
||||
const command = import("../../scripts/dev/realtime-talk-live-smoke.ts");
|
||||
await Promise.race([
|
||||
callbackStarted,
|
||||
command.then(() => {
|
||||
throw new Error("Smoke command completed before the Google browser callback");
|
||||
}),
|
||||
]);
|
||||
await vi.runAllTimersAsync();
|
||||
await command;
|
||||
|
||||
expect(output).toHaveBeenCalledWith("google-live-browser-ws: failed", {
|
||||
error: expect.stringContaining("not valid JSON"),
|
||||
});
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(browser.close).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "failed browser media",
|
||||
|
|
|
|||
|
|
@ -9,6 +9,83 @@ const scriptPath = ".agents/skills/openclaw-secret-scanning-maintainer/scripts/s
|
|||
const { createTempDir } = createScriptTestHarness();
|
||||
|
||||
describe("secret scanning maintainer script", () => {
|
||||
it.each([
|
||||
["issue_body", "Issue", "issue_number"],
|
||||
["pull_request_body", "PullRequest", "pr_number"],
|
||||
["pull_request_review_comment", "PullRequestReviewComment", "issue_number"],
|
||||
])(
|
||||
"keeps %s content private while reporting identity and edit history",
|
||||
(type, nodeType, numberKey) => {
|
||||
const tempDir = createTempDir("openclaw-secret-scan-content-");
|
||||
const ghPath = path.join(tempDir, "gh");
|
||||
const fixturePath = path.join(tempDir, "content.json");
|
||||
const body = "synthetic private content\n";
|
||||
const htmlUrl = "https://github.com/openclaw/openclaw/pull/123";
|
||||
fs.writeFileSync(
|
||||
fixturePath,
|
||||
JSON.stringify({
|
||||
id: 456,
|
||||
node_id: "fixture-node",
|
||||
number: 123,
|
||||
user: { login: "contributor" },
|
||||
body,
|
||||
html_url: htmlUrl,
|
||||
merged: false,
|
||||
state: "open",
|
||||
}),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
ghPath,
|
||||
`#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] !== "api") process.exit(1);
|
||||
if (args[1] === "graphql") {
|
||||
if (!args.join(" ").includes("... on " + process.env.EXPECTED_NODE_TYPE)) process.exit(2);
|
||||
console.log(JSON.stringify({ data: { node: { userContentEdits: { totalCount: 3 } } } }));
|
||||
} else {
|
||||
process.stdout.write(fs.readFileSync(process.env.CONTENT_FIXTURE, "utf8"));
|
||||
}
|
||||
`,
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
scriptPath,
|
||||
"fetch-content",
|
||||
JSON.stringify({ type, details: { [`${type}_url`]: "https://api.github.com/fixture" } }),
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
TMPDIR: tempDir,
|
||||
OPENCLAW_GH_BIN: ghPath,
|
||||
PATH: `${tempDir}${path.delimiter}${process.env.PATH ?? ""}`,
|
||||
EXPECTED_NODE_TYPE: nodeType,
|
||||
CONTENT_FIXTURE: fixturePath,
|
||||
},
|
||||
},
|
||||
);
|
||||
const result = JSON.parse(output);
|
||||
expect(result).toEqual({
|
||||
type,
|
||||
...(type === "pull_request_review_comment" ? { comment_id: 456 } : {}),
|
||||
[numberKey]: type === "pull_request_review_comment" ? "123" : 123,
|
||||
node_id: "fixture-node",
|
||||
author: "contributor",
|
||||
...(type === "pull_request_body" ? { merged: false, state: "open" } : {}),
|
||||
html_url: htmlUrl,
|
||||
edit_history_count: 3,
|
||||
body_file: expect.any(String),
|
||||
});
|
||||
expect(output).not.toContain(body.trim());
|
||||
expect(fs.readFileSync(result.body_file, "utf8")).toBe(body);
|
||||
expect(fs.statSync(result.body_file).mode & 0o777).toBe(0o600);
|
||||
},
|
||||
);
|
||||
|
||||
it("marks body alerts as not requiring notification when redaction is unchanged", () => {
|
||||
const tempDir = createTempDir("openclaw-secret-scan-");
|
||||
const currentBody = path.join(tempDir, "current.md");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue