diff --git a/.agents/skills/agent-transcript/scripts/agent-transcript b/.agents/skills/agent-transcript/scripts/agent-transcript index 9ff72dd28aaa..bd1376f9044d 100755 --- a/.agents/skills/agent-transcript/scripts/agent-transcript +++ b/.agents/skills/agent-transcript/scripts/agent-transcript @@ -330,12 +330,8 @@ function toolCallFamily(row) { } function compactToolSummary(familyCounts, dropped) { - const families = new Map(); - for (const [family, count] of familyCounts.entries()) { - families.set(family, (families.get(family) || 0) + count); - } const ordered = ["read", "write", "execute", "network", "other"] - .map((family) => [family, families.get(family) || 0]) + .map((family) => [family, familyCounts.get(family) || 0]) .filter(([, count]) => count > 0) .map(([family, count]) => `${count} ${family}`); const calls = ordered.length ? ordered.join(", ") : "0 tool"; @@ -503,10 +499,6 @@ function recentFiles(files, maxFiles) { .map((entry) => entry.file); } -function candidateFiles(roots, terms, sinceMs, options = {}) { - return recentFiles(roots.flatMap((root) => walkJsonl(root, sinceMs)), Number(options["max-files"] || 400)); -} - function findSessions(options) { const sinceDays = Number(options["since-days"] || 14); const sinceMs = Date.now() - sinceDays * 24 * 60 * 60 * 1000; @@ -516,14 +508,13 @@ function findSessions(options) { .split(/\s+/) .concat(query.match(/https?:\/\/\S+/g) || []) .filter(Boolean); - const files = candidateFiles(roots, terms, sinceMs, options); + const files = recentFiles(roots.flatMap((root) => walkJsonl(root, sinceMs)), Number(options["max-files"] || 400)); const scanBytes = Number(options["scan-bytes"] || 60000); - const results = files + return files .map((file) => scoreScanRecord(sessionScanRecord(file, scanBytes), terms, options.cwd)) .filter((result) => result.score > 0) .sort((a, b) => b.score - a.score || b.mtime.localeCompare(a.mtime)) .slice(0, Number(options.limit || 10)); - return results; } function sessionScanRecords(options) { @@ -578,10 +569,6 @@ ${rows} `; } -function singlePreviewDocument(record) { - return htmlDocument([record]); -} - function readPrs(file) { const raw = fs.readFileSync(file, "utf8"); const parsed = JSON.parse(raw); @@ -599,39 +586,28 @@ function main() { console.log(JSON.stringify(findSessions(args), null, 2)); return; } - if (command === "render") { - if (!args.session) throw new Error("--session is required"); + if (["render", "preview", "append-body"].includes(command)) { + if (command === "append-body") { + if (!args.body || !args.session) throw new Error("--body and --session are required"); + } else if (!args.session) throw new Error("--session is required"); const rendered = renderSession(args.session, args); if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`); - if (args.out) fs.writeFileSync(args.out, rendered.markdown); - else process.stdout.write(rendered.markdown); - return; - } - if (command === "preview") { - if (!args.session) throw new Error("--session is required"); - const rendered = renderSession(args.session, args); - if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`); - const output = singlePreviewDocument({ - title: args.title || "Agent Transcript Preview", - url: args.url || "", - session: args.session, - safe: rendered.safe, - markdown: rendered.markdown, - }); + let output = rendered.markdown; + if (command === "preview") { + output = htmlDocument([{ + title: args.title || "Agent Transcript Preview", + url: args.url || "", + session: args.session, + safe: rendered.safe, + markdown: rendered.markdown, + }]); + } else if (command === "append-body") { + output = replaceSection(fs.readFileSync(args.body, "utf8"), rendered.markdown); + } if (args.out) fs.writeFileSync(args.out, output); else process.stdout.write(output); return; } - if (command === "append-body") { - if (!args.body || !args.session) throw new Error("--body and --session are required"); - const rendered = renderSession(args.session, args); - if (!rendered.safe) throw new Error(`unsafe transcript after redaction: ${rendered.unsafeAfter.join(", ")}`); - const body = fs.readFileSync(args.body, "utf8"); - const next = replaceSection(body, rendered.markdown); - if (args.out) fs.writeFileSync(args.out, next); - else process.stdout.write(next); - return; - } if (command === "html") { if (!args.prs) throw new Error("--prs is required"); const records = []; diff --git a/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs b/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs index b2f09106082a..19a5e9bd2625 100644 --- a/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs +++ b/.agents/skills/openclaw-changelog-update/scripts/verify-release-notes.mjs @@ -129,65 +129,44 @@ export function parseArgs(argv) { shippedRefs: [], writeLedger: false, }; + const booleanOptions = new Map([ + ["--help", "help"], + ["--check-github", "checkGithub"], + ["--json", "json"], + ["--no-github-snapshot", "noGithubSnapshot"], + ["--refresh-github-snapshot", "refreshGithubSnapshot"], + ["--write-ledger", "writeLedger"], + ]); + const valueOptions = new Map([ + ["--base", "base"], + ["--target", "target"], + ["--version", "version"], + ["--release-tag", "releaseTags"], + ["--release-provenance", "releaseProvenance"], + ["--shipped-ref", "shippedRefs"], + ["--github-snapshot", "githubSnapshotPath"], + ["--main-ref", "mainRef"], + ["--manifest", "manifestPath"], + ["--seed-ref", "seedRef"], + ]); for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; - if (arg === "--help") { - options.help = true; + const booleanKey = booleanOptions.get(arg); + if (booleanKey) { + options[booleanKey] = true; continue; } - if ( - arg === "--check-github" || - arg === "--json" || - arg === "--no-github-snapshot" || - arg === "--refresh-github-snapshot" || - arg === "--write-ledger" - ) { - options[ - arg === "--check-github" - ? "checkGithub" - : arg === "--write-ledger" - ? "writeLedger" - : arg === "--no-github-snapshot" - ? "noGithubSnapshot" - : arg === "--refresh-github-snapshot" - ? "refreshGithubSnapshot" - : "json" - ] = true; - continue; - } - if ( - arg === "--base" || - arg === "--target" || - arg === "--version" || - arg === "--release-tag" || - arg === "--release-provenance" || - arg === "--shipped-ref" || - arg === "--github-snapshot" || - arg === "--main-ref" || - arg === "--manifest" || - arg === "--seed-ref" - ) { + const valueKey = valueOptions.get(arg); + if (valueKey) { const value = argv[index + 1]; if (!value || value.startsWith("--")) { fail(`missing value for ${arg}`); } - if (arg === "--release-tag") { - options.releaseTags.push(value); - } else if (arg === "--release-provenance") { - options.releaseProvenance.push(value); - } else if (arg === "--shipped-ref") { - options.shippedRefs.push(value); - } else if (arg === "--manifest") { - options.manifestPath = value; - } else if (arg === "--github-snapshot") { - options.githubSnapshotPath = value; - } else if (arg === "--main-ref") { - options.mainRef = value; - } else if (arg === "--seed-ref") { - options.seedRef = value; + if (Array.isArray(options[valueKey])) { + options[valueKey].push(value); } else { - options[arg.slice(2)] = value; + options[valueKey] = value; } index += 1; continue; @@ -894,12 +873,8 @@ export function withoutExcludedContributionRecords(record, excludedReferences) { return filtered; } -function contributionRecordReferences(record) { - return [...record.pullRequests.keys()]; -} - function contributionRecordMetadataReferences(record) { - const references = contributionRecordReferences(record); + const references = [...record.pullRequests.keys()]; for (const entry of record.pullRequests.values()) { appendReferences(references, entry.references); } @@ -1265,35 +1240,15 @@ function sourceCommits(base, target, mainRef, releaseProvenance = []) { const coauthorsByReference = new Map(); const activeCommits = []; for (const commit of commits.values()) { - if (commit.isRevert && isActive(commit.hash)) { - const coauthorEmails = [...commit.body.matchAll(/^Co-authored-by:\s*.+?<([^>\s]+)>$/gim)].map( - (match) => match[1], - ); - activeCommits.push({ - authorEmail: commit.authorEmail, - authorHandle: githubHandleFromNoreply(commit.authorEmail), - authorName: commit.authorName, - body: commit.body, - closingReferences: [], - committedAt: commit.committedAt, - coauthors: coauthorEmails.map(githubHandleFromNoreply).filter(isEligibleHandle), - coauthorEmails, - hash: commit.hash, - isRevert: true, - pullRequests: [], - references: [], - subject: commit.subject, - }); - continue; - } - if (commit.isRevert) { - continue; - } - const uniqueReferences = [...new Set(referencesIn(`${commit.subject}\n${commit.body}`))]; + const uniqueReferences = commit.isRevert + ? [] + : [...new Set(referencesIn(`${commit.subject}\n${commit.body}`))]; if (!isActive(commit.hash)) { - revertedCommitHashes.add(commit.hash); - for (const number of uniqueReferences) { - revertedReferences.add(number); + if (!commit.isRevert) { + revertedCommitHashes.add(commit.hash); + for (const number of uniqueReferences) { + revertedReferences.add(number); + } } continue; } @@ -1306,12 +1261,14 @@ function sourceCommits(base, target, mainRef, releaseProvenance = []) { authorHandle: githubHandleFromNoreply(commit.authorEmail), authorName: commit.authorName, body: commit.body, - closingReferences: closingReferencesIn(`${commit.subject}\n${commit.body}`), + closingReferences: commit.isRevert + ? [] + : closingReferencesIn(`${commit.subject}\n${commit.body}`), committedAt: commit.committedAt, coauthors, coauthorEmails, hash: commit.hash, - isRevert: false, + isRevert: commit.isRevert, pullRequests: [], references: uniqueReferences, subject: commit.subject, @@ -2009,23 +1966,12 @@ function thanksFor(node, coauthorHandles) { if (node.author?.__typename === "User" && isEligibleHandle(node.author.login)) { handles.push(node.author.login); } - for (const handle of coauthorHandles) { - if (!handles.some((candidate) => candidate.toLowerCase() === handle.toLowerCase())) { - handles.push(handle); - } - } + appendUnique(handles, coauthorHandles); return handles; } function addHandles(handles, additions) { - for (const handle of additions) { - if (!isEligibleHandle(handle)) { - continue; - } - if (!handles.some((candidate) => candidate.toLowerCase() === handle.toLowerCase())) { - handles.push(handle); - } - } + appendUnique(handles, [...additions].filter(isEligibleHandle)); return handles; } @@ -2033,10 +1979,6 @@ function titleReferences(entries) { return [...new Set(entries.flatMap((entry) => referencesIn(entry.title)))]; } -function releaseTitle(title) { - return title; -} - function withSentenceEnding(value) { return /[.!?]$/.test(value) ? value : `${value}.`; } @@ -2236,7 +2178,7 @@ export function ledgerFor( .filter(Boolean); return { number, - title: releaseTitle(node.title.replace(/\s+/g, " ").trim()), + title: node.title.replace(/\s+/g, " ").trim(), type: node.__typename, mergedAt: node.mergedAt, closingIssuesReferences: node.closingIssuesReferences, @@ -2863,10 +2805,8 @@ function main() { ); } } - if (errors.length === 0) { - if (options.writeLedger) { - writeReleaseChangelog({ rootDir, version: options.version, section: candidateChangelog }); - } + if (errors.length === 0 && options.writeLedger) { + writeReleaseChangelog({ rootDir, version: options.version, section: candidateChangelog }); } const result = { diff --git a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs index 19d014dc4c2d..ce67ffa3312f 100644 --- a/.agents/skills/openclaw-live-updater/scripts/update-main.mjs +++ b/.agents/skills/openclaw-live-updater/scripts/update-main.mjs @@ -22,7 +22,7 @@ import { import { tmpdir } from "node:os"; import path from "node:path"; import process from "node:process"; -import { setTimeout as delay } from "node:timers/promises"; +import { setTimeout as defaultSleep } from "node:timers/promises"; import { fileURLToPath } from "node:url"; import { detectChangedScope } from "../../../../scripts/ci-changed-scope.mjs"; import { isDirectRunUrl } from "../../../../scripts/lib/direct-run.mjs"; @@ -1986,29 +1986,16 @@ function defaultResumeGatewaySuspension(checkout, suspensionId, deployment) { } async function stopManagedGateway(runCommand, checkout, deployment) { - if (!deployment) { - await runUpdateCommand( - runCommand, - "gateway.stop", - process.execPath, - ["dist/index.js", "gateway", "stop"], - checkout, - { - phase: "gateway stop", - serviceState: "stopping", - timeoutMs: COMMAND_TIMEOUT_MS.gatewayService, - }, - ); - return; - } await runUpdateCommand( runCommand, - "launchd.bootout", - "/bin/launchctl", - ["bootout", `gui/${process.getuid()}/${deployment.label}`], + deployment ? "launchd.bootout" : "gateway.stop", + deployment ? "/bin/launchctl" : process.execPath, + deployment + ? ["bootout", `gui/${process.getuid()}/${deployment.label}`] + : ["dist/index.js", "gateway", "stop"], checkout, { - phase: "Gateway LaunchAgent bootout", + phase: deployment ? "Gateway LaunchAgent bootout" : "gateway stop", serviceState: "stopping", timeoutMs: COMMAND_TIMEOUT_MS.gatewayService, }, @@ -2105,12 +2092,7 @@ function isTrustedSourceControlBuild(checkout, buildState, currentHead) { ) { return false; } - try { - git(checkout, ["merge-base", "--is-ancestor", commit, currentHead]); - return true; - } catch { - return false; - } + return isAncestorCommit(checkout, commit, currentHead); } function resolveGatewayControlDeployment(checkout, deployment, buildBefore, currentHead) { @@ -2418,7 +2400,7 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options const serviceTarget = `${domain}/${deployment.label}`; const waitForProcess = options.waitForProcess ?? waitForManagedGatewayProcess; const now = options.now ?? Date.now; - if (!options.startupTrace) { + const start = async () => { await runUpdateCommand( runCommand, "launchd.enable", @@ -2439,7 +2421,10 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options waitForProcess, options.sleep ?? defaultSleep, ); - return { processStartedAt: timestampAt(now) }; + return timestampAt(now); + }; + if (!options.startupTrace) { + return { processStartedAt: await start() }; } const readLaunchdEnvironment = options.readLaunchdEnvironment ?? readLaunchdEnvironmentVariable; @@ -2461,27 +2446,7 @@ async function bootstrapManagedGateway(runCommand, checkout, deployment, options }, ); try { - await runUpdateCommand( - runCommand, - "launchd.enable", - "/bin/launchctl", - ["enable", serviceTarget], - checkout, - { - phase: "Gateway LaunchAgent enable", - serviceState: "stopped", - timeoutMs: COMMAND_TIMEOUT_MS.gatewayService, - }, - ); - await bootstrapLaunchAgentAndWait( - runCommand, - checkout, - deployment, - domain, - waitForProcess, - options.sleep ?? defaultSleep, - ); - processStartedAt = timestampAt(now); + processStartedAt = await start(); } catch (error) { restartError = error; } @@ -2751,21 +2716,15 @@ function markGatewayMilestones(timing, observation, observedAt, deepRpcUpperBoun if (!observation) { return; } - if (observation.listenerReady) { - recordGatewayTimestamp( - timing, - "listenerReadyAt", - deepRpcUpperBoundAt ?? observedAt, - deepRpcUpperBoundAt ? "no-later-than" : "observed", - ); - } - if (observation.healthzReady) { - recordGatewayTimestamp( - timing, - "healthzReadyAt", - deepRpcUpperBoundAt ?? observedAt, - deepRpcUpperBoundAt ? "no-later-than" : "observed", - ); + for (const key of ["listenerReady", "healthzReady"]) { + if (observation[key]) { + recordGatewayTimestamp( + timing, + `${key}At`, + deepRpcUpperBoundAt ?? observedAt, + deepRpcUpperBoundAt ? "no-later-than" : "observed", + ); + } } if (observation.readyzReady) { recordGatewayTimestamp(timing, "readyzReadyAt", observedAt); @@ -2804,16 +2763,14 @@ async function readGatewayHealth(runCommand, checkout, deployment) { ["health", "--verbose", "--json"], deployment, ); - let healthSummary; try { - healthSummary = JSON.parse(healthOutput); + return JSON.parse(healthOutput); } catch (error) { throw new UpdateInvariantError( "gateway_health_invalid", `Gateway health probe did not return JSON: ${error instanceof Error ? error.message : String(error)}`, ); } - return healthSummary; } await runUpdateCommand( runCommand, @@ -2844,10 +2801,6 @@ async function verifyGateway(runCommand, checkout, expectedSha, deployment = nul }; } -function defaultSleep(ms) { - return delay(ms); -} - /** * @param {(command: string, args: string[], checkout: string, options?: Record) => void | Promise} runCommand * @param {string} checkout @@ -3255,7 +3208,7 @@ export function findExactMacTarget(processes, executable) { } async function defaultVerifyMacTarget(checkout) { - await delay(10_000); + await defaultSleep(10_000); const executable = path.join(checkout, "dist/OpenClaw.app/Contents/MacOS/OpenClaw"); const processes = execFileSync( "ps", diff --git a/.agents/skills/openclaw-secret-scanning-maintainer/scripts/secret-scanning.mjs b/.agents/skills/openclaw-secret-scanning-maintainer/scripts/secret-scanning.mjs index 72026867e460..8a90fd9e6df1 100644 --- a/.agents/skills/openclaw-secret-scanning-maintainer/scripts/secret-scanning.mjs +++ b/.agents/skills/openclaw-secret-scanning-maintainer/scripts/secret-scanning.mjs @@ -14,8 +14,6 @@ import { execPlainGh } from "../../../../scripts/lib/plain-gh.mjs"; const REPO = "openclaw/openclaw"; const REPO_URL = `https://github.com/${REPO}`; -// ─── Helpers ──────────────────────────────────────────────────────────────── - function fail(message) { console.error(`error: ${message}`); process.exit(1); @@ -23,7 +21,6 @@ function fail(message) { function tmpFile(purpose) { const filePath = path.join(os.tmpdir(), `secretscan-${purpose}-${crypto.randomUUID()}`); - // 预创建文件,限制权限为 owner-only fs.writeFileSync(filePath, "", { mode: 0o600 }); return filePath; } @@ -60,6 +57,19 @@ function ghGraphQL(query, options = {}) { return gh(["api", "graphql", "-f", `query=${query}`], options); } +function contentEditCount(nodeId, typeName) { + const result = ghGraphQL(`{ + node(id: "${nodeId}") { + ... on ${typeName} { + userContentEdits(first: 50) { + totalCount + } + } + } + }`); + return result?.data?.node?.userContentEdits?.totalCount ?? 0; +} + function isBodyLocationType(locationType) { return locationType === "issue_body" || locationType === "pull_request_body"; } @@ -251,8 +261,6 @@ function createDiscussionComment(discussionNodeId, body, replyToNodeId) { return result?.data?.addDiscussionComment?.comment; } -// ─── Commands ─────────────────────────────────────────────────────────────── - /** * fetch-alert * Fetch alert metadata + locations. Never exposes .secret. @@ -270,7 +278,6 @@ function cmdFetchAlert(alertNumber) { "--paginate", "--slurp", ]); - // --paginate + --slurp 确保多页结果合并为一个 JSON 数组 const flatLocations = Array.isArray(locations?.[0]) ? locations.flat() : Array.isArray(locations) @@ -305,6 +312,7 @@ function cmdFetchContent(locationJson) { const location = JSON.parse(locationJson); const type = location.type; const details = location.details; + let result; if (type === "discussion_comment") { const commentUrl = details.discussion_comment_url; @@ -332,30 +340,23 @@ function cmdFetchContent(locationJson) { const bodyFile = tmpFile("body.md"); fs.writeFileSync(bodyFile, comment.body || ""); - console.log( - JSON.stringify( - { - type, - comment_node_id: comment.id, - discussion_node_id: discussionId, - reply_to_node_id: comment.replyTo?.id ?? null, - discussion_number: Number(discussionNumber), - discussion_comment_db_id: Number(discussionCommentDbId), - author: comment.author?.login, - html_url: comment.url || commentUrl, - edit_history_count: comment.userContentEdits?.totalCount ?? 0, - body_file: bodyFile, - }, - null, - 2, - ), - ); + result = { + type, + comment_node_id: comment.id, + discussion_node_id: discussionId, + reply_to_node_id: comment.replyTo?.id ?? null, + discussion_number: Number(discussionNumber), + discussion_comment_db_id: Number(discussionCommentDbId), + author: comment.author?.login, + html_url: comment.url || commentUrl, + edit_history_count: comment.userContentEdits?.totalCount ?? 0, + body_file: bodyFile, + }; } else if ( type === "issue_comment" || type === "pull_request_comment" || type === "pull_request_review_comment" ) { - // Extract comment ID from URL const commentUrl = details.issue_comment_url || details.pull_request_comment_url || @@ -368,150 +369,62 @@ function cmdFetchContent(locationJson) { const bodyFile = tmpFile("body.md"); fs.writeFileSync(bodyFile, comment.body || ""); - // Fetch edit history const nodeId = comment.node_id; const typeName = type === "pull_request_review_comment" ? "PullRequestReviewComment" : "IssueComment"; - const gql = ghGraphQL(`{ - node(id: "${nodeId}") { - ... on ${typeName} { - userContentEdits(first: 50) { - totalCount - } - } - } - }`); - const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0; + const editCount = contentEditCount(nodeId, typeName); - // Extract issue number from html_url const htmlUrl = comment.html_url || details.html_url || ""; const issueMatch = htmlUrl.match(/\/(issues|pull)\/(\d+)/); const issueNumber = issueMatch ? issueMatch[2] : null; - console.log( - JSON.stringify( - { - type, - comment_id: comment.id, - node_id: nodeId, - author: comment.user?.login, - issue_number: issueNumber, - html_url: htmlUrl, - edit_history_count: editCount, - body_file: bodyFile, - }, - null, - 2, - ), - ); - } else if (type === "issue_body") { - const issueUrl = details.issue_body_url || details.issue_url; - if (!issueUrl) { - fail("No issue URL in location details"); + result = { + type, + comment_id: comment.id, + node_id: nodeId, + author: comment.user?.login, + issue_number: issueNumber, + html_url: htmlUrl, + edit_history_count: editCount, + body_file: bodyFile, + }; + } else if (isBodyLocationType(type)) { + const isPullRequest = type === "pull_request_body"; + const url = isPullRequest + ? details.pull_request_body_url || details.pull_request_url + : details.issue_body_url || details.issue_url; + if (!url) { + fail(`No ${isPullRequest ? "PR" : "issue"} URL in location details`); } - const issue = gh(["api", issueUrl]); + const content = gh(["api", url]); const bodyFile = tmpFile("body.md"); - fs.writeFileSync(bodyFile, issue.body || ""); - - const nodeId = issue.node_id; - const number = issue.number; - const gql = ghGraphQL(`{ - node(id: "${nodeId}") { - ... on Issue { - userContentEdits(first: 50) { - totalCount - } - } - } - }`); - const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0; - - console.log( - JSON.stringify( - { - type, - issue_number: number, - node_id: nodeId, - author: issue.user?.login, - html_url: issue.html_url, - edit_history_count: editCount, - body_file: bodyFile, - }, - null, - 2, - ), - ); - } else if (type === "pull_request_body") { - const prUrl = details.pull_request_body_url || details.pull_request_url; - if (!prUrl) { - fail("No PR URL in location details"); - } - - const pr = gh(["api", prUrl]); - const bodyFile = tmpFile("body.md"); - fs.writeFileSync(bodyFile, pr.body || ""); - - const nodeId = pr.node_id; - const number = pr.number; - const gql = ghGraphQL(`{ - node(id: "${nodeId}") { - ... on PullRequest { - userContentEdits(first: 50) { - totalCount - } - } - } - }`); - const editCount = gql?.data?.node?.userContentEdits?.totalCount ?? 0; - - console.log( - JSON.stringify( - { - type, - pr_number: number, - node_id: nodeId, - author: pr.user?.login, - merged: pr.merged, - state: pr.state, - html_url: pr.html_url, - edit_history_count: editCount, - body_file: bodyFile, - }, - null, - 2, - ), - ); + fs.writeFileSync(bodyFile, content.body || ""); + const editCount = contentEditCount(content.node_id, isPullRequest ? "PullRequest" : "Issue"); + result = { + type, + [isPullRequest ? "pr_number" : "issue_number"]: content.number, + node_id: content.node_id, + author: content.user?.login, + ...(isPullRequest ? { merged: content.merged, state: content.state } : {}), + html_url: content.html_url, + edit_history_count: editCount, + body_file: bodyFile, + }; } else if (type === "commit") { - console.log( - JSON.stringify( - { - type, - commit_sha: details.commit_sha, - path: details.path, - start_line: details.start_line, - end_line: details.end_line, - html_url: details.html_url || details.commit_url || details.blob_url || null, - // No body file for commits - body_file: null, - }, - null, - 2, - ), - ); + result = { + type, + commit_sha: details.commit_sha, + path: details.path, + start_line: details.start_line, + end_line: details.end_line, + html_url: details.html_url || details.commit_url || details.blob_url || null, + body_file: null, + }; } else { - console.log( - JSON.stringify( - { - type, - unsupported: true, - details, - }, - null, - 2, - ), - ); + result = { type, unsupported: true, details }; } + console.log(JSON.stringify(result, null, 2)); } /** @@ -720,11 +633,8 @@ function cmdNotify(target, author, locationType, secretTypes, replyToNodeId) { "**Please rotate these credentials immediately.**", "", "These secrets were publicly exposed and should be considered compromised.", - ] - .filter((line) => line !== undefined) - .join("\n"); + ].join("\n"); - // Discussion comments must be notified via GraphQL if (locationType === "discussion_comment") { const newComment = createDiscussionComment(target, body, replyToNodeId); console.log( @@ -737,7 +647,6 @@ function cmdNotify(target, author, locationType, secretTypes, replyToNodeId) { return; } - // Issue/PR comments via REST const bodyFile = tmpFile("notify.md"); fs.writeFileSync(bodyFile, body); @@ -807,7 +716,6 @@ function cmdListOpen() { "--slurp", ]); - // --slurp 将分页结果合并为 [[page1], [page2], ...] 需要 flat const flat = Array.isArray(alerts?.[0]) ? alerts.flat() : Array.isArray(alerts) ? alerts : []; const rows = flat.map((a) => ({ number: a.number, @@ -888,29 +796,23 @@ function cmdSummary(jsonFile) { console.log(lines.join("\n")); } -// ─── Dispatch ─────────────────────────────────────────────────────────────── - -const args = []; - const commands = { - "fetch-alert": () => cmdFetchAlert(args[0]), - "fetch-content": () => cmdFetchContent(args[0]), - "redact-body": () => cmdRedactBody(args[0], args[1], args[2]), - "redact-body-if-needed": () => cmdRedactBodyIfNeeded(args[0], args[1], args[2], args[3], args[4]), - "delete-comment": () => cmdDeleteComment(args[0]), - "delete-discussion-comment": () => cmdDeleteDiscussionComment(args[0]), - "recreate-comment": () => cmdRecreateComment(args[0], args[1]), - "recreate-discussion-comment": () => cmdRecreateDiscussionComment(args[0], args[1], args[2]), - notify: () => cmdNotify(args[0], args[1], args[2], args[3], args[4]), - resolve: () => cmdResolve(args[0], args[1], args[2]), - "list-open": () => cmdListOpen(), - summary: () => cmdSummary(args[0]), + "fetch-alert": cmdFetchAlert, + "fetch-content": cmdFetchContent, + "redact-body": cmdRedactBody, + "redact-body-if-needed": cmdRedactBodyIfNeeded, + "delete-comment": cmdDeleteComment, + "delete-discussion-comment": cmdDeleteDiscussionComment, + "recreate-comment": cmdRecreateComment, + "recreate-discussion-comment": cmdRecreateDiscussionComment, + notify: cmdNotify, + resolve: cmdResolve, + "list-open": cmdListOpen, + summary: cmdSummary, }; function main(argv = process.argv.slice(2)) { const [command, ...commandArgs] = argv; - args.length = 0; - args.push(...commandArgs); if (!command || !commands[command]) { console.error( @@ -935,7 +837,7 @@ function main(argv = process.argv.slice(2)) { process.exit(1); } - commands[command](); + commands[command](...commandArgs); } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/.agents/skills/telegram-e2e-userbot/scripts/followup-drain-control-preload.mjs b/.agents/skills/telegram-e2e-userbot/scripts/followup-drain-control-preload.mjs index 5cb77c46b084..87ef84515c51 100644 --- a/.agents/skills/telegram-e2e-userbot/scripts/followup-drain-control-preload.mjs +++ b/.agents/skills/telegram-e2e-userbot/scripts/followup-drain-control-preload.mjs @@ -31,7 +31,8 @@ if (commandPath && statusPath) { inFlight: queue?.inFlight.size ?? 0, }; }; - const writeStatus = (value) => { + const writeStatus = (seq, command) => { + const value = { seq, command, status: "completed", ...queueState(wrappedKey) }; const pending = `${statusPath}.${process.pid}.tmp`; fs.writeFileSync(pending, `${JSON.stringify(value)}\n`, { mode: 0o600 }); fs.renameSync(pending, statusPath); @@ -59,35 +60,20 @@ if (commandPath && statusPath) { if (!heldOnce) { heldOnce = true; if (waitSeq) { - writeStatus({ - seq: waitSeq, - command: "waitHeld", - status: "completed", - ...queueState(wrappedKey), - }); + writeStatus(waitSeq, "waitHeld"); } await gate.promise; } return originalCallback(run); }); - writeStatus({ - seq: command.seq, - command: "arm", - status: "completed", - ...queueState(wrappedKey), - }); + writeStatus(command.seq, "arm"); return; } if (!wrappedKey) return; lastSeq = command.seq; if (command.command === "waitHeld") { if (heldOnce) { - writeStatus({ - seq: command.seq, - command: "waitHeld", - status: "completed", - ...queueState(wrappedKey), - }); + writeStatus(command.seq, "waitHeld"); } else { waitSeq = command.seq; } @@ -95,12 +81,7 @@ if (commandPath && statusPath) { } if (command.command === "release") { gate.resolve(); - writeStatus({ - seq: command.seq, - command: "release", - status: "completed", - ...queueState(wrappedKey), - }); + writeStatus(command.seq, "release"); return; } }; diff --git a/.agents/skills/telegram-e2e-userbot/scripts/triage-mock-openai.mjs b/.agents/skills/telegram-e2e-userbot/scripts/triage-mock-openai.mjs index 702c2a2d1d98..bbf0e57c6bf5 100644 --- a/.agents/skills/telegram-e2e-userbot/scripts/triage-mock-openai.mjs +++ b/.agents/skills/telegram-e2e-userbot/scripts/triage-mock-openai.mjs @@ -13,56 +13,48 @@ function writeJson(response, status, body) { response.end(JSON.stringify(body)); } -function writeEvents(response, events) { +function startEvents(response) { response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-store", connection: "keep-alive", }); +} + +function completionChunk(id, delta, finishReason = null) { + return { + id, + object: "chat.completion.chunk", + choices: [{ index: 0, delta, finish_reason: finishReason }], + }; +} + +function writeEvents(response, events) { + startEvents(response); for (const event of events) response.write(`data: ${JSON.stringify(event)}\n\n`); response.end("data: [DONE]\n\n"); } function writeInterleavedMonologue(response) { + const id = "chatcmpl_interleaved_monologue"; writeEvents(response, [ - { - id: "chatcmpl_interleaved_monologue", - object: "chat.completion.chunk", - choices: [ - { - index: 0, - delta: { - role: "assistant", - reasoning_details: [ - { type: "response.output_text", text: "PRIVATE_MONOLOGUE" }, - { type: "reasoning.text", text: "HIDDEN_REASONING" }, - { type: "response.text", text: "PUBLIC_FINAL" }, - ], - }, - finish_reason: null, - }, + completionChunk(id, { + role: "assistant", + reasoning_details: [ + { type: "response.output_text", text: "PRIVATE_MONOLOGUE" }, + { type: "reasoning.text", text: "HIDDEN_REASONING" }, + { type: "response.text", text: "PUBLIC_FINAL" }, ], - }, - { - id: "chatcmpl_interleaved_monologue", - object: "chat.completion.chunk", - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - }, + }), + completionChunk(id, {}, "stop"), ]); } function writeIncompleteToolUse(response) { + const id = "chatcmpl_incomplete_tool_use"; writeEvents(response, [ - { - id: "chatcmpl_incomplete_tool_use", - object: "chat.completion.chunk", - choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], - }, - { - id: "chatcmpl_incomplete_tool_use", - object: "chat.completion.chunk", - choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], - }, + completionChunk(id, { role: "assistant" }), + completionChunk(id, {}, "tool_calls"), ]); } @@ -78,18 +70,10 @@ async function writePreviewToolBoundary(response, body) { writeJson(response, 400, { error: { message: "Preview fixture requires streaming and exec" } }); return; } - response.writeHead(200, { - "content-type": "text/event-stream", - "cache-control": "no-store", - connection: "keep-alive", - }); + startEvents(response); const writeChunk = (delta, finishReason = null) => response.write( - `data: ${JSON.stringify({ - id: `chatcmpl_preview_${toolResults}`, - object: "chat.completion.chunk", - choices: [{ index: 0, delta, finish_reason: finishReason }], - })}\n\n`, + `data: ${JSON.stringify(completionChunk(`chatcmpl_preview_${toolResults}`, delta, finishReason))}\n\n`, ); writeChunk({ role: "assistant" }); if (toolResults === 0) { @@ -119,63 +103,82 @@ async function writePreviewToolBoundary(response, body) { response.end("data: [DONE]\n\n"); } -async function writeStreamingThrottle(response) { - const itemId = "msg_streaming_throttle_107179"; - const finalText = "STREAM_FINAL_107179"; - response.writeHead(200, { - "content-type": "text/event-stream", - "cache-control": "no-store", - connection: "keep-alive", - }); - const previewEvents = [ +function messageStartEvents(item, deltas = [item.content[0].text]) { + return [ { type: "response.output_item.added", output_index: 0, - item: { - type: "message", - id: itemId, - role: "assistant", - phase: "final_answer", - status: "in_progress", - content: [], - }, + item: { ...item, status: "in_progress", content: [] }, }, - ...["QA streaming ", "preview in ", "progress"].map((delta) => ({ + ...deltas.map((delta) => ({ type: "response.output_text.delta", - item_id: itemId, + item_id: item.id, output_index: 0, content_index: 0, delta, })), ]; - for (const event of previewEvents) response.write(`data: ${JSON.stringify(event)}\n\n`); - await new Promise((resolve) => setTimeout(resolve, 1_500)); +} + +function messageDoneEvents(item) { + return [ + { + type: "response.output_text.done", + item_id: item.id, + output_index: 0, + content_index: 0, + text: item.content[0].text, + }, + { type: "response.output_item.done", output_index: 0, item }, + ]; +} + +function functionCallEvents(item, outputIndex = 0) { + return [ + { + type: "response.output_item.added", + output_index: outputIndex, + item: { ...item, arguments: "" }, + }, + { + type: "response.function_call_arguments.delta", + item_id: item.id, + output_index: outputIndex, + delta: item.arguments, + }, + { type: "response.output_item.done", output_index: outputIndex, item }, + ]; +} + +function completedEvent(id, output) { + return { + type: "response.completed", + response: { + id, + status: "completed", + output, + usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, + }, + }; +} + +async function writeStreamingThrottle(response) { const item = { type: "message", - id: itemId, + id: "msg_streaming_throttle_107179", role: "assistant", phase: "final_answer", status: "completed", - content: [{ type: "output_text", text: finalText, annotations: [] }], + content: [{ type: "output_text", text: "STREAM_FINAL_107179", annotations: [] }], }; + startEvents(response); + for (const event of messageStartEvents(item, ["QA streaming ", "preview in ", "progress"])) { + response.write(`data: ${JSON.stringify(event)}\n\n`); + } + await new Promise((resolve) => setTimeout(resolve, 1_500)); for (const event of [ - { - type: "response.output_text.done", - item_id: itemId, - output_index: 0, - content_index: 0, - text: finalText, - }, - { type: "response.output_item.done", output_index: 0, item }, - { - type: "response.completed", - response: { - id: "resp_streaming_throttle_107179", - status: "completed", - output: [item], - usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, - }, - }, + ...messageDoneEvents(item), + completedEvent("resp_streaming_throttle_107179", [item]), ]) { response.write(`data: ${JSON.stringify(event)}\n\n`); } @@ -191,67 +194,27 @@ function responseEvents(text) { content: [{ type: "output_text", text, annotations: [] }], }; return [ - { - type: "response.output_item.added", - output_index: 0, - item: { ...item, status: "in_progress", content: [] }, - }, - { - type: "response.output_text.delta", - item_id: item.id, - output_index: 0, - content_index: 0, - delta: text, - }, - { - type: "response.output_text.done", - item_id: item.id, - output_index: 0, - content_index: 0, - text, - }, - { type: "response.output_item.done", output_index: 0, item }, - { - type: "response.completed", - response: { - id: "resp_telegram_triage_fixture", - status: "completed", - output: [item], - usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, - }, - }, + ...messageStartEvents(item), + ...messageDoneEvents(item), + completedEvent("resp_telegram_triage_fixture", [item]), ]; } +function functionCallResponse(name, argumentsText, suffix) { + const item = { + type: "function_call", + id: `fc_${name}_${suffix}`, + call_id: `call_${name}_${suffix}`, + name, + arguments: argumentsText, + }; + return [...functionCallEvents(item), completedEvent(`resp_${name}_${suffix}`, [item])]; +} + function toolCallEvents(sequence) { const args = JSON.stringify({ args: { id: "session_status", args: {} } }); const suffix = createHash("sha256").update(`${sequence}:${args}`).digest("hex").slice(0, 10); - const item = { - type: "function_call", - id: `fc_tool_call_${suffix}`, - call_id: `call_tool_call_${suffix}`, - name: "tool_call", - arguments: args, - }; - return [ - { type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } }, - { - type: "response.function_call_arguments.delta", - item_id: item.id, - output_index: 0, - delta: args, - }, - { type: "response.output_item.done", output_index: 0, item }, - { - type: "response.completed", - response: { - id: `resp_tool_call_${suffix}`, - status: "completed", - output: [item], - usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, - }, - }, - ]; + return functionCallResponse("tool_call", args, suffix); } function namedToolCallEvents(name, args, sequence) { @@ -260,32 +223,7 @@ function namedToolCallEvents(name, args, sequence) { .update(`${name}:${sequence}:${argumentsText}`) .digest("hex") .slice(0, 10); - const item = { - type: "function_call", - id: `fc_${name}_${suffix}`, - call_id: `call_${name}_${suffix}`, - name, - arguments: argumentsText, - }; - return [ - { type: "response.output_item.added", output_index: 0, item: { ...item, arguments: "" } }, - { - type: "response.function_call_arguments.delta", - item_id: item.id, - output_index: 0, - delta: argumentsText, - }, - { type: "response.output_item.done", output_index: 0, item }, - { - type: "response.completed", - response: { - id: `resp_${name}_${suffix}`, - status: "completed", - output: [item], - usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, - }, - }, - ]; + return functionCallResponse(name, argumentsText, suffix); } function hasTool(body, name) { @@ -300,56 +238,18 @@ function draftThenExecEvents() { status: "completed", content: [{ type: "output_text", text: "GOOD_DRAFT_115041", annotations: [] }], }; - const argumentsText = JSON.stringify({ command: "printf tool-ok" }); const call = { type: "function_call", id: "fc_exec_115041", call_id: "call_exec_115041", name: "exec", - arguments: argumentsText, + arguments: JSON.stringify({ command: "printf tool-ok" }), }; return [ - { - type: "response.output_item.added", - output_index: 0, - item: { ...message, status: "in_progress", content: [] }, - }, - { - type: "response.output_text.delta", - item_id: message.id, - output_index: 0, - content_index: 0, - delta: "GOOD_DRAFT_115041", - }, - { - type: "response.output_text.done", - item_id: message.id, - output_index: 0, - content_index: 0, - text: "GOOD_DRAFT_115041", - }, - { type: "response.output_item.done", output_index: 0, item: message }, - { - type: "response.output_item.added", - output_index: 1, - item: { ...call, arguments: "" }, - }, - { - type: "response.function_call_arguments.delta", - item_id: call.id, - output_index: 1, - delta: argumentsText, - }, - { type: "response.output_item.done", output_index: 1, item: call }, - { - type: "response.completed", - response: { - id: "resp_good_draft_115041", - status: "completed", - output: [message, call], - usage: { input_tokens: 32, output_tokens: 8, total_tokens: 40 }, - }, - }, + ...messageStartEvents(message), + ...messageDoneEvents(message), + ...functionCallEvents(call, 1), + completedEvent("resp_good_draft_115041", [message, call]), ]; } diff --git a/.agents/skills/telegram-e2e-userbot/scripts/user-driver.py b/.agents/skills/telegram-e2e-userbot/scripts/user-driver.py index c31288c43786..49257e26a2d5 100755 --- a/.agents/skills/telegram-e2e-userbot/scripts/user-driver.py +++ b/.agents/skills/telegram-e2e-userbot/scripts/user-driver.py @@ -324,7 +324,6 @@ class TdClient: self.lib.td_execute(json.dumps({"@type": "setLogVerbosityLevel", "new_verbosity_level": 0}).encode()) self.client = self.lib.td_json_client_create() self.extra = 0 - self.pending = {} self.users = {} self.updates = [] atexit.register(self.destroy) @@ -361,7 +360,6 @@ class TdClient: extra = str(self.extra) payload = dict(payload) payload["@extra"] = extra - self.pending[extra] = payload["@type"] self.lib.td_json_client_send(self.client, json.dumps(payload).encode()) return extra @@ -399,17 +397,13 @@ class TdClient: tdlib_method=payload["@type"], ) return item - self.handle_update(item) + self.updates.append(item) raise DriverError( f"Timed out waiting for {payload['@type']}", tdlib_method=payload["@type"], tdlib_timed_out=True, ) - def handle_update(self, item): - self.updates.append(item) - return item - def next_update(self, timeout=1.0): if self.updates: return self.updates.pop(0) @@ -915,10 +909,7 @@ def print_result(payload, as_json=False, output=""): output_path = Path(output).expanduser() output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") - if as_json: - print(json.dumps(payload, indent=2, sort_keys=True)) - else: - print(json.dumps(payload, indent=2, sort_keys=True)) + print(json.dumps(payload, indent=2, sort_keys=True)) def command_configure(args): @@ -1719,59 +1710,51 @@ def main(): parser = argparse.ArgumentParser(description="Telegram real-user E2E driver backed by TDLib.") sub = parser.add_subparsers(dest="command", required=True) - configure = sub.add_parser("configure") + def command_parser(name, func, common=True): + command = sub.add_parser(name) + if common: + add_common(command) + command.set_defaults(func=func) + return command + + configure = command_parser("configure", command_configure, common=False) configure.add_argument("--api-id") configure.add_argument("--api-hash") configure.add_argument("--tdlib-path") configure.add_argument("--chat") configure.add_argument("--sut-username") configure.add_argument("--sut-id") - configure.set_defaults(func=command_configure) - doctor = sub.add_parser("doctor") + doctor = command_parser("doctor", command_doctor, common=False) doctor.add_argument("--json", action="store_true") doctor.add_argument("--output", default="") - doctor.set_defaults(func=command_doctor) - login = sub.add_parser("login") - add_common(login) + login = command_parser("login", command_login) login.add_argument("--qr", action="store_true", default=True) login.add_argument("--phone", default="") login.add_argument("--code", default="") login.add_argument("--password", default="") login.add_argument("--first-name", default="") login.add_argument("--last-name", default="") - login.set_defaults(func=command_login) - status = sub.add_parser("status") - add_common(status) + status = command_parser("status", command_status) status.add_argument("--check-chat", default="") status.add_argument("--require-chat", default="") - status.set_defaults(func=command_status) - resolve_chat = sub.add_parser("resolve-chat") - add_common(resolve_chat) + resolve_chat = command_parser("resolve-chat", command_resolve_chat) resolve_chat.add_argument("--chat", required=True) - resolve_chat.set_defaults(func=command_resolve_chat) for name in ("prepare-group", "cleanup-group"): - group = sub.add_parser(name) - add_common(group) + group = command_parser(name, command_test_group) group.add_argument("--chat", default="") - group.set_defaults(func=command_test_group) for name in ("prepare-private-forum", "cleanup-private-forum"): - private_forum = sub.add_parser(name) - add_common(private_forum) - private_forum.set_defaults(func=command_private_forum) + command_parser(name, command_private_forum) - confirm_qr = sub.add_parser("confirm-qr") - add_common(confirm_qr) + confirm_qr = command_parser("confirm-qr", command_confirm_qr) confirm_qr.add_argument("--link", required=True) - confirm_qr.set_defaults(func=command_confirm_qr) - send = sub.add_parser("send") - add_common(send) + send = command_parser("send", command_send) send.add_argument("--chat", default="") send.add_argument("--text") send.add_argument("--photo", action="append", default=[]) @@ -1779,20 +1762,16 @@ def main(): send.add_argument("--reply-to") send.add_argument("--thread-id", type=int, default=0) send.add_argument("--forum-topic-id", type=int) - send.set_defaults(func=command_send) - wait = sub.add_parser("wait") - add_common(wait) + wait = command_parser("wait", command_wait) wait.add_argument("--chat", default="") wait.add_argument("--expect", action="append", default=[]) wait.add_argument("--from-bot", default="") wait.add_argument("--reply-to") wait.add_argument("--thread-id", type=int, default=0) wait.add_argument("--after-message-id", type=int, default=0) - wait.set_defaults(func=command_wait) - probe = sub.add_parser("probe") - add_common(probe) + probe = command_parser("probe", command_probe) probe.add_argument("--chat", default="") probe.add_argument("--text", default="@{sut} Reply exactly: USER-E2E-{run}") probe.add_argument("--photo", action="append", default=[]) @@ -1803,24 +1782,18 @@ def main(): probe.add_argument("--thread-id", type=int, default=0) probe.add_argument("--require-reply", action="store_true", default=True) probe.add_argument("--any-sut-reply", dest="require_reply", action="store_false") - probe.set_defaults(func=command_probe) - transcript = sub.add_parser("transcript") - add_common(transcript) + transcript = command_parser("transcript", command_transcript) transcript.add_argument("--chat", default="") transcript.add_argument("--limit", type=int, default=20) - transcript.set_defaults(func=command_transcript) - chats = sub.add_parser("chats") - add_common(chats) + chats = command_parser("chats", command_chats) chats.add_argument("--limit", type=int, default=50) - chats.set_defaults(func=command_chats) - serve = sub.add_parser("serve") + serve = command_parser("serve", command_serve, common=False) serve.add_argument("--chat", default="") serve.add_argument("--observe-chat", action="append", default=[]) serve.add_argument("--timeout-ms", type=int, default=120000) - serve.set_defaults(func=command_serve) args = parser.parse_args() if args.command == "send" and not args.text and not args.photo: diff --git a/.agents/skills/telegram-e2e-userbot/scripts/user-record.py b/.agents/skills/telegram-e2e-userbot/scripts/user-record.py index 383a31fc6226..b38eb7f7618e 100755 --- a/.agents/skills/telegram-e2e-userbot/scripts/user-record.py +++ b/.agents/skills/telegram-e2e-userbot/scripts/user-record.py @@ -51,10 +51,6 @@ def message_text(message): return content_text(message.get("content") or {}) -def content_kind(message): - return (message.get("content") or {}).get("@type", "") - - class EventRecorder: def __init__(self, client, chat_id, record_path, sut_user_id=None): self.client = client @@ -156,7 +152,7 @@ class EventRecorder: "senderId": sender, "isSut": self.sut_user_id is not None and sender == self.sut_user_id, "isOutgoing": bool(message.get("is_outgoing")), - "contentType": content_kind(message), + "contentType": content.get("@type", ""), "textLen": len(text), "text": text, "richMessageIsFull": rich_message.get("is_full") if isinstance(rich_message, dict) else None, @@ -214,13 +210,14 @@ class EventRecorder: # Ack and status reactions arrive here, on the *user's own* message. # A bot reacting to its own message produces no update for the user, # so probe this by reacting to a message the QA user sent. - reactions = ( - ((update.get("interaction_info") or {}).get("reactions") or {}).get("reactions") or [] - ) + reactions = [ + reaction + for reaction in ((update.get("interaction_info") or {}).get("reactions") or {}).get("reactions") or [] + if isinstance(reaction, dict) + ] emojis = "".join( (reaction.get("type") or {}).get("emoji", "") for reaction in reactions - if isinstance(reaction, dict) ) yield ( "reaction", @@ -232,11 +229,8 @@ class EventRecorder: "reactionCount": sum( int(reaction.get("total_count") or 0) for reaction in reactions - if isinstance(reaction, dict) ), - "reactionTypes": [ - reaction.get("type") for reaction in reactions if isinstance(reaction, dict) - ], + "reactionTypes": [reaction.get("type") for reaction in reactions], }, ) elif kind == "updateChatAction": @@ -306,23 +300,12 @@ class EventRecorder: "elapsedMs": e["elapsedMs"], "kind": e["kind"], "messageId": e["messageId"], - "botApiMessageId": e.get("botApiMessageId"), - "textLen": e.get("textLen"), - "contentType": e.get("contentType"), - "senderId": e.get("senderId"), - "isSut": e.get("isSut"), - "isOutgoing": e.get("isOutgoing"), - "replyToMessageId": e.get("replyToMessageId"), - "quoteText": e.get("quoteText"), - "topicType": e.get("topicType"), - "topicId": e.get("topicId"), - "reactionText": e.get("reactionText"), - "reactionCount": e.get("reactionCount"), - "actionType": e.get("actionType"), - "status": e.get("status"), - "buttonText": e.get("buttonText"), - "durationMs": e.get("durationMs"), - "error": e.get("error"), + **{key: e.get(key) for key in ( + "botApiMessageId", "textLen", "contentType", "senderId", "isSut", + "isOutgoing", "replyToMessageId", "quoteText", "topicType", "topicId", + "reactionText", "reactionCount", "actionType", "status", "buttonText", + "durationMs", "error", + )}, } for e in self.events ], diff --git a/scripts/dev/discord-acp-plain-language-smoke.ts b/scripts/dev/discord-acp-plain-language-smoke.ts index 6e7cda1cb885..fb4f643d506c 100644 --- a/scripts/dev/discord-acp-plain-language-smoke.ts +++ b/scripts/dev/discord-acp-plain-language-smoke.ts @@ -215,24 +215,21 @@ function isTooLargeError(error: unknown): boolean { return (error as NodeJS.ErrnoException | undefined)?.code === "ETOOBIG"; } -async function readDiscordResponseText(params: { +type DiscordResponseRead = { response: Response; label: string; signal: AbortSignal; maxBytes: number; -}): Promise { +}; + +async function readDiscordResponseText(params: DiscordResponseRead): Promise { return await readBoundedResponseText(params.response, params.label, params.maxBytes, { createTooLargeError: createDiscordResponseTooLargeError, signal: params.signal, }); } -async function readDiscordResponseJson(params: { - response: Response; - label: string; - signal: AbortSignal; - maxBytes: number; -}): Promise { +async function readDiscordResponseJson(params: DiscordResponseRead): Promise { const text = await readDiscordResponseText(params); if (!text) { return {}; @@ -361,79 +358,32 @@ function usage(): string { function parseArgs(argv = process.argv.slice(2)): Args { validateCliArgs(argv); - const channelId = - resolveArg("--channel", argv) || process.env.OPENCLAW_DISCORD_SMOKE_CHANNEL_ID || ""; - const driverModeRaw = - resolveArg("--driver", argv) || process.env.OPENCLAW_DISCORD_SMOKE_DRIVER || "token"; - const driverMode = parseDriverMode(driverModeRaw); - const driverToken = - resolveArg("--token", argv) || process.env.OPENCLAW_DISCORD_SMOKE_DRIVER_TOKEN || ""; - const driverTokenPrefix = - resolveArg("--token-prefix", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_DRIVER_TOKEN_PREFIX || - "Bot"; - const botToken = - resolveArg("--bot-token", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_BOT_TOKEN || - process.env.DISCORD_BOT_TOKEN || - ""; - const botTokenPrefix = - resolveArg("--bot-token-prefix", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_BOT_TOKEN_PREFIX || - "Bot"; - const targetAgent = - resolveArg("--agent", argv) || process.env.OPENCLAW_DISCORD_SMOKE_AGENT || "codex"; - const mentionUserId = - resolveArg("--mention", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_MENTION_USER_ID || - undefined; - const instruction = - resolveArg("--instruction", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_INSTRUCTION || - undefined; - const timeoutMs = parseNumber( - resolveArg("--timeout-ms", argv) || process.env.OPENCLAW_DISCORD_SMOKE_TIMEOUT_MS, - 240_000, - "--timeout-ms", - ); - const pollMs = parseNumber( - resolveArg("--poll-ms", argv) || process.env.OPENCLAW_DISCORD_SMOKE_POLL_MS, - 1_500, - "--poll-ms", - ); - const stateDir = path.resolve(resolveArg("--state-dir", argv) || resolveStateDir()); - const openclawBin = - resolveArg("--openclaw-bin", argv) || - process.env.OPENCLAW_DISCORD_SMOKE_OPENCLAW_BIN || - "openclaw"; - const json = hasFlag("--json", argv); - - if (!channelId) { - throw new Error(usage()); - } - if (driverMode === "token" && !driverToken) { - throw new Error(usage()); - } - if (driverMode === "webhook" && !botToken) { - throw new Error(usage()); - } - - return { - channelId, - driverMode, - driverToken, - driverTokenPrefix, - botToken, - botTokenPrefix, - targetAgent, - timeoutMs, - pollMs, - mentionUserId, - instruction, - stateDir, - openclawBin, - json, + const option = (flag: string, env: string, fallback = "") => + resolveArg(flag, argv) || process.env[`OPENCLAW_DISCORD_SMOKE_${env}`] || fallback; + const args: Args = { + channelId: option("--channel", "CHANNEL_ID"), + driverMode: parseDriverMode(option("--driver", "DRIVER", "token")), + driverToken: option("--token", "DRIVER_TOKEN"), + driverTokenPrefix: option("--token-prefix", "DRIVER_TOKEN_PREFIX", "Bot"), + botToken: option("--bot-token", "BOT_TOKEN", process.env.DISCORD_BOT_TOKEN || ""), + botTokenPrefix: option("--bot-token-prefix", "BOT_TOKEN_PREFIX", "Bot"), + targetAgent: option("--agent", "AGENT", "codex"), + mentionUserId: option("--mention", "MENTION_USER_ID") || undefined, + instruction: option("--instruction", "INSTRUCTION") || undefined, + timeoutMs: parseNumber(option("--timeout-ms", "TIMEOUT_MS"), 240_000, "--timeout-ms"), + pollMs: parseNumber(option("--poll-ms", "POLL_MS"), 1_500, "--poll-ms"), + stateDir: path.resolve(resolveArg("--state-dir", argv) || resolveStateDir()), + openclawBin: option("--openclaw-bin", "OPENCLAW_BIN", "openclaw"), + json: hasFlag("--json", argv), }; + if ( + !args.channelId || + (args.driverMode === "token" && !args.driverToken) || + (args.driverMode === "webhook" && !args.botToken) + ) { + throw new Error(usage()); + } + return args; } async function openclawCliJson(params: { @@ -575,25 +525,30 @@ async function requestDiscordJson(params: { timeoutError, onTimeout: () => controller.abort(), }); - - if (response.status === 429) { - const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError); - const body = (await withTimeout({ - operation: readDiscordResponseJson({ + const readBody = (reader: (input: DiscordResponseRead) => Promise): Promise => { + const timeoutMs = remainingTimeoutMs(deadlineMs, timeoutError); + return withTimeout({ + operation: reader({ response, label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`, signal: controller.signal, maxBytes: responseBodyMaxBytes, - }).catch((error: unknown) => { + }), + timeoutMs, + timeoutError, + onTimeout: () => controller.abort(), + }); + }; + + if (response.status === 429) { + const body = (await readBody((input) => + readDiscordResponseJson(input).catch((error: unknown) => { if (isTooLargeError(error)) { throw error; } return {}; }), - timeoutMs: bodyTimeoutMs, - timeoutError, - onTimeout: () => controller.abort(), - })) as { retry_after?: number }; + )) as { retry_after?: number }; const waitSeconds = typeof body.retry_after === "number" ? body.retry_after : 1; const waitMs = Math.ceil(waitSeconds * 1000); const remainingMs = remainingTimeoutMs(deadlineMs, timeoutError); @@ -607,18 +562,7 @@ async function requestDiscordJson(params: { } if (!response.ok) { - const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError); - const text = await withTimeout({ - operation: readDiscordResponseText({ - response, - label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`, - signal: controller.signal, - maxBytes: responseBodyMaxBytes, - }), - timeoutMs: bodyTimeoutMs, - timeoutError, - onTimeout: () => controller.abort(), - }); + const text = await readBody(readDiscordResponseText); throw new Error( redactForDevToolLog( `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)} failed: ${response.status} ${response.statusText}${text ? ` :: ${text}` : ""}`, @@ -630,18 +574,7 @@ async function requestDiscordJson(params: { return undefined as T; } - const bodyTimeoutMs = remainingTimeoutMs(deadlineMs, timeoutError); - return (await withTimeout({ - operation: readDiscordResponseJson({ - response, - label: `${params.errorPrefix} ${params.method} ${redactDiscordApiPath(params.path)}`, - signal: controller.signal, - maxBytes: responseBodyMaxBytes, - }), - timeoutMs: bodyTimeoutMs, - timeoutError, - onTimeout: () => controller.abort(), - })) as T; + return (await readBody(readDiscordResponseJson)) as T; } throw new Error( @@ -706,22 +639,24 @@ function toRecentMessageRow(message: DiscordMessage) { }; } -async function loadParentRecentMessages(params: { +async function loadMessages(params: { args: Args; readAuthHeader: string; + target: string; + limit: number; timeoutMs?: number; }): Promise { if (params.args.driverMode === "openclaw") { return await readMessagesWithOpenclaw({ openclawBin: params.args.openclawBin, - target: params.args.channelId, - limit: 20, + target: params.target, + limit: params.limit, timeoutMs: params.timeoutMs, }); } return await discordApi({ method: "GET", - path: `/channels/${encodeURIComponent(params.args.channelId)}/messages?limit=20`, + path: `/channels/${encodeURIComponent(params.target)}/messages?limit=${params.limit}`, authHeader: params.readAuthHeader, timeoutMs: params.timeoutMs, }); @@ -941,6 +876,31 @@ async function run(argv = process.argv.slice(2)): Promise { + let parentRecent: DiscordMessage[] = []; + try { + parentRecent = await loadMessages({ + args, + readAuthHeader, + target: args.channelId, + limit: 20, + timeoutMs: remainingTimeoutMs(deadline), + }); + } catch { + // Best effort diagnostics only. + } + return { + bindingCandidates: candidates.map((entry) => ({ + threadId: entry.threadId || "", + targetSessionKey: maskIdentifier(entry.targetSessionKey), + targetKind: entry.targetKind, + agentId: entry.agentId, + boundAt: entry.boundAt, + })), + parentChannelRecent: parentRecent.map(toRecentMessageRow), + }; + }; + let winningBinding: ThreadBindingRecord | undefined; let latestCandidates: ThreadBindingRecord[] = []; @@ -965,31 +925,12 @@ async function run(argv = process.argv.slice(2)): Promise ({ - threadId: entry.threadId || "", - targetSessionKey: maskIdentifier(entry.targetSessionKey), - targetKind: entry.targetKind, - agentId: entry.agentId, - boundAt: entry.boundAt, - })), - parentChannelRecent: parentRecent.map(toRecentMessageRow), - }, + diagnostics: await diagnostics(latestCandidates.slice(0, 6)), }; } @@ -997,20 +938,13 @@ async function run(argv = process.argv.slice(2)): Promise({ - method: "GET", - path: `/channels/${encodeURIComponent(threadId)}/messages?limit=50`, - authHeader: readAuthHeader, - timeoutMs: remainingTimeoutMs(deadline), - }); + const threadMessages = await loadMessages({ + args, + readAuthHeader, + target: threadId, + limit: 50, + timeoutMs: remainingTimeoutMs(deadline), + }); ackMessage = threadMessages.find((message) => { const content = message.content || ""; if (!content.includes(ackToken)) { @@ -1028,34 +962,12 @@ async function run(argv = process.argv.slice(2)): Promise; type GatewaySmokeCliOptions = { help: boolean; token?: string; @@ -131,18 +130,6 @@ function connectHelloScopes(response: unknown): string[] | null { return payload.auth.scopes; } -function hasConnectHelloPayload(response: unknown): boolean { - return connectHelloScopes(response) !== null; -} - -function hasUnpairedOperatorScopes(response: unknown): boolean { - const scopes = connectHelloScopes(response); - if (!scopes) { - return false; - } - return scopes.length > 0; -} - export async function runGatewaySmoke( input: { token: string; urlRaw: string }, deps: GatewaySmokeDeps = {}, @@ -151,13 +138,7 @@ export async function runGatewaySmoke( const createClient = deps.createClient ?? createGatewayWsClient; const stderr = deps.stderr ?? writeStderrLine; const stdout = deps.stdout ?? writeStdoutLine; - const client: GatewaySmokeClient = createClient({ - url: url.toString(), - onEvent: (evt) => { - // Ignore noisy connect handshakes. - void evt; - }, - }); + const client = createClient({ url: url.toString() }); const { request, waitOpen, close } = client; try { @@ -187,11 +168,12 @@ export async function runGatewaySmoke( stderr(`connect failed: ${String(connectRes.error)}`); return 2; } - if (!hasConnectHelloPayload(connectRes)) { + const scopes = connectHelloScopes(connectRes); + if (scopes === null) { stderr("connect failed: missing hello-ok payload"); return 2; } - if (hasUnpairedOperatorScopes(connectRes)) { + if (scopes.length > 0) { stderr("connect failed: unpaired iOS smoke unexpectedly received operator scopes"); return 2; } @@ -225,7 +207,7 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { if (cli.help) { writeStdoutLine(usage()); } else if (!cli.urlRaw || !cli.token) { - writeUsage(); + writeStderrLine(usage()); process.exitCode = 1; } else { process.exitCode = await runGatewaySmoke({ token: cli.token, urlRaw: cli.urlRaw }); diff --git a/scripts/dev/gateway-ws-client.ts b/scripts/dev/gateway-ws-client.ts deleted file mode 100644 index a807d908ffdc..000000000000 --- a/scripts/dev/gateway-ws-client.ts +++ /dev/null @@ -1,2 +0,0 @@ -// Gateway Ws Client script supports OpenClaw repository automation. -export * from "../lib/gateway-ws-client.ts"; diff --git a/scripts/dev/ios-node-e2e.ts b/scripts/dev/ios-node-e2e.ts index 5b3ba11a5b4f..4cfe06f80788 100644 --- a/scripts/dev/ios-node-e2e.ts +++ b/scripts/dev/ios-node-e2e.ts @@ -101,14 +101,13 @@ if (!urlRaw || !token) { } const waitSeconds = parseWaitSeconds(getArg("--wait-seconds")); -const { createGatewayWsClient, resolveGatewayUrl } = await import("./gateway-ws-client.ts"); +const { createGatewayWsClient, resolveGatewayUrl } = await import("../lib/gateway-ws-client.ts"); const url = resolveGatewayUrl(urlRaw); const isoNow = () => new Date().toISOString(); const isoMinusMs = (ms: number) => new Date(Date.now() - ms).toISOString(); type TestCase = { - id: string; command: string; params?: unknown; timeoutMs?: number; @@ -273,47 +272,39 @@ async function main() { } const tests: TestCase[] = [ - { id: "device.info", command: "device.info" }, - { id: "device.status", command: "device.status" }, + { command: "device.info" }, + { command: "device.status" }, { - id: "system.notify", command: "system.notify", params: { title: "OpenClaw E2E", body: `ios-node-e2e @ ${isoNow()}`, delivery: "system" }, }, { - id: "contacts.search", command: "contacts.search", params: { query: null, limit: 5 }, }, { - id: "calendar.events", command: "calendar.events", params: { startISO: isoMinusMs(6 * 60 * 60 * 1000), endISO: isoNow(), limit: 10 }, }, { - id: "reminders.list", command: "reminders.list", params: { status: "incomplete", limit: 10 }, }, { - id: "motion.pedometer", command: "motion.pedometer", params: { startISO: isoMinusMs(60 * 60 * 1000), endISO: isoNow() }, }, { - id: "photos.latest", command: "photos.latest", params: { limit: 1, maxWidth: 512, quality: 0.7 }, }, { - id: "camera.snap", command: "camera.snap", params: { facing: "back", maxWidth: 768, quality: 0.7, format: "jpeg" }, dangerous: true, timeoutMs: 20_000, }, { - id: "screen.record", command: "screen.record", params: { durationMs: 2_000, fps: 15, includeAudio: false }, dangerous: true, @@ -342,7 +333,7 @@ async function main() { }, (t.timeoutMs ?? 12_000) + 2_000, ).catch((err: unknown) => { - results.push({ id: t.id, ok: false, error: formatErr(err) }); + results.push({ id: t.command, ok: false, error: formatErr(err) }); return null; }); @@ -351,18 +342,18 @@ async function main() { } if (!invokeRes.ok) { - results.push({ id: t.id, ok: false, error: invokeRes.error }); + results.push({ id: t.command, ok: false, error: invokeRes.error }); continue; } const commandPayload = commandPayloadFromInvokePayload(invokeRes.payload); const payloadError = payloadShapeError(t.command, commandPayload); if (payloadError) { - results.push({ id: t.id, ok: false, error: payloadError, payload: invokeRes.payload }); + results.push({ id: t.command, ok: false, error: payloadError, payload: invokeRes.payload }); continue; } - results.push({ id: t.id, ok: true, payload: invokeRes.payload }); + results.push({ id: t.command, ok: true, payload: invokeRes.payload }); } if (jsonOut) { @@ -377,7 +368,6 @@ async function main() { results, }); } else { - const pad = (s: string, n: number) => (s.length >= n ? s : s + " ".repeat(n - s.length)); const rows = results.map((r) => ({ cmd: r.id, ok: r.ok ? "ok" : "fail", @@ -388,7 +378,7 @@ async function main() { writeStdoutLine(`dangerous: ${dangerous ? "on" : "off"}`); writeStdoutLine(); for (const r of rows) { - writeStdoutLine(`${pad(r.cmd, width)} ${pad(r.ok, 4)} ${r.note}`); + writeStdoutLine(`${r.cmd.padEnd(width)} ${r.ok.padEnd(4)} ${r.note}`); } } diff --git a/scripts/dev/realtime-talk-live-smoke.ts b/scripts/dev/realtime-talk-live-smoke.ts index 04d0717d09c6..aecb56f704f1 100644 --- a/scripts/dev/realtime-talk-live-smoke.ts +++ b/scripts/dev/realtime-talk-live-smoke.ts @@ -9,7 +9,6 @@ import { previewForDevToolLog, redactJsonValueForDevToolLog, } from "../lib/dev-tooling-safety.ts"; -import { toErrorObject as toLintErrorObject } from "../lib/error-format.mts"; const OPENAI_REALTIME_MODEL = process.env.OPENCLAW_REALTIME_OPENAI_MODEL?.trim() || "gpt-realtime-2.1"; @@ -971,7 +970,18 @@ async function smokeGoogleLiveBrowserWs(browser: Browser, apiKey: string): Promi } })().catch((error: unknown) => { window.clearTimeout(timeout); - reject(toLintErrorObject(error, "Non-Error rejection")); + // This callback is serialized into the browser without module imports. + if (error instanceof Error) { + reject(error); + } else if (typeof error === "string") { + reject(new Error(error)); + } else { + const failure = new Error("Non-Error rejection", { cause: error }); + if ((typeof error === "object" && error !== null) || typeof error === "function") { + Object.assign(failure, error); + } + reject(failure); + } }); }); ws.addEventListener("error", () => { diff --git a/scripts/docker/install-sh-e2e/run.sh b/scripts/docker/install-sh-e2e/run.sh index 6339a2ab59c3..9276d9a0965b 100755 --- a/scripts/docker/install-sh-e2e/run.sh +++ b/scripts/docker/install-sh-e2e/run.sh @@ -178,31 +178,19 @@ time_phase "Preinstall previous" preinstall_previous_version time_phase "Run official installer one-liner" run_official_installer time_phase "Verify installed version" verify_installed_version -set_image_model() { +set_profile_model() { local profile="$1" - shift + local command="$2" + local label="$3" + shift 3 local candidate for candidate in "$@"; do - if openclaw --profile "$profile" models set-image "$candidate" >/dev/null 2>&1; then + if openclaw --profile "$profile" models "$command" "$candidate" >/dev/null 2>&1; then echo "$candidate" return 0 fi done - echo "ERROR: could not set an image model (tried: $*)" >&2 - return 1 -} - -set_agent_model() { - local profile="$1" - local candidate - shift - for candidate in "$@"; do - if openclaw --profile "$profile" models set "$candidate" >/dev/null 2>&1; then - echo "$candidate" - return 0 - fi - done - echo "ERROR: could not set agent model (tried: $*)" >&2 + echo "ERROR: could not set $label model (tried: $*)" >&2 return 1 } @@ -419,14 +407,9 @@ run_agent_turn_logged_or_skip_profile() { } run_agent_turn_bg() { - local label="$1" - local profile="$2" - local session_id="$3" - local prompt="$4" - local out_json="$5" ( set -euo pipefail - run_agent_turn_logged "$label" "$profile" "$session_id" "$prompt" "$out_json" + run_agent_turn_logged "$@" ) & RUN_AGENT_TURN_BG_PID="$!" } @@ -795,56 +778,24 @@ run_profile() { CURRENT_AGENT_MODEL_PROVIDER="$agent_model_provider" phase_mark_start "Onboard ($profile)" - if [[ "$agent_model_provider" == "openai" ]]; then - openclaw --profile "$profile" onboard \ - --non-interactive \ - --accept-risk \ - --flow quickstart \ - --auth-choice openai-api-key \ - --openai-api-key "$OPENAI_API_KEY" \ - --gateway-port "$port" \ - --gateway-bind loopback \ - --gateway-auth token \ - --workspace "$workspace" \ - --skip-health - elif [[ -n "$ANTHROPIC_API_KEY" ]]; then - openclaw --profile "$profile" onboard \ - --non-interactive \ - --accept-risk \ - --flow quickstart \ - --auth-choice apiKey \ - --anthropic-api-key "$ANTHROPIC_API_KEY" \ - --gateway-port "$port" \ - --gateway-bind loopback \ - --gateway-auth token \ - --workspace "$workspace" \ - --skip-health - elif [[ -n "$ANTHROPIC_API_TOKEN" ]]; then - openclaw --profile "$profile" onboard \ - --non-interactive \ - --accept-risk \ - --flow quickstart \ - --auth-choice token \ - --token-provider anthropic \ - --token "$ANTHROPIC_API_TOKEN" \ - --gateway-port "$port" \ - --gateway-bind loopback \ - --gateway-auth token \ - --workspace "$workspace" \ - --skip-health - else - openclaw --profile "$profile" onboard \ - --non-interactive \ - --accept-risk \ - --flow quickstart \ - --auth-choice apiKey \ - --anthropic-api-key "$ANTHROPIC_API_KEY" \ - --gateway-port "$port" \ - --gateway-bind loopback \ - --gateway-auth token \ - --workspace "$workspace" \ - --skip-health + local auth_args=() + if [[ "$agent_model_provider" == "openai" ]]; then + auth_args=(--auth-choice openai-api-key --openai-api-key "$OPENAI_API_KEY") + elif [[ -z "$ANTHROPIC_API_KEY" && -n "$ANTHROPIC_API_TOKEN" ]]; then + auth_args=(--auth-choice token --token-provider anthropic --token "$ANTHROPIC_API_TOKEN") + else + auth_args=(--auth-choice apiKey --anthropic-api-key "$ANTHROPIC_API_KEY") fi + openclaw --profile "$profile" onboard \ + --non-interactive \ + --accept-risk \ + --flow quickstart \ + "${auth_args[@]}" \ + --gateway-port "$port" \ + --gateway-bind loopback \ + --gateway-auth token \ + --workspace "$workspace" \ + --skip-health phase_mark_passed "Onboard ($profile)" phase_mark_start "Verify workspace identity files ($profile)" @@ -862,18 +813,18 @@ run_profile() { local agent_model local image_model if [[ "$agent_model_provider" == "openai" ]]; then - agent_model="$(set_agent_model "$profile" \ + agent_model="$(set_profile_model "$profile" set agent \ "$OPENAI_AGENT_MODEL" \ "openai/gpt-5.5" \ "openai/gpt-5.4-mini")" openclaw --profile "$profile" config set models.providers.openai "{\"baseUrl\":\"https://api.openai.com/v1\",\"models\":[],\"timeoutSeconds\":${OPENAI_PROVIDER_TIMEOUT_SECONDS},\"agentRuntime\":{\"id\":\"openclaw\"}}" --strict-json >/dev/null - image_model="$(set_image_model "$profile" \ + image_model="$(set_profile_model "$profile" set-image "an image" \ "openai/gpt-5.4-image-2")" else - agent_model="$(set_agent_model "$profile" \ + agent_model="$(set_profile_model "$profile" set agent \ "anthropic/claude-opus-4-6" \ "claude-opus-4-6")" - image_model="$(set_image_model "$profile" \ + image_model="$(set_profile_model "$profile" set-image "an image" \ "anthropic/claude-opus-4-6" \ "claude-opus-4-6")" fi diff --git a/scripts/docker/setup.sh b/scripts/docker/setup.sh index 4a9116bfb6f3..f6a3bf7d5a35 100755 --- a/scripts/docker/setup.sh +++ b/scripts/docker/setup.sh @@ -50,12 +50,6 @@ require_cmd() { fi } -run_docker_build() { - # Dockerfile uses BuildKit-only syntax (RUN --mount=type=cache). Force - # BuildKit so hosts defaulting to the legacy builder do not fail. - docker_build_exec "$@" -} - run_docker_pull() { local image="$1" openclaw_host_timeout_cmd "$DOCKER_PULL_TIMEOUT" docker pull "$image" @@ -598,53 +592,35 @@ COMPOSE_ARGS=() write_extra_compose() { local home_volume="$1" shift - local mount - local gateway_home_mount - local gateway_config_mount - local gateway_workspace_mount - local gateway_auth_profile_secret_mount - - cat >"$EXTRA_COMPOSE_FILE" <<'YAML' -services: - openclaw-gateway: - volumes: -YAML + local mount service + local home_mounts=() if [[ -n "$home_volume" ]]; then - gateway_home_mount="${home_volume}:/home/node" - gateway_config_mount="${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw" - gateway_workspace_mount="${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace" - gateway_auth_profile_secret_mount="${OPENCLAW_AUTH_PROFILE_SECRET_DIR}:/home/node/.config/openclaw" - validate_mount_spec "$gateway_home_mount" - validate_mount_spec "$gateway_config_mount" - validate_mount_spec "$gateway_workspace_mount" - validate_mount_spec "$gateway_auth_profile_secret_mount" - printf ' - %s\n' "$(quote_yaml_string "$gateway_home_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_config_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_workspace_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_auth_profile_secret_mount")" >>"$EXTRA_COMPOSE_FILE" + home_mounts=( + "${home_volume}:/home/node" + "${OPENCLAW_CONFIG_DIR}:/home/node/.openclaw" + "${OPENCLAW_WORKSPACE_DIR}:/home/node/.openclaw/workspace" + "${OPENCLAW_AUTH_PROFILE_SECRET_DIR}:/home/node/.config/openclaw" + ) fi - for mount in "$@"; do - validate_mount_spec "$mount" - printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE" - done - - cat >>"$EXTRA_COMPOSE_FILE" <<'YAML' - openclaw-cli: - volumes: -YAML - - if [[ -n "$home_volume" ]]; then - printf ' - %s\n' "$(quote_yaml_string "$gateway_home_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_config_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_workspace_mount")" >>"$EXTRA_COMPOSE_FILE" - printf ' - %s\n' "$(quote_yaml_string "$gateway_auth_profile_secret_mount")" >>"$EXTRA_COMPOSE_FILE" - fi - - for mount in "$@"; do - validate_mount_spec "$mount" - printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE" + printf 'services:\n' >"$EXTRA_COMPOSE_FILE" + for service in openclaw-gateway openclaw-cli; do + printf ' %s:\n volumes:\n' "$service" >>"$EXTRA_COMPOSE_FILE" + if [[ -n "$home_volume" ]]; then + if [[ "$service" == "openclaw-gateway" ]]; then + for mount in "${home_mounts[@]}"; do + validate_mount_spec "$mount" + done + fi + for mount in "${home_mounts[@]}"; do + printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE" + done + fi + for mount in "$@"; do + validate_mount_spec "$mount" + printf ' - %s\n' "$(quote_yaml_string "$mount")" >>"$EXTRA_COMPOSE_FILE" + done done if [[ -n "$home_volume" && "$home_volume" != *"/"* ]]; then @@ -783,7 +759,7 @@ elif [[ "$IMAGE_NAME" == "openclaw:local" ]]; then if [[ "$BUILD_GIT_COMMIT" =~ ^[0-9a-fA-F]{40}$ ]]; then PROVENANCE_BUILD_ARGS+=(--build-arg "GIT_COMMIT=${BUILD_GIT_COMMIT}") fi - run_docker_build \ + docker_build_exec \ "${PROVENANCE_BUILD_ARGS[@]}" \ --build-arg "OPENCLAW_IMAGE_APT_PACKAGES=${OPENCLAW_IMAGE_APT_PACKAGES}" \ --build-arg "OPENCLAW_IMAGE_PIP_PACKAGES=${OPENCLAW_IMAGE_PIP_PACKAGES}" \ @@ -898,7 +874,7 @@ if [[ -n "$SANDBOX_ENABLED" ]]; then if [[ -n "$SANDBOX_ENABLED" && -z "$OFFLINE_MODE" && -f "$sandbox_dockerfile" ]]; then echo "Building sandbox image: $DEFAULT_SANDBOX_IMAGE" - run_docker_build \ + docker_build_exec \ -t "$DEFAULT_SANDBOX_IMAGE" \ -f "$sandbox_dockerfile" \ "$ROOT_DIR" @@ -945,21 +921,14 @@ fi if [[ -n "$SANDBOX_ENABLED" ]]; then # Enable sandbox in OpenClaw config. sandbox_config_ok=true - if ! run_runtime_cli current no-deps \ - config set agents.defaults.sandbox.mode "non-main" >/dev/null; then - echo "WARNING: Failed to set agents.defaults.sandbox.mode" >&2 - sandbox_config_ok=false - fi - if ! run_runtime_cli current no-deps \ - config set agents.defaults.sandbox.scope "agent" >/dev/null; then - echo "WARNING: Failed to set agents.defaults.sandbox.scope" >&2 - sandbox_config_ok=false - fi - if ! run_runtime_cli current no-deps \ - config set agents.defaults.sandbox.workspaceAccess "none" >/dev/null; then - echo "WARNING: Failed to set agents.defaults.sandbox.workspaceAccess" >&2 - sandbox_config_ok=false - fi + for sandbox_setting in mode:non-main scope:agent workspaceAccess:none; do + sandbox_path="agents.defaults.sandbox.${sandbox_setting%%:*}" + if ! run_runtime_cli current no-deps \ + config set "$sandbox_path" "${sandbox_setting#*:}" >/dev/null; then + echo "WARNING: Failed to set $sandbox_path" >&2 + sandbox_config_ok=false + fi + done if [[ "$sandbox_config_ok" == true ]]; then echo "Sandbox enabled: mode=non-main, scope=agent, workspaceAccess=none" diff --git a/scripts/docker/shared-image-artifact.sh b/scripts/docker/shared-image-artifact.sh index 66f14607bd6b..ff0baf330df1 100755 --- a/scripts/docker/shared-image-artifact.sh +++ b/scripts/docker/shared-image-artifact.sh @@ -349,19 +349,11 @@ load_artifact() { } trap cleanup_load EXIT - EXPECTED_IMAGES_JSON="$( - printf '%s\0' "${image_refs[@]}" | - node -e ' - const fs = require("node:fs"); - const refs = fs.readFileSync(0).toString("utf8").split("\0").filter(Boolean); - process.stdout.write(JSON.stringify(refs)); - ' - )" node - "$manifest_path" > "$validated_path" <<'NODE' + node - "$manifest_path" "${image_refs[@]}" > "$validated_path" <<'NODE' const fs = require("node:fs"); -const [manifestPath] = process.argv.slice(2); +const [manifestPath, ...expectedRefs] = process.argv.slice(2); const value = JSON.parse(fs.readFileSync(manifestPath, "utf8")); -const expectedRefs = JSON.parse(process.env.EXPECTED_IMAGES_JSON); const fail = (message) => { throw new Error(`invalid shared Docker image artifact: ${message}`); }; diff --git a/scripts/docs-i18n/doc_chunked_raw.go b/scripts/docs-i18n/doc_chunked_raw.go index fba8202549f3..d7696add9a53 100644 --- a/scripts/docs-i18n/doc_chunked_raw.go +++ b/scripts/docs-i18n/doc_chunked_raw.go @@ -111,34 +111,7 @@ func validateDocBodyFencedLiterals(source, translated string) error { if sourceStructure.fenceCount != translatedStructure.fenceCount { return fmt.Errorf("code fence mismatch: source=%d translated=%d", sourceStructure.fenceCount, translatedStructure.fenceCount) } - if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) { - return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes) - } - if !slices.Equal(sourceStructure.listMarkerPrefixes, translatedStructure.listMarkerPrefixes) { - return fmt.Errorf("list marker structure mismatch: source=%q translated=%q", sourceStructure.listMarkerPrefixes, translatedStructure.listMarkerPrefixes) - } - if !sameStringMultiset(sourceStructure.inlineCodeSpans, translatedStructure.inlineCodeSpans) { - return fmt.Errorf("inline code mismatch: source=%d translated=%d", len(sourceStructure.inlineCodeSpans), len(translatedStructure.inlineCodeSpans)) - } - if !slices.Equal(sourceStructure.fencedPlaceholders, translatedStructure.fencedPlaceholders) { - return fmt.Errorf("fenced placeholder mismatch: source=%d translated=%d", len(sourceStructure.fencedPlaceholders), len(translatedStructure.fencedPlaceholders)) - } - if !slices.Equal(sourceStructure.fencedProtocolTokens, translatedStructure.fencedProtocolTokens) { - return fmt.Errorf("fenced protocol marker mismatch: source=%d translated=%d", len(sourceStructure.fencedProtocolTokens), len(translatedStructure.fencedProtocolTokens)) - } - if !slices.Equal(sourceStructure.fencedDirectiveTokens, translatedStructure.fencedDirectiveTokens) { - return fmt.Errorf("fenced directive mismatch: source=%d translated=%d", len(sourceStructure.fencedDirectiveTokens), len(translatedStructure.fencedDirectiveTokens)) - } - if !sameStringMultiset(sourceStructure.linkDestinations, translatedStructure.linkDestinations) { - return fmt.Errorf("link destination mismatch: source=%d translated=%d", len(sourceStructure.linkDestinations), len(translatedStructure.linkDestinations)) - } - if !sameStringMultiset(sourceStructure.protectedLinkLabels, translatedStructure.protectedLinkLabels) { - return fmt.Errorf("protected link label mismatch: source=%d translated=%d", len(sourceStructure.protectedLinkLabels), len(translatedStructure.protectedLinkLabels)) - } - if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) { - return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues)) - } - return nil + return validateDocChunkLiterals(sourceStructure, translatedStructure) } func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chunkID string, blocks []string, protectedPlaceholders []string, listPlaceholders map[string]string, srcLang, tgtLang string) (string, error) { @@ -161,14 +134,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun log.Printf("docs-i18n: rejected raw chunk %s input=%q output=%q err=%v", chunkID, normalizedSource, translated, err) } if err == nil { - translated = sanitizeDocChunkProtocolWrappers(source, translated) - translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) - translated = reapplyCommonIndent(translated, commonIndent) - translated = normalizeMaskedListMarkerPlaceholders(translated, listPlaceholders) - translated = normalizeMaskedListMarkerSpacing(source, translated, listPlaceholders) - translated = escapeUnexpectedListItemBodyMarkers(source, translated, listPlaceholders) - translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) - translated = unwrapUnexpectedInlineCodeSpans(source, translated) + translated = normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent, listPlaceholders) if validationErr := validateDocChunkTranslation(source, translated); validationErr == nil { log.Printf("docs-i18n: chunk done %s out_bytes=%d", chunkID, len(translated)) return translated, nil @@ -182,7 +148,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun } else if os.Getenv("OPENCLAW_DOCS_I18N_LOG_REJECTED_BODY") == "1" { log.Printf("docs-i18n: chunk leaf-fallback failed %s err=%v", chunkID, fallbackErr) } - if plan, ok := planSingletonDocChunkRetry(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget()); ok { + if plan, ok := planSingletonDocChunkWithMode(source, docsI18nDocChunkMaxBytes(), docsI18nDocChunkPromptBudget(), true); ok { logDocChunkPlanSplit(chunkID, plan, source) return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } @@ -192,7 +158,7 @@ func translateDocBlockGroup(ctx context.Context, translator docsTranslator, chun logDocChunkSplit(chunkID, len(blocks), err) return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } - if plan, ok := splitDocChunkBlocksMidpointSimple(blocks); ok { + if plan, ok := splitDocChunkBlocksMidpoint(blocks, "retry-midpoint"); ok { logDocChunkSplit(chunkID, len(blocks), err) return translatePlannedDocChunkGroups(ctx, translator, chunkID, source, plan.groups, protectedPlaceholders, listPlaceholders, srcLang, tgtLang) } @@ -217,6 +183,15 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk if err != nil { return "", err } + translated = normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent, listPlaceholders) + if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil { + return "", validationErr + } + log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated)) + return translated, nil +} + +func normalizeDocChunkTranslation(source, normalizedSource, translated, commonIndent string, listPlaceholders map[string]string) string { translated = sanitizeDocChunkProtocolWrappers(source, translated) translated = preserveDocChunkBoundaryWhitespace(normalizedSource, translated) translated = reapplyCommonIndent(translated, commonIndent) @@ -225,11 +200,7 @@ func translateDocLeafBlock(ctx context.Context, translator docsTranslator, chunk translated = escapeUnexpectedListItemBodyMarkers(source, translated, listPlaceholders) translated = escapeUnexpectedMarkdownListMarkers(translated, listPlaceholders) translated = unwrapUnexpectedInlineCodeSpans(source, translated) - if validationErr := validateDocChunkTranslation(source, translated); validationErr != nil { - return "", validationErr - } - log.Printf("docs-i18n: chunk leaf-fallback done %s out_bytes=%d", chunkID, len(translated)) - return translated, nil + return translated } func splitDocBodyIntoBlocks(body string) []string { @@ -324,6 +295,21 @@ func validateDocChunkTranslation(source, translated string) error { if !slices.Equal(sourceStructure.headingLevels, translatedStructure.headingLevels) { return fmt.Errorf("heading structure mismatch: source=%v translated=%v", sourceStructure.headingLevels, translatedStructure.headingLevels) } + if err := validateDocChunkLiterals(sourceStructure, translatedStructure); err != nil { + return err + } + if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) { + return fmt.Errorf("component tag set mismatch") + } + for _, key := range sortedKeys(sourceStructure.tagCounts) { + if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] { + return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key]) + } + } + return nil +} + +func validateDocChunkLiterals(sourceStructure, translatedStructure docChunkStructure) error { if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) { return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes) } @@ -351,14 +337,6 @@ func validateDocChunkTranslation(source, translated string) error { if !sameStringMultiset(sourceStructure.numericValues, translatedStructure.numericValues) { return fmt.Errorf("numeric value mismatch: source=%d translated=%d", len(sourceStructure.numericValues), len(translatedStructure.numericValues)) } - if !slices.Equal(sortedKeys(sourceStructure.tagCounts), sortedKeys(translatedStructure.tagCounts)) { - return fmt.Errorf("component tag set mismatch") - } - for _, key := range sortedKeys(sourceStructure.tagCounts) { - if sourceStructure.tagCounts[key] != translatedStructure.tagCounts[key] { - return fmt.Errorf("component tag mismatch for %s: source=%d translated=%d", key, sourceStructure.tagCounts[key], translatedStructure.tagCounts[key]) - } - } return nil } @@ -410,29 +388,9 @@ func sanitizeDocChunkProtocolWrappers(source, translated string) string { } func preserveDocChunkBoundaryWhitespace(source, translated string) string { - prefixEnd := 0 - for prefixEnd < len(source) && isDocChunkBoundaryWhitespace(source[prefixEnd]) { - prefixEnd++ - } - suffixStart := len(source) - for suffixStart > prefixEnd && isDocChunkBoundaryWhitespace(source[suffixStart-1]) { - suffixStart-- - } - - translatedStart := 0 - for translatedStart < len(translated) && isDocChunkBoundaryWhitespace(translated[translatedStart]) { - translatedStart++ - } - translatedEnd := len(translated) - for translatedEnd > translatedStart && isDocChunkBoundaryWhitespace(translated[translatedEnd-1]) { - translatedEnd-- - } - - return source[:prefixEnd] + translated[translatedStart:translatedEnd] + source[suffixStart:] -} - -func isDocChunkBoundaryWhitespace(value byte) bool { - return value == ' ' || value == '\t' || value == '\r' || value == '\n' + prefix, _, suffix := splitWhitespace(source) + _, core, _ := splitWhitespace(translated) + return prefix + core + suffix } func stripBodyOnlyWrapper(source, text string) (string, bool) { @@ -506,13 +464,14 @@ func logDocChunkPlanSplit(chunkID string, plan docChunkSplitPlan, source string) func summarizeDocChunkStructure(text string) docChunkStructure { counts := map[string]int{} + fenceCount := 0 lines := strings.Split(text, "\n") fenceDelimiter := "" for _, line := range lines { var toggled bool fenceDelimiter, toggled = updateFenceDelimiter(fenceDelimiter, line) if toggled { - counts["__fence_toggle__"]++ + fenceCount++ } for _, match := range docsComponentTagRE.FindAllStringSubmatch(line, -1) { if len(match) < 3 { @@ -532,8 +491,8 @@ func summarizeDocChunkStructure(text string) docChunkStructure { } fencedPlaceholders, fencedProtocolTokens, fencedDirectiveTokens := extractMarkdownFencedLiteralValues(text) return docChunkStructure{ - fenceCount: counts["__fence_toggle__"], - tagCounts: countsWithoutFence(counts), + fenceCount: fenceCount, + tagCounts: counts, headingLevels: extractMarkdownHeadingLevels(text), listShapes: extractMarkdownListShapes(text), listMarkerPrefixes: extractMarkdownListMarkerPrefixes(text), @@ -675,17 +634,6 @@ func parseDocsMarkdown(source []byte) ast.Node { return goldmark.New(goldmark.WithExtensions(extension.GFM, extension.Footnote)).Parser().Parse(textpkg.NewReader(source)) } -func countsWithoutFence(counts map[string]int) map[string]int { - filtered := map[string]int{} - for key, value := range counts { - if key == "__fence_toggle__" { - continue - } - filtered[key] = value - } - return filtered -} - func sortedKeys(counts map[string]int) []string { keys := make([]string, 0, len(counts)) for key := range counts { @@ -842,7 +790,7 @@ func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSpl normalizedSource, _ := stripCommonIndent(source) estimatedPromptCost := estimateDocPromptCost(normalizedSource) if len(blocks) > 1 && promptBudget > 0 && estimatedPromptCost > promptBudget { - return splitDocChunkBlocksMidpoint(blocks, estimatedPromptCost, promptBudget) + return splitDocChunkBlocksMidpoint(blocks, fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget)) } if len(blocks) == 1 { return planSingletonDocChunk(blocks[0], maxBytes, promptBudget) @@ -850,32 +798,12 @@ func planDocChunkSplit(blocks []string, maxBytes, promptBudget int) (docChunkSpl return docChunkSplitPlan{}, false } -func splitDocChunkBlocksMidpoint(blocks []string, estimatedPromptCost, promptBudget int) (docChunkSplitPlan, bool) { +func splitDocChunkBlocksMidpoint(blocks []string, reason string) (docChunkSplitPlan, bool) { if len(blocks) <= 1 { return docChunkSplitPlan{}, false } mid := len(blocks) / 2 - if mid <= 0 || mid >= len(blocks) { - return docChunkSplitPlan{}, false - } - return docChunkSplitPlan{ - groups: [][]string{blocks[:mid], blocks[mid:]}, - reason: fmt.Sprintf("prompt-budget:%d>%d", estimatedPromptCost, promptBudget), - }, true -} - -func splitDocChunkBlocksMidpointSimple(blocks []string) (docChunkSplitPlan, bool) { - if len(blocks) <= 1 { - return docChunkSplitPlan{}, false - } - mid := len(blocks) / 2 - if mid <= 0 || mid >= len(blocks) { - return docChunkSplitPlan{}, false - } - return docChunkSplitPlan{ - groups: [][]string{blocks[:mid], blocks[mid:]}, - reason: "retry-midpoint", - }, true + return docChunkSplitPlan{groups: [][]string{blocks[:mid], blocks[mid:]}, reason: reason}, true } func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) { @@ -890,43 +818,31 @@ func planSingletonDocChunk(block string, maxBytes, promptBudget int) (docChunkSp return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, false) } -func planSingletonDocChunkRetry(block string, maxBytes, promptBudget int) (docChunkSplitPlan, bool) { - return planSingletonDocChunkWithMode(block, maxBytes, promptBudget, true) -} - func planSingletonDocChunkWithMode(block string, maxBytes, promptBudget int, force bool) (docChunkSplitPlan, bool) { + prefix := "singleton-" + if force { + prefix = "singleton-retry-" + } if sections := splitDocBlockSections(block); len(sections) > 1 { if groups := wrapDocChunkSections(sections); len(groups) > 1 { - reason := "singleton-structural" - if force { - reason = "singleton-retry-structural" - } return docChunkSplitPlan{ groups: groups, - reason: reason, + reason: prefix + "structural", }, true } } if groups, ok := splitPureFencedDocSectionWithMode(block, maxBytes, promptBudget, force); ok { - reason := "singleton-fence" - if force { - reason = "singleton-retry-fence" - } return docChunkSplitPlan{ groups: groups, - reason: reason, + reason: prefix + "fence", }, true } if groups, ok := splitPlainDocSectionWithMode(block, maxBytes, promptBudget, force); ok { - reason := "singleton-lines" - if force { - reason = "singleton-retry-lines" - } return docChunkSplitPlan{ groups: groups, - reason: reason, + reason: prefix + "lines", }, true } @@ -946,9 +862,6 @@ func wrapDocChunkSections(sections []string) [][]string { func splitDocBlockSections(block string) []string { lines := strings.SplitAfter(block, "\n") - if len(lines) == 0 { - return nil - } sections := make([]string, 0, len(lines)) var current strings.Builder fenceDelimiter := "" @@ -1051,9 +964,6 @@ func splitPlainDocSectionMidpoint(lines []string) ([][]string, bool) { return nil, false } mid := len(lines) / 2 - if mid <= 0 || mid >= len(lines) { - return nil, false - } left := strings.Join(lines[:mid], "") right := strings.Join(lines[mid:], "") if strings.TrimSpace(left) == "" || strings.TrimSpace(right) == "" { @@ -1126,7 +1036,7 @@ func stripCommonIndent(text string) (string, string) { if strings.TrimSpace(trimmed) == "" { continue } - indent := leadingIndent(trimmed) + indent := leadingWhitespace(trimmed) if common == "" { common = indent } else { @@ -1173,22 +1083,8 @@ func reapplyCommonIndent(text, indent string) string { return out.String() } -func leadingIndent(line string) string { - index := 0 - for index < len(line) { - if line[index] != ' ' && line[index] != '\t' { - break - } - index++ - } - return line[:index] -} - func commonIndentPrefix(a, b string) string { - limit := len(a) - if len(b) < limit { - limit = len(b) - } + limit := min(len(a), len(b)) index := 0 for index < limit && a[index] == b[index] { index++ diff --git a/scripts/docs-i18n/doc_mode.go b/scripts/docs-i18n/doc_mode.go index a8c5dfbd86f5..1606e7f2f287 100644 --- a/scripts/docs-i18n/doc_mode.go +++ b/scripts/docs-i18n/doc_mode.go @@ -121,25 +121,11 @@ func findTaggedBodyEnd(text string, bodyStart int) int { if bodyStart < 0 || bodyStart > len(text) { return -1 } - search := text[bodyStart:] - candidate := -1 - offset := 0 - for { - index := strings.Index(search[offset:], bodyTagEnd) - if index == -1 { - return candidate - } - index += offset - absolute := bodyStart + index - suffix := strings.TrimSpace(text[absolute+len(bodyTagEnd):]) - if suffix == "" { - candidate = absolute - } - offset = index + len(bodyTagEnd) - if offset >= len(search) { - return candidate - } + end := strings.LastIndex(text[bodyStart:], bodyTagEnd) + if end < 0 || strings.TrimSpace(text[bodyStart+end+len(bodyTagEnd):]) != "" { + return -1 } + return bodyStart + end } func trimTagNewlines(value string) string { @@ -164,7 +150,7 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string) if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil { return docOutputNeedsTranslation, nil } - storedHash := extractSourceHash(frontData) + storedHash := extractI18NString(frontData, "source_hash") if storedHash == "" { return docOutputNeedsTranslation, nil } @@ -179,7 +165,7 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string) return docOutputNeedsTranslation, nil } - postprocessVersion := extractPostprocessVersion(frontData) + postprocessVersion := extractI18NString(frontData, "postprocess_version") if strings.EqualFold(postprocessVersion, localizedLinkPostprocessVersion) { return docOutputReady, nil } @@ -187,49 +173,17 @@ func classifyDocOutput(outputPath string, sourceHash string, targetLang string) } func extractI18NVersion(frontData map[string]any, field string) int { - xi, ok := extractXI18N(frontData) - if !ok { - return 0 - } - value, ok := xi[field].(int) - if !ok { - return 0 - } + xi, _ := frontData["x-i18n"].(map[string]any) + value, _ := xi[field].(int) return value } -func extractSourceHash(frontData map[string]any) string { - xi, ok := extractXI18N(frontData) - if !ok { - return "" - } - value, ok := xi["source_hash"].(string) - if !ok { - return "" - } +func extractI18NString(frontData map[string]any, field string) string { + xi, _ := frontData["x-i18n"].(map[string]any) + value, _ := xi[field].(string) return strings.TrimSpace(value) } -func extractPostprocessVersion(frontData map[string]any) string { - xi, ok := extractXI18N(frontData) - if !ok { - return "" - } - value, ok := xi["postprocess_version"].(string) - if !ok { - return "" - } - return strings.TrimSpace(value) -} - -func extractXI18N(frontData map[string]any) (map[string]any, bool) { - xi, ok := frontData["x-i18n"].(map[string]any) - if ok { - return xi, true - } - return nil, false -} - func logDocChunkPlan(relPath string, blocks []string, groups [][]string) { totalBytes := 0 for _, block := range blocks { diff --git a/scripts/docs-i18n/html_translate.go b/scripts/docs-i18n/html_translate.go index 28b15c261e9e..6102baa9d848 100644 --- a/scripts/docs-i18n/html_translate.go +++ b/scripts/docs-i18n/html_translate.go @@ -3,6 +3,7 @@ package main import ( "context" "io" + "sort" "strings" "github.com/yuin/goldmark" @@ -10,15 +11,8 @@ import ( "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/text" "golang.org/x/net/html" - "sort" ) -type htmlReplacement struct { - Start int - Stop int - Value string -} - func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, srcLang, tgtLang string) (string, error) { source := []byte(body) r := text.NewReader(source) @@ -27,9 +21,9 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s ) doc := md.Parser().Parse(r) - replacements := make([]htmlReplacement, 0, 8) + replacements := make([]Segment, 0, 8) - _ = ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } @@ -37,7 +31,7 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s if !ok { return ast.WalkContinue, nil } - start, stop, ok := htmlBlockSpan(block, source) + start, stop, ok := htmlBlockSpan(block) if !ok { return ast.WalkSkipChildren, nil } @@ -46,53 +40,27 @@ func translateHTMLBlocks(ctx context.Context, translator docsTranslator, body, s if err != nil { return ast.WalkStop, err } - replacements = append(replacements, htmlReplacement{Start: start, Stop: stop, Value: translated}) + replacements = append(replacements, Segment{Start: start, Stop: stop, Translated: translated}) return ast.WalkSkipChildren, nil }) - if len(replacements) == 0 { - return body, nil + if err != nil { + return "", err } - - return applyHTMLReplacements(body, replacements), nil + sort.Slice(replacements, func(i, j int) bool { + return replacements[i].Start < replacements[j].Start + }) + return applyTranslations(body, replacements), nil } -func htmlBlockSpan(block *ast.HTMLBlock, source []byte) (int, int, bool) { +func htmlBlockSpan(block *ast.HTMLBlock) (int, int, bool) { lines := block.Lines() if lines.Len() == 0 { return 0, 0, false } start := lines.At(0).Start stop := lines.At(lines.Len() - 1).Stop - if start >= stop { - return 0, 0, false - } - return start, stop, true -} - -func applyHTMLReplacements(body string, replacements []htmlReplacement) string { - if len(replacements) == 0 { - return body - } - sortHTMLReplacements(replacements) - var out strings.Builder - last := 0 - for _, rep := range replacements { - if rep.Start < last { - continue - } - out.WriteString(body[last:rep.Start]) - out.WriteString(rep.Value) - last = rep.Stop - } - out.WriteString(body[last:]) - return out.String() -} - -func sortHTMLReplacements(replacements []htmlReplacement) { - sort.Slice(replacements, func(i, j int) bool { - return replacements[i].Start < replacements[j].Start - }) + return start, stop, start < stop } func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText, srcLang, tgtLang string) (string, error) { @@ -123,8 +91,6 @@ func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText if isSkipTag(strings.ToLower(tok.Data)) && skipDepth > 0 { skipDepth-- } - case html.SelfClosingTagToken: - out.WriteString(raw) case html.TextToken: if shouldTranslateHTMLText(skipDepth, raw) { translated, err := translator.Translate(ctx, raw, srcLang, tgtLang) @@ -144,10 +110,7 @@ func translateHTMLBlock(ctx context.Context, translator docsTranslator, htmlText } func shouldTranslateHTMLText(skipDepth int, text string) bool { - if strings.TrimSpace(text) == "" { - return false - } - return skipDepth == 0 + return skipDepth == 0 && strings.TrimSpace(text) != "" } func isSkipTag(tag string) bool { diff --git a/scripts/docs-i18n/localized_links.go b/scripts/docs-i18n/localized_links.go index 2c777c82ad73..190e042e91df 100644 --- a/scripts/docs-i18n/localized_links.go +++ b/scripts/docs-i18n/localized_links.go @@ -230,64 +230,22 @@ func (ri *routeIndex) localizeBodyLinks(body string) string { masked = maskMatches(masked, fencedTildeCodeBlock, state.Next, &placeholders, mapping) masked = maskMatches(masked, inlineCodeRe, state.Next, &placeholders, mapping) - masked = rewriteMarkdownLinkTargets(masked, ri) - masked = rewriteHrefTargets(masked, ri) + masked = rewriteCapturedTargets(masked, markdownLinkTargetRe, ri, true) + masked = rewriteCapturedTargets(masked, hrefDoubleQuotedValueRe, ri, false) + masked = rewriteCapturedTargets(masked, hrefSingleQuotedValueRe, ri, false) return unmaskMarkdown(masked, placeholders, mapping) } -func rewriteMarkdownLinkTargets(text string, ri *routeIndex) string { - matches := markdownLinkTargetRe.FindAllStringSubmatchIndex(text, -1) - if len(matches) == 0 { - return text - } - - var out strings.Builder - pos := 0 - for _, span := range matches { - fullStart, targetStart, targetEnd := span[0], span[2], span[3] - if fullStart < pos { - continue +func rewriteCapturedTargets(text string, re *regexp.Regexp, ri *routeIndex, skipImages bool) string { + return re.ReplaceAllStringFunc(text, func(match string) string { + if skipImages && strings.HasPrefix(match, "!") { + return match } - - out.WriteString(text[pos:targetStart]) - target := text[targetStart:targetEnd] - if text[fullStart] == '!' { - out.WriteString(target) - } else { - out.WriteString(ri.localizeURL(target)) - } - pos = targetEnd - } - out.WriteString(text[pos:]) - return out.String() -} - -func rewriteHrefTargets(text string, ri *routeIndex) string { - text = rewriteCapturedTargets(text, hrefDoubleQuotedValueRe, 2, ri) - text = rewriteCapturedTargets(text, hrefSingleQuotedValueRe, 2, ri) - return text -} - -func rewriteCapturedTargets(text string, re *regexp.Regexp, groupIndex int, ri *routeIndex) string { - matches := re.FindAllStringSubmatchIndex(text, -1) - if len(matches) == 0 { - return text - } - - var out strings.Builder - pos := 0 - for _, span := range matches { - start, end := span[groupIndex], span[groupIndex+1] - if start < pos || start < 0 || end < 0 { - continue - } - out.WriteString(text[pos:start]) - out.WriteString(ri.localizeURL(text[start:end])) - pos = end - } - out.WriteString(text[pos:]) - return out.String() + span := re.FindStringSubmatchIndex(match) + start, end := span[2], span[3] + return match[:start] + ri.localizeURL(match[start:end]) + match[end:] + }) } func (ri *routeIndex) localizeURL(raw string) string { diff --git a/scripts/docs-i18n/main.go b/scripts/docs-i18n/main.go index a6b53c99f2fa..cd4440e7ea59 100644 --- a/scripts/docs-i18n/main.go +++ b/scripts/docs-i18n/main.go @@ -138,28 +138,18 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel) switch cfg.mode { case "doc": + var outputs []string if parallel > 1 { - proc, skip, outputs, err := runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial, parallel, glossary, cfg.thinking, newTranslator) - processed += proc - skipped += skip - localizedFiles = append(localizedFiles, outputs...) - if err != nil { - translationErr = err - } + processed, skipped, outputs, translationErr = runDocParallel(ctx, ordered, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial, parallel, glossary, cfg.thinking, newTranslator) } else { translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking) if err != nil { return err } defer translator.Close() - proc, skip, outputs, err := runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial) - processed += proc - skipped += skip - localizedFiles = append(localizedFiles, outputs...) - if err != nil { - translationErr = err - } + processed, skipped, outputs, translationErr = runDocSequential(ctx, ordered, translator, resolvedDocsRoot, cfg.sourceLang, cfg.targetLang, cfg.overwrite, cfg.allowPartial) } + localizedFiles = append(localizedFiles, outputs...) case "segment": if parallel > 1 { return fmt.Errorf("parallel processing is only supported in doc mode") @@ -305,7 +295,7 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond)) } else if result.err != nil { log.Printf("docs-i18n: [w* %d/%d] failed %s (%s): %v", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond), result.err) - } else if result.err == nil { + } else { processed++ outputs = append(outputs, result.output) log.Printf("docs-i18n: [w* %d/%d] done %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond)) diff --git a/scripts/docs-i18n/main_test.go b/scripts/docs-i18n/main_test.go index 33cc7a04b1b8..2dfa899acada 100644 --- a/scripts/docs-i18n/main_test.go +++ b/scripts/docs-i18n/main_test.go @@ -65,6 +65,22 @@ func (errorTranslator) TranslateRaw(context.Context, string, string, string) (st func (errorTranslator) Close() {} +func TestProcessFileRejectsFailedHTMLTranslation(t *testing.T) { + docsRoot := t.TempDir() + sourcePath := filepath.Join(docsRoot, "page.md") + if err := os.WriteFile(sourcePath, []byte("
Translate this text.
\n"), 0o644); err != nil { + t.Fatal(err) + } + tm := &TranslationMemory{entries: map[string]TMEntry{}} + _, _, err := processFile(context.Background(), errorTranslator{}, tm, docsRoot, sourcePath, "en", "de") + if err == nil || err.Error() != "codex exec failed: exit status 1" { + t.Fatalf("expected HTML translation failure, got %v", err) + } + if _, err := os.Stat(filepath.Join(docsRoot, "de", "page.md")); !os.IsNotExist(err) { + t.Fatalf("failed translation wrote an output page: %v", err) + } +} + type partialFailTranslator struct{} func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) { diff --git a/scripts/docs-i18n/markdown_segments.go b/scripts/docs-i18n/markdown_segments.go index c6e671e93ed6..2655e02c6c13 100644 --- a/scripts/docs-i18n/markdown_segments.go +++ b/scripts/docs-i18n/markdown_segments.go @@ -219,10 +219,7 @@ func unwrapUnexpectedInlineCodeSpans(source, translated string) string { if rangeOverlapsAny(span, fenced) { continue } - runLength := 0 - for span[0]+runLength < span[1] && translated[span[0]+runLength] == '`' { - runLength++ - } + runLength := backtickRunLength(translated, span[0], span[1]) if runLength == 0 || span[1]-runLength < span[0]+runLength { continue } @@ -237,46 +234,29 @@ func extractMarkdownFencedLiteralValues(body string) ([]string, []string, []stri allSquareTokens := []string{} state := markdownLiteralFenceState{} lines := []string{} - flush := func() { + flush := func(info string) { for _, line := range lines { - if state.info != "mermaid" { + if info != "mermaid" { allSquareTokens = append(allSquareTokens, extractSquareBracketValues(line)...) - } - } - for _, line := range lines { - linePlaceholders := extractAngleBracketValues(line) - placeholders = append(placeholders, linePlaceholders...) - if state.info != "mermaid" { directiveTokens = append(directiveTokens, extractDoubleBracketValues(line)...) } + placeholders = append(placeholders, extractAngleBracketValues(line)...) } lines = lines[:0] } for _, line := range strings.Split(body, "\n") { - if state.delimiter == "" { - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening + previous := state + if !state.consumeLine(line) { + if previous.delimiter != "" { + flush(previous.info) } continue } - if !continuesMarkdownLiteralFenceContainer(line, state) { - flush() - state = markdownLiteralFenceState{} - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening - } - continue - } - if isMarkdownLiteralFenceClosing(line, state) { - flush() - state = markdownLiteralFenceState{} - continue - } lines = append(lines, strings.TrimSpace(stripMarkdownQuotePrefix(line, state.quoteDepth))) } if state.delimiter != "" { - flush() + flush(state.info) } closingNames := map[string]struct{}{} for _, token := range allSquareTokens { @@ -296,22 +276,7 @@ func extractMarkdownFencedLiteralValues(body string) ([]string, []string, []stri func markdownLiteralFencesBalanced(body string) bool { state := markdownLiteralFenceState{} for _, line := range strings.Split(body, "\n") { - if state.delimiter == "" { - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening - } - continue - } - if !continuesMarkdownLiteralFenceContainer(line, state) { - state = markdownLiteralFenceState{} - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening - } - continue - } - if isMarkdownLiteralFenceClosing(line, state) { - state = markdownLiteralFenceState{} - } + state.consumeLine(line) } return state.delimiter == "" } @@ -323,6 +288,19 @@ type markdownLiteralFenceState struct { containerIndent int } +// consumeLine reports literal content, excluding opening and closing fence lines. +func (state *markdownLiteralFenceState) consumeLine(line string) bool { + if state.delimiter != "" && continuesMarkdownLiteralFenceContainer(line, *state) { + if !isMarkdownLiteralFenceClosing(line, *state) { + return true + } + *state = markdownLiteralFenceState{} + return false + } + *state, _ = parseMarkdownLiteralFenceOpening(line) + return false +} + func parseMarkdownLiteralFenceOpening(line string) (markdownLiteralFenceState, bool) { leadingIndent := len(line) - len(strings.TrimLeft(line, " \t")) remaining := strings.TrimLeft(line, " \t") @@ -540,7 +518,7 @@ func isTranslatableBracketLabelContext(line string, start, end int, candidate st } func isASCIIIdentifierByte(value byte) bool { - return (value >= 'a' && value <= 'z') || (value >= 'A' && value <= 'Z') || (value >= '0' && value <= '9') || value == '_' + return isASCIIAlphaNumeric(value) || value == '_' } func extractDoubleBracketValues(line string) []string { @@ -662,10 +640,7 @@ func extractFallbackBacktickValues(body string) []string { if rangeOverlapsAny(span, fenced) { continue } - runLength := 0 - for span[0]+runLength < span[1] && body[span[0]+runLength] == '`' { - runLength++ - } + runLength := backtickRunLength(body, span[0], span[1]) if runLength == 0 || span[1]-runLength < span[0]+runLength { continue } @@ -753,10 +728,7 @@ func normalizeDocComponentsForMarkdownParse(body string) string { } func isLikelyFencedBacktickRange(body string, span [2]int) bool { - runLength := 0 - for span[0]+runLength < span[1] && body[span[0]+runLength] == '`' { - runLength++ - } + runLength := backtickRunLength(body, span[0], span[1]) if runLength < 3 { return false } diff --git a/scripts/docs-i18n/masking.go b/scripts/docs-i18n/masking.go index 12ba457f37f1..fc436b170f5a 100644 --- a/scripts/docs-i18n/masking.go +++ b/scripts/docs-i18n/masking.go @@ -55,24 +55,9 @@ func maskMarkdownFencedLiterals(text string, nextPlaceholder func() string, plac state := markdownLiteralFenceState{} lines := strings.SplitAfter(text, "\n") for index, line := range lines { - if state.delimiter == "" { - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening - } - continue + if state.consumeLine(line) { + lines[index] = maskMatches(line, literalRE, nextPlaceholder, placeholders, mapping) } - if !continuesMarkdownLiteralFenceContainer(line, state) { - state = markdownLiteralFenceState{} - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - state = opening - } - continue - } - if isMarkdownLiteralFenceClosing(line, state) { - state = markdownLiteralFenceState{} - continue - } - lines[index] = maskMatches(line, literalRE, nextPlaceholder, placeholders, mapping) } return strings.Join(lines, "") } @@ -96,23 +81,9 @@ func markdownListMarkerRanges(text string) [][2]int { fenceState := markdownLiteralFenceState{} offset := 0 for _, line := range strings.SplitAfter(text, "\n") { - insideFence := false - if fenceState.delimiter != "" { - if continuesMarkdownLiteralFenceContainer(line, fenceState) { - insideFence = true - if isMarkdownLiteralFenceClosing(line, fenceState) { - fenceState = markdownLiteralFenceState{} - } - } else { - fenceState = markdownLiteralFenceState{} - } - } - if !insideFence { - if opening, ok := parseMarkdownLiteralFenceOpening(line); ok { - fenceState = opening - insideFence = true - } - } + insideFence := fenceState.delimiter != "" && continuesMarkdownLiteralFenceContainer(line, fenceState) + fenceState.consumeLine(line) + insideFence = insideFence || fenceState.delimiter != "" if !insideFence { if match := listMarkerRe.FindStringSubmatchIndex(line); len(match) >= 6 { listRanges = append(listRanges, [2]int{offset + match[0], offset + match[1]}) @@ -206,10 +177,7 @@ func markdownWhitespaceRunStart(text string, position int) int { } func escapeUnexpectedListItemBodyMarkers(source, translated string, listPlaceholders map[string]string) string { - type insertion struct { - position int - } - insertions := make([]insertion, 0) + insertions := make([]int, 0) for placeholder := range listPlaceholders { sourcePosition := strings.Index(source, placeholder) translatedPosition := strings.Index(translated, placeholder) @@ -228,11 +196,11 @@ func escapeUnexpectedListItemBodyMarkers(source, translated string, listPlacehol if markerEnd-markerStart > 1 { insertAt = markerEnd - 1 } - insertions = append(insertions, insertion{position: translatedPosition + len(placeholder) + insertAt}) + insertions = append(insertions, translatedPosition+len(placeholder)+insertAt) } - sort.Slice(insertions, func(i, j int) bool { return insertions[i].position > insertions[j].position }) - for _, item := range insertions { - translated = translated[:item.position] + `\` + translated[item.position:] + sort.Slice(insertions, func(i, j int) bool { return insertions[i] > insertions[j] }) + for _, position := range insertions { + translated = translated[:position] + `\` + translated[position:] } return translated } @@ -340,7 +308,7 @@ func hasClockMeridiemSuffix(text string, span [2]int) bool { if suffix != "am" && suffix != "pm" { return false } - return span[1]+2 == len(text) || !isCompositeNumericWordByte(text[span[1]+2]) + return span[1]+2 == len(text) || !isASCIIAlphaNumeric(text[span[1]+2]) } func hasCompositeNumericLeadingContinuation(text string, position int) bool { @@ -352,9 +320,9 @@ func hasCompositeNumericLeadingContinuation(text string, position int) bool { for position > 0 && text[position-1] == '_' { position-- } - return position > 0 && isCompositeNumericWordByte(text[position-1]) + return position > 0 && isASCIIAlphaNumeric(text[position-1]) } - return value == '.' || isCompositeNumericWordByte(value) + return value == '.' || isASCIIAlphaNumeric(value) } func hasCompositeNumericContinuation(text string, position int) bool { @@ -366,16 +334,12 @@ func hasCompositeNumericContinuation(text string, position int) bool { for position < len(text) && text[position] == '_' { position++ } - return position < len(text) && isCompositeNumericWordByte(text[position]) + return position < len(text) && isASCIIAlphaNumeric(text[position]) } - if isCompositeNumericWordByte(value) { + if isASCIIAlphaNumeric(value) { return true } - return value == '.' && position+1 < len(text) && isCompositeNumericWordByte(text[position+1]) -} - -func isCompositeNumericWordByte(value byte) bool { - return value >= '0' && value <= '9' || value >= 'A' && value <= 'Z' || value >= 'a' && value <= 'z' + return value == '.' && position+1 < len(text) && isASCIIAlphaNumeric(text[position+1]) } func markdownLiteralFenceByteRanges(text string) [][2]int { @@ -449,53 +413,20 @@ func maskByteRanges(text string, ranges [][2]int, nextPlaceholder func() string, } func maskMatches(text string, re *regexp.Regexp, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string { - matches := re.FindAllStringIndex(text, -1) - if len(matches) == 0 { - return text - } - var out strings.Builder - pos := 0 - for _, span := range matches { - start, end := span[0], span[1] - if start < pos { - continue - } - out.WriteString(text[pos:start]) + return re.ReplaceAllStringFunc(text, func(match string) string { placeholder := nextPlaceholder() - mapping[placeholder] = text[start:end] + mapping[placeholder] = match *placeholders = append(*placeholders, placeholder) - out.WriteString(placeholder) - pos = end - } - out.WriteString(text[pos:]) - return out.String() + return placeholder + }) } func maskLinkURLs(text string, nextPlaceholder func() string, placeholders *[]string, mapping map[string]string) string { - matches := linkURLRe.FindAllStringSubmatchIndex(text, -1) - if len(matches) == 0 { - return text + ranges := make([][2]int, 0) + for _, span := range linkURLRe.FindAllStringSubmatchIndex(text, -1) { + ranges = append(ranges, [2]int{span[2], span[3]}) } - var out strings.Builder - pos := 0 - for _, span := range matches { - fullStart := span[0] - urlStart, urlEnd := span[2], span[3] - if urlStart < 0 || urlEnd < 0 { - continue - } - if fullStart < pos { - continue - } - out.WriteString(text[pos:urlStart]) - placeholder := nextPlaceholder() - mapping[placeholder] = text[urlStart:urlEnd] - *placeholders = append(*placeholders, placeholder) - out.WriteString(placeholder) - pos = urlEnd - } - out.WriteString(text[pos:]) - return out.String() + return maskByteRanges(text, ranges, nextPlaceholder, placeholders, mapping) } func unmaskMarkdown(text string, placeholders []string, mapping map[string]string) string { diff --git a/scripts/docs-i18n/process.go b/scripts/docs-i18n/process.go index a4601ccd8349..69787382444b 100644 --- a/scripts/docs-i18n/process.go +++ b/scripts/docs-i18n/process.go @@ -139,31 +139,22 @@ func translateFrontMatter(ctx context.Context, translator docsTranslator, tm *Tr if len(data) == 0 { return nil } - if summary, ok := data["summary"].(string); ok { - if docsI18nVerboseLogs() { - log.Printf("docs-i18n: frontmatter start %s field=summary bytes=%d", relPath, len(summary)) + for _, field := range []string{"summary", "title"} { + text, ok := data[field].(string) + if !ok { + continue } - translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:summary", summary, srcLang, tgtLang) + if docsI18nVerboseLogs() { + log.Printf("docs-i18n: frontmatter start %s field=%s bytes=%d", relPath, field, len(text)) + } + translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:"+field, text, srcLang, tgtLang) if err != nil { return err } if docsI18nVerboseLogs() { - log.Printf("docs-i18n: frontmatter done %s field=summary out_bytes=%d", relPath, len(translated)) + log.Printf("docs-i18n: frontmatter done %s field=%s out_bytes=%d", relPath, field, len(translated)) } - data["summary"] = translated - } - if title, ok := data["title"].(string); ok { - if docsI18nVerboseLogs() { - log.Printf("docs-i18n: frontmatter start %s field=title bytes=%d", relPath, len(title)) - } - translated, err := translateSnippet(ctx, translator, tm, relPath+":frontmatter:title", title, srcLang, tgtLang) - if err != nil { - return err - } - if docsI18nVerboseLogs() { - log.Printf("docs-i18n: frontmatter done %s field=title out_bytes=%d", relPath, len(translated)) - } - data["title"] = translated + data[field] = translated } if readWhen, ok := data["read_when"].([]any); ok { translated := make([]any, 0, len(readWhen)) @@ -218,11 +209,9 @@ func translateSnippet(ctx context.Context, translator docsTranslator, tm *Transl log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, err) return textValue, nil } - shouldCache := true if validationErr := validateFrontmatterScalarTranslation(textValue, translated); validationErr != nil { log.Printf("docs-i18n: frontmatter fallback %s reason=%v", segmentID, validationErr) - translated = textValue - shouldCache = false + return textValue, nil } sourcePath := segmentID if path, _, ok := strings.Cut(segmentID, ":frontmatter:"); ok { @@ -239,9 +228,7 @@ func translateSnippet(ctx context.Context, translator docsTranslator, tm *Transl TgtLang: tgtLang, UpdatedAt: time.Now().UTC().Format(time.RFC3339), } - if shouldCache { - tm.Put(entry) - } + tm.Put(entry) return translated, nil } diff --git a/scripts/docs-i18n/util.go b/scripts/docs-i18n/util.go index 5df1e1e52420..5ab5190d518b 100644 --- a/scripts/docs-i18n/util.go +++ b/scripts/docs-i18n/util.go @@ -30,14 +30,11 @@ func cacheNamespace() string { func cacheKey(namespace, srcLang, tgtLang, segmentID, textHash string) string { raw := fmt.Sprintf("%s|%s|%s|%s|%s", namespace, srcLang, tgtLang, segmentID, textHash) - hash := sha256.Sum256([]byte(raw)) - return hex.EncodeToString(hash[:]) + return hashBytes([]byte(raw)) } func hashText(text string) string { - normalized := normalizeText(text) - hash := sha256.Sum256([]byte(normalized)) - return hex.EncodeToString(hash[:]) + return hashBytes([]byte(normalizeText(text))) } func hashBytes(data []byte) string { diff --git a/scripts/github/barnacle-auto-response.mjs b/scripts/github/barnacle-auto-response.mjs index 6cdfc4692b5e..11406dff7b6c 100644 --- a/scripts/github/barnacle-auto-response.mjs +++ b/scripts/github/barnacle-auto-response.mjs @@ -733,11 +733,7 @@ async function isPrivilegedTargetAuthor(github, context, target, labelSet, isMai if (labelSet.has(maintainerAuthorLabel) || privilegedAuthorAssociations.has(authorAssociation)) { return true; } - if (await isPrivilegedActor(github, context, authorLogin, isMaintainer)) { - return true; - } - - return false; + return isPrivilegedActor(github, context, authorLogin, isMaintainer); } async function countMaintainerMentions(body, authorLogin, isMaintainer, owner) { @@ -772,15 +768,6 @@ async function countMaintainerMentions(body, authorLogin, isMaintainer, owner) { return count; } -async function listPullRequestFiles(github, context, pullRequest) { - return github.paginate(github.rest.pulls.listFiles, { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pullRequest.number, - per_page: 100, - }); -} - async function addMissingLabels(github, context, core, issueNumber, labels, labelSet) { const missingLabels = labels.filter((label) => !labelSet.has(label)); if (missingLabels.length === 0) { @@ -803,7 +790,12 @@ function isClawSweeperOwnedLabel(label) { } async function applyPullRequestCandidateLabels(github, context, core, pullRequest, labelSet) { - const files = await listPullRequestFiles(github, context, pullRequest); + const files = await github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pullRequest.number, + per_page: 100, + }); const candidateLabelsToApply = classifyPullRequestCandidateLabels( { ...pullRequest, @@ -851,10 +843,6 @@ function isClawSweeperProofSufficientLabelEvent(context) { ); } -function isGitHubAppPullRequestAuthor(pullRequest) { - return isAutomationUser(pullRequest.user); -} - function candidateActionRuleForLabelSet(labelSet, preferredLabel = "") { const preferredRule = candidateActionRules.find( (rule) => rule.label === preferredLabel && labelSet.has(rule.label), @@ -891,23 +879,29 @@ async function applyPullRequestCandidateAction({ return false; } - await github.rest.issues.createComment({ + await applyResponseRule(github, context, pullRequest.number, rule); + return true; +} + +async function applyResponseRule(github, context, issueNumber, rule) { + const target = { owner: context.repo.owner, repo: context.repo.repo, - issue_number: pullRequest.number, - body: rule.message, - }); - + issue_number: issueNumber, + }; + if (rule.message) { + await github.rest.issues.createComment({ ...target, body: rule.message }); + } if (rule.close) { await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, + ...target, state: "closed", + ...(rule.stateReason ? { state_reason: rule.stateReason } : {}), }); } - - return true; + if (rule.lock) { + await github.rest.issues.lock({ ...target, lock_reason: rule.lockReason ?? "resolved" }); + } } async function removeLabels(github, context, issueNumber, labels, labelSet) { @@ -1067,35 +1061,21 @@ export async function runBarnacleAutoResponse({ github, context, core = console const title = issue.title ?? ""; const body = issue.body ?? ""; const haystack = `${title}\n${body}`.toLowerCase(); - const hasMoltbookLabel = labelSet.has("r: moltbook"); - const hasTestflightLabel = labelSet.has("r: testflight"); - const hasSecurityLabel = labelSet.has("security"); - if (title.toLowerCase().includes("security") && !hasSecurityLabel) { + for (const [label, matches] of [ + ["security", title.toLowerCase().includes("security")], + ["r: testflight", title.toLowerCase().includes("testflight")], + ["r: moltbook", haystack.includes("moltbook")], + ]) { + if (!matches || labelSet.has(label)) { + continue; + } await github.rest.issues.addLabels({ owner: context.repo.owner, repo: context.repo.repo, issue_number: issue.number, - labels: ["security"], + labels: [label], }); - labelSet.add("security"); - } - if (title.toLowerCase().includes("testflight") && !hasTestflightLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: ["r: testflight"], - }); - labelSet.add("r: testflight"); - } - if (haystack.includes("moltbook") && !hasMoltbookLabel) { - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - labels: ["r: moltbook"], - }); - labelSet.add("r: moltbook"); + labelSet.add(label); } } @@ -1116,7 +1096,7 @@ export async function runBarnacleAutoResponse({ github, context, core = console return; } - if (isGitHubAppPullRequestAuthor(pullRequest)) { + if (isAutomationUser(pullRequest.user)) { await removeLabels(github, context, pullRequest.number, [activePrLimitLabel], labelSet); core.info(`Skipping active PR limit for GitHub App-authored PR #${pullRequest.number}.`); } @@ -1131,42 +1111,22 @@ export async function runBarnacleAutoResponse({ github, context, core = console } if (labelSet.has(dirtyLabel)) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - body: noisyPrMessage, - }); - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", + await applyResponseRule(github, context, pullRequest.number, { + message: noisyPrMessage, + close: true, }); return; } if (labelSet.has(spamLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", - }); - await github.rest.issues.lock({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - lock_reason: "spam", + await applyResponseRule(github, context, pullRequest.number, { + close: true, + lock: true, + lockReason: "spam", }); return; } if (labelSet.has(invalidLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: pullRequest.number, - state: "closed", - }); + await applyResponseRule(github, context, pullRequest.number, { close: true }); return; } @@ -1184,29 +1144,19 @@ export async function runBarnacleAutoResponse({ github, context, core = console } if (issue && labelSet.has(spamLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - state: "closed", - state_reason: "not_planned", - }); - await github.rest.issues.lock({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - lock_reason: "spam", + await applyResponseRule(github, context, issue.number, { + close: true, + stateReason: "not_planned", + lock: true, + lockReason: "spam", }); return; } if (issue && labelSet.has(invalidLabel)) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issue.number, - state: "closed", - state_reason: "not_planned", + await applyResponseRule(github, context, issue.number, { + close: true, + stateReason: "not_planned", }); return; } @@ -1214,10 +1164,7 @@ export async function runBarnacleAutoResponse({ github, context, core = console if (pullRequest && labelSet.has(activePrLimitOverrideLabel)) { labelSet.delete(activePrLimitLabel); } - if ( - pullRequest && - (isAutomationPullRequest(pullRequest) || isGitHubAppPullRequestAuthor(pullRequest)) - ) { + if (pullRequest && (isAutomationPullRequest(pullRequest) || isAutomationUser(pullRequest.user))) { await removeLabels(github, context, pullRequest.number, [activePrLimitLabel], labelSet); } @@ -1226,30 +1173,5 @@ export async function runBarnacleAutoResponse({ github, context, core = console return; } - const issueNumber = target.number; - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - body: rule.message, - }); - - if (rule.close) { - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - state: "closed", - }); - } - - if (rule.lock) { - await github.rest.issues.lock({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber, - lock_reason: rule.lockReason ?? "resolved", - }); - } + await applyResponseRule(github, context, target.number, rule); } diff --git a/scripts/github/dependency-guard.mjs b/scripts/github/dependency-guard.mjs index 396c7a6fade7..839ee4c85003 100644 --- a/scripts/github/dependency-guard.mjs +++ b/scripts/github/dependency-guard.mjs @@ -395,19 +395,8 @@ function decodeContentFile(payload) { return Buffer.from(payload.content, payload.encoding ?? "base64").toString("utf8"); } -async function readJsonFileAtRef(api, { owner, repo, path, ref }) { - if (!ref) { - return null; - } - const encodedPath = path.split("/").map(encodeURIComponent).join("/"); - const payload = await api - .request(`/repos/${owner}/${repo}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`) - .catch((error) => { - if (error?.status === 404) { - return null; - } - throw error; - }); +async function readJsonFileAtRef(api, options) { + const payload = await readContentFileMetadataAtRef(api, options); const text = decodeContentFile(payload); return text ? JSON.parse(text) : null; } diff --git a/scripts/github/real-behavior-proof-policy.mjs b/scripts/github/real-behavior-proof-policy.mjs index dbc64bbdd9b8..47b9f23f7c02 100644 --- a/scripts/github/real-behavior-proof-policy.mjs +++ b/scripts/github/real-behavior-proof-policy.mjs @@ -114,10 +114,6 @@ export async function readBoundedGitHubApiJson( return JSON.parse(text); } -async function cancelGitHubApiResponseBody(response) { - await response.body?.cancel?.().catch(() => undefined); -} - function normalizeLineEndings(text = "") { return text.replace(/\r\n?/g, "\n"); } @@ -172,10 +168,6 @@ function maskHtmlComments(text) { .join("\n"); } -function stripHtmlComments(text) { - return maskHtmlComments(text); -} - function isAutomationUser(user = {}, fallbackLogin = "") { const login = user?.login ?? fallbackLogin; return user?.type === "Bot" || /\[bot\]$/i.test(login) || login.startsWith("app/"); @@ -252,7 +244,7 @@ export async function isMaintainerTeamMember({ ); return body?.state === "active"; } finally { - await cancelGitHubApiResponseBody(response); + await response.body?.cancel?.().catch(() => undefined); } } @@ -336,10 +328,6 @@ function legacyProofFieldLineValue(line) { return match?.[1] ?? null; } -function isAnyLegacyProofFieldLine(line) { - return legacyProofFieldLineValue(line) !== null; -} - function extractFieldValue(section, field) { const lines = maskHtmlComments(normalizeLineEndings(section)).split("\n"); let fenceMarker = ""; @@ -361,7 +349,7 @@ function extractFieldValue(section, field) { const lineLocal = lines[next]; if ( !fenceMarker && - (markdownHeadingLevel(lineLocal) > 0 || isAnyLegacyProofFieldLine(lineLocal)) + (markdownHeadingLevel(lineLocal) > 0 || legacyProofFieldLineValue(lineLocal) !== null) ) { break; } @@ -374,7 +362,7 @@ function extractFieldValue(section, field) { } function stripMarkdownFenceMarkers(value) { - return stripHtmlComments(normalizeLineEndings(value)) + return maskHtmlComments(normalizeLineEndings(value)) .split("\n") .filter((line) => !/^ {0,3}(?:`{3,}|~{3,})(?:.*)?$/.test(line)) .join("\n") diff --git a/scripts/k8s/create-kind.sh b/scripts/k8s/create-kind.sh index 4e8ba8e4945e..110ee8fa2541 100755 --- a/scripts/k8s/create-kind.sh +++ b/scripts/k8s/create-kind.sh @@ -3,10 +3,7 @@ if [[ ${OSTYPE:-} == darwin* && $BASH != /bin/bash ]] && ((BASH_VERSINFO[0] > 5 || (BASH_VERSINFO[0] == 5 && BASH_VERSINFO[1] >= 3))); then exec /bin/bash "$0" "$@" fi -# ============================================================================ -# KIND CLUSTER BOOTSTRAP SCRIPT -# ============================================================================ -# +# Bootstrap a local Kind cluster. # Usage: # ./scripts/k8s/create-kind.sh # Create with auto-detected engine # ./scripts/k8s/create-kind.sh --name mycluster @@ -14,16 +11,13 @@ fi # # After creation, deploy with: # export _API_KEY="..." && ./scripts/k8s/deploy.sh -# ============================================================================ set -euo pipefail -# Defaults CLUSTER_NAME="openclaw" CONTAINER_CMD="" DELETE=false -# Colors GREEN='\033[0;32m' BLUE='\033[0;34m' YELLOW='\033[0;33m' @@ -53,9 +47,6 @@ EOF exit 0 } -# --------------------------------------------------------------------------- -# Argument parsing -# --------------------------------------------------------------------------- while [[ $# -gt 0 ]]; do case "$1" in --name) @@ -70,39 +61,18 @@ while [[ $# -gt 0 ]]; do esac done -# --------------------------------------------------------------------------- -# Container engine detection -# --------------------------------------------------------------------------- -provider_installed() { - command -v "$1" &>/dev/null -} - -provider_responsive() { - case "$1" in - docker) - docker info &>/dev/null - ;; - podman) - podman info &>/dev/null - ;; - *) - return 1 - ;; - esac -} - detect_provider() { local candidate for candidate in podman docker; do - if provider_installed "$candidate" && provider_responsive "$candidate"; then + if command -v "$candidate" &>/dev/null && "$candidate" info &>/dev/null; then echo "$candidate" return 0 fi done for candidate in podman docker; do - if provider_installed "$candidate"; then + if command -v "$candidate" &>/dev/null; then case "$candidate" in podman) fail "Podman is installed but not responding, and no responsive Docker daemon was found. Ensure the podman machine is running (podman machine start) or start Docker." @@ -120,9 +90,6 @@ detect_provider() { CONTAINER_CMD=$(detect_provider) info "Auto-detected container engine: $CONTAINER_CMD" -# --------------------------------------------------------------------------- -# Prerequisites -# --------------------------------------------------------------------------- if ! command -v kind &>/dev/null; then fail "kind is not installed. Install it from https://kind.sigs.k8s.io/" fi @@ -131,8 +98,7 @@ if ! command -v kubectl &>/dev/null; then fail "kubectl is not installed. Install it before creating or managing a Kind cluster." fi -# Verify the container engine is responsive -if ! provider_responsive "$CONTAINER_CMD"; then +if ! "$CONTAINER_CMD" info &>/dev/null; then if [[ "$CONTAINER_CMD" == "docker" ]]; then fail "Docker daemon is not running. Start it and try again." elif [[ "$CONTAINER_CMD" == "podman" ]]; then @@ -140,9 +106,6 @@ if ! provider_responsive "$CONTAINER_CMD"; then fi fi -# --------------------------------------------------------------------------- -# Delete mode -# --------------------------------------------------------------------------- if $DELETE; then info "Deleting Kind cluster '$CLUSTER_NAME'..." if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then @@ -154,9 +117,6 @@ if $DELETE; then exit 0 fi -# --------------------------------------------------------------------------- -# Check if cluster already exists -# --------------------------------------------------------------------------- if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | grep -qx "$CLUSTER_NAME"; then warn "Cluster '$CLUSTER_NAME' already exists." info "To recreate it, run: $0 --name \"$CLUSTER_NAME\" --delete && $0 --name \"$CLUSTER_NAME\"" @@ -165,9 +125,6 @@ if KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind get clusters 2>/dev/null | g exit 0 fi -# --------------------------------------------------------------------------- -# Create cluster -# --------------------------------------------------------------------------- info "Creating Kind cluster '$CLUSTER_NAME' (provider: $CONTAINER_CMD)..." KIND_EXPERIMENTAL_PROVIDER="$CONTAINER_CMD" kind create cluster \ @@ -191,16 +148,10 @@ KINDCFG success "Kind cluster '$CLUSTER_NAME' created." -# --------------------------------------------------------------------------- -# Wait for readiness -# --------------------------------------------------------------------------- info "Waiting for cluster to be ready..." kubectl --context "kind-$CLUSTER_NAME" wait --for=condition=Ready nodes --all --timeout=120s >/dev/null success "All nodes are Ready." -# --------------------------------------------------------------------------- -# Summary -# --------------------------------------------------------------------------- echo "" echo "---------------------------------------------------------------" echo " Kind cluster '$CLUSTER_NAME' is ready" diff --git a/scripts/k8s/deploy.sh b/scripts/k8s/deploy.sh index 9c25f86499b9..f28790eb822f 100755 --- a/scripts/k8s/deploy.sh +++ b/scripts/k8s/deploy.sh @@ -24,15 +24,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MANIFESTS="$SCRIPT_DIR/manifests" NS="${OPENCLAW_NAMESPACE:-openclaw}" -# Check prerequisites for cmd in kubectl openssl; do command -v "$cmd" &>/dev/null || { echo "Missing: $cmd" >&2; exit 1; } done kubectl cluster-info &>/dev/null || { echo "Cannot connect to cluster. Check kubeconfig." >&2; exit 1; } -# --------------------------------------------------------------------------- -# -h / --help -# --------------------------------------------------------------------------- if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then cat <<'HELP' Usage: ./scripts/k8s/deploy.sh [OPTION] @@ -61,17 +57,8 @@ MODE="deploy" while [[ $# -gt 0 ]]; do case "$1" in - --create-secret) - MODE="create-secret" - ;; - --delete) - MODE="delete" - ;; - --delete-resources) - MODE="delete-resources" - ;; - --delete-namespace) - MODE="delete-namespace" + --create-secret | --delete | --delete-resources | --delete-namespace) + MODE="${1#--}" ;; --show-token) SHOW_TOKEN=true @@ -85,9 +72,6 @@ while [[ $# -gt 0 ]]; do shift done -# --------------------------------------------------------------------------- -# --delete / --delete-namespace -# --------------------------------------------------------------------------- if [[ "$MODE" == "delete" && "$NS" != "openclaw" ]]; then MODE="delete-resources" fi @@ -99,9 +83,6 @@ if [[ "$MODE" == "delete" || "$MODE" == "delete-namespace" ]]; then exit 0 fi -# --------------------------------------------------------------------------- -# --delete-resources -# --------------------------------------------------------------------------- if [[ "$MODE" == "delete-resources" ]]; then echo "Deleting OpenClaw resources from namespace '$NS'..." kubectl delete -k "$MANIFESTS" -n "$NS" --ignore-not-found @@ -110,9 +91,6 @@ if [[ "$MODE" == "delete-resources" ]]; then exit 0 fi -# --------------------------------------------------------------------------- -# Create and apply Secret to the cluster -# --------------------------------------------------------------------------- _apply_secret() { local TMP_DIR local EXISTING_SECRET=false @@ -189,9 +167,6 @@ _apply_secret() { fi } -# --------------------------------------------------------------------------- -# --create-secret -# --------------------------------------------------------------------------- if [[ "$MODE" == "create-secret" ]]; then HAS_KEY=false for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY; do @@ -215,9 +190,6 @@ if [[ "$MODE" == "create-secret" ]]; then exit 0 fi -# --------------------------------------------------------------------------- -# Check that the secret exists in the cluster -# --------------------------------------------------------------------------- if ! kubectl get secret openclaw-secrets -n "$NS" &>/dev/null; then HAS_KEY=false for key in ANTHROPIC_API_KEY OPENAI_API_KEY GEMINI_API_KEY OPENROUTER_API_KEY; do @@ -238,9 +210,6 @@ if ! kubectl get secret openclaw-secrets -n "$NS" &>/dev/null; then fi fi -# --------------------------------------------------------------------------- -# Deploy -# --------------------------------------------------------------------------- echo "Deploying to namespace '$NS'..." kubectl create namespace "$NS" --dry-run=client -o yaml | kubectl apply -f - >/dev/null kubectl apply -k "$MANIFESTS" -n "$NS" diff --git a/scripts/mantis/publish-pr-evidence.mjs b/scripts/mantis/publish-pr-evidence.mjs index 9df72d9c7f72..d3dcaad56a09 100644 --- a/scripts/mantis/publish-pr-evidence.mjs +++ b/scripts/mantis/publish-pr-evidence.mjs @@ -467,9 +467,6 @@ function laneLine(label, lane) { } return pieces.join(""); } -function publicSummary(manifest) { - return manifest.summary ?? "Mantis captured QA evidence for this scenario."; -} function overallStatus(manifest) { const outcome = manifest.comparison?.outcome; if (outcome === "blocked" || outcome === "fail" || outcome === "pass") { @@ -478,13 +475,6 @@ function overallStatus(manifest) { const pass = manifest.comparison?.pass; return typeof pass === "boolean" ? String(pass) : ""; } -/** - * @param {EvidenceManifest} manifest - * @param {{ requestSource?: string }} [options] - */ -export function shouldPublishPrComment() { - return true; -} /** @param {RenderEvidenceCommentOptions} options */ export function renderEvidenceComment({ artifactUrl: actionsArtifactUrl, @@ -511,7 +501,7 @@ export function renderEvidenceComment({ marker, `## ${manifest.title}`, "", - `Summary: ${publicSummary(manifest)}`, + `Summary: ${manifest.summary ?? "Mantis captured QA evidence for this scenario."}`, "", `- Scenario: \`${manifest.scenario}\``, ]; @@ -542,8 +532,7 @@ export function renderEvidenceComment({ lines.push(`- Overall: \`${overall}\``); } lines.push(""); - const pairedSections = pairs.map((pair) => renderPairTable({ pair, rawBase })); - lines.push(...pairedSections); + lines.push(...pairs.map((pair) => renderPairTable({ pair, rawBase }))); const singleTables = renderSingleImageTables({ artifacts: manifest.artifacts, pairedKeys, @@ -552,23 +541,14 @@ export function renderEvidenceComment({ if (singleTables) { lines.push(singleTables); } - const motionClips = renderLinkList({ - artifacts: manifest.artifacts, - kind: "motionClip", - rawBase, - title: "Motion-trimmed clips", - }); - if (motionClips) { - lines.push(motionClips); - } - const fullVideos = renderLinkList({ - artifacts: manifest.artifacts, - kind: "fullVideo", - rawBase, - title: "Full videos", - }); - if (fullVideos) { - lines.push(fullVideos); + for (const [kind, title] of [ + ["motionClip", "Motion-trimmed clips"], + ["fullVideo", "Full videos"], + ]) { + const links = renderLinkList({ artifacts: manifest.artifacts, kind, rawBase, title }); + if (links) { + lines.push(links); + } } lines.push(`Raw QA files: ${treeUrl ?? rawBase}`); return `${lines.join("\n").replace(/\n{3,}/gu, "\n\n")}\n`; @@ -769,10 +749,6 @@ export async function publishEvidence(rawArgs = process.argv.slice(2)) { runUrl: args.run_url, treeUrl: published.treeUrl, }); - if (!shouldPublishPrComment(manifest, { requestSource: args.request_source })) { - console.log("Skipped Mantis QA evidence PR comment because the run did not capture proof."); - return; - } upsertPrComment({ body, createMissing: args.create_missing !== "false", diff --git a/scripts/perf/summarize-cpuprofile.mts b/scripts/perf/summarize-cpuprofile.mts index e92c432739b2..5350003302c1 100644 --- a/scripts/perf/summarize-cpuprofile.mts +++ b/scripts/perf/summarize-cpuprofile.mts @@ -63,9 +63,6 @@ export function shouldPrintHelp(argv: readonly string[]): boolean { return false; } -/** - * Parses CPU profile file paths and --limit. - */ export function parseArgs(argv: readonly string[]): { files: string[]; limit: number } { const files: string[] = []; let limit = DEFAULT_LIMIT; @@ -150,7 +147,7 @@ function summarizeProfile(file: string, limit: number): void { const profile: unknown = JSON.parse(fs.readFileSync(file, "utf8")); validateProfile(profile, file); const nodes = new Map(profile.nodes.map((node) => [node.id, node])); - const samples = Array.isArray(profile.samples) ? profile.samples : []; + const samples = profile.samples; const deltas = Array.isArray(profile.timeDeltas) ? profile.timeDeltas : []; const byFrame = new Map(); const byModule = new Map(); @@ -178,17 +175,16 @@ function summarizeProfile(file: string, limit: number): void { const durationMs = (profile.endTime - profile.startTime) / 1000; console.log(`\n${file}`); console.log(`duration_ms: ${durationMs.toFixed(1)} samples: ${samples.length}`); - console.log("top_frames:"); - for (const [key, micros] of [...byFrame.entries()] - .toSorted((left, right) => right[1] - left[1]) - .slice(0, limit)) { - console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`); - } - console.log("top_modules:"); - for (const [key, micros] of [...byModule.entries()] - .toSorted((left, right) => right[1] - left[1]) - .slice(0, limit)) { - console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`); + for (const [label, totals] of [ + ["top_frames", byFrame], + ["top_modules", byModule], + ] as const) { + console.log(`${label}:`); + for (const [key, micros] of [...totals.entries()] + .toSorted((left, right) => right[1] - left[1]) + .slice(0, limit)) { + console.log(`${(micros / 1000).toFixed(1)}ms\t${key}`); + } } } diff --git a/scripts/podman/setup.sh b/scripts/podman/setup.sh index d7bdddef4920..2c0476e7037a 100755 --- a/scripts/podman/setup.sh +++ b/scripts/podman/setup.sh @@ -46,8 +46,6 @@ require_cmd() { fi } -is_root() { [[ "$(id -u)" -eq 0 ]]; } - run_podman_pull() { local image="$1" openclaw_host_timeout_cmd "$PODMAN_PULL_TIMEOUT" podman pull "$image" @@ -181,7 +179,7 @@ if [[ "$INSTALL_QUADLET" == true ]]; then fi require_cmd podman -if is_root; then +if [[ "$(id -u)" -eq 0 ]]; then echo "Run scripts/podman/setup.sh as your normal user so Podman stays rootless." >&2 exit 1 fi diff --git a/test/scripts/gateway-ws-client.test.ts b/test/scripts/gateway-ws-client.test.ts index e64e6c9ac26c..7af336d3f393 100644 --- a/test/scripts/gateway-ws-client.test.ts +++ b/test/scripts/gateway-ws-client.test.ts @@ -2,7 +2,7 @@ import { createServer, type Server } from "node:http"; import type { Duplex } from "node:stream"; import { afterEach, describe, expect, it } from "vitest"; import { WebSocket, WebSocketServer } from "ws"; -import { createGatewayWsClient } from "../../scripts/dev/gateway-ws-client.js"; +import { createGatewayWsClient } from "../../scripts/lib/gateway-ws-client.js"; let server: Server | undefined; let wss: WebSocketServer | undefined; diff --git a/test/scripts/realtime-talk-live-smoke.test.ts b/test/scripts/realtime-talk-live-smoke.test.ts index 96578e41f88d..b4d0ca8b0ec5 100644 --- a/test/scripts/realtime-talk-live-smoke.test.ts +++ b/test/scripts/realtime-talk-live-smoke.test.ts @@ -1,4 +1,5 @@ import path from "node:path"; +import { runInNewContext } from "node:vm"; import { afterEach, expect, it, vi } from "vitest"; import type { RealtimeVoiceBridgeCreateRequest } from "../../src/talk/provider-types.js"; @@ -7,6 +8,7 @@ const backend = vi.hoisted(() => ({ onFirstAudio: () => {} })); const browser = vi.hoisted(() => ({ close: vi.fn(async () => {}), contextClose: vi.fn(async () => {}), + googleEvaluate: vi.fn(), evaluate: vi.fn(async () => ({ answerHasAudio: true, remoteDescriptionApplied: true, @@ -25,6 +27,7 @@ vi.mock("playwright", () => ({ chromium: { launch: async () => ({ close: browser.close, + newPage: async () => ({ evaluate: browser.googleEvaluate, close: async () => {} }), newContext: async () => ({ close: browser.contextClose, newPage: async () => ({ evaluate: browser.evaluate }), @@ -33,6 +36,24 @@ vi.mock("playwright", () => ({ }, })); +vi.mock("../../extensions/google/realtime-voice-provider.ts", () => ({ + buildGoogleRealtimeVoiceProvider: () => ({ + createBrowserSession: async () => ({ + transport: "provider-websocket", + protocol: "google-live-bidi", + clientSecret: "synthetic-google-session", + websocketUrl: "wss://google.example.test/live", + initialMessage: { setup: {} }, + }), + }), +})); + +vi.mock("vite", () => ({ + createServer: async () => { + throw new Error("Synthetic relay smoke unavailable"); + }, +})); + vi.mock("../../extensions/openai/realtime-voice-provider.ts", () => ({ buildOpenAIRealtimeVoiceProvider: () => ({ createBridge: (options: RealtimeVoiceBridgeCreateRequest) => { @@ -83,6 +104,58 @@ afterEach(() => { vi.restoreAllMocks(); }); +it("reports malformed Google Live frames from the serialized browser callback", async () => { + vi.resetModules(); + vi.clearAllMocks(); + vi.stubEnv("OPENAI_API_KEY", ""); + vi.stubEnv("GEMINI_API_KEY", "synthetic-google-key"); + const output = vi.spyOn(console, "log").mockImplementation(() => {}); + process.argv = [process.execPath, path.resolve("scripts/dev/realtime-talk-live-smoke.ts")]; + process.exitCode = 0; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + let started!: () => void; + const callbackStarted = new Promise((resolve) => { + started = resolve; + }); + browser.googleEvaluate.mockImplementation((callback, payload) => { + if (typeof callback === "string") { + return undefined; + } + const result = runInNewContext(`(${callback.toString()})(payload)`, { + payload, + URL, + __name: (value: unknown) => value, + window: { setTimeout, clearTimeout }, + WebSocket: class { + addEventListener(event: string, listener: (message: { data: string }) => void) { + if (event === "message") { + queueMicrotask(() => listener({ data: "not-json" })); + } + } + close() {} + }, + }); + started(); + return result; + }); + const command = import("../../scripts/dev/realtime-talk-live-smoke.ts"); + await Promise.race([ + callbackStarted, + command.then(() => { + throw new Error("Smoke command completed before the Google browser callback"); + }), + ]); + await vi.runAllTimersAsync(); + await command; + + expect(output).toHaveBeenCalledWith("google-live-browser-ws: failed", { + error: expect.stringContaining("not valid JSON"), + }); + expect(vi.getTimerCount()).toBe(0); + expect(process.exitCode).toBe(1); + expect(browser.close).toHaveBeenCalledOnce(); +}); + it.each([ { name: "failed browser media", diff --git a/test/scripts/secret-scanning-maintainer.test.ts b/test/scripts/secret-scanning-maintainer.test.ts index 56ea411d870f..33b1206fef56 100644 --- a/test/scripts/secret-scanning-maintainer.test.ts +++ b/test/scripts/secret-scanning-maintainer.test.ts @@ -9,6 +9,83 @@ const scriptPath = ".agents/skills/openclaw-secret-scanning-maintainer/scripts/s const { createTempDir } = createScriptTestHarness(); describe("secret scanning maintainer script", () => { + it.each([ + ["issue_body", "Issue", "issue_number"], + ["pull_request_body", "PullRequest", "pr_number"], + ["pull_request_review_comment", "PullRequestReviewComment", "issue_number"], + ])( + "keeps %s content private while reporting identity and edit history", + (type, nodeType, numberKey) => { + const tempDir = createTempDir("openclaw-secret-scan-content-"); + const ghPath = path.join(tempDir, "gh"); + const fixturePath = path.join(tempDir, "content.json"); + const body = "synthetic private content\n"; + const htmlUrl = "https://github.com/openclaw/openclaw/pull/123"; + fs.writeFileSync( + fixturePath, + JSON.stringify({ + id: 456, + node_id: "fixture-node", + number: 123, + user: { login: "contributor" }, + body, + html_url: htmlUrl, + merged: false, + state: "open", + }), + ); + fs.writeFileSync( + ghPath, + `#!/usr/bin/env node +const fs = require("node:fs"); +const args = process.argv.slice(2); +if (args[0] !== "api") process.exit(1); +if (args[1] === "graphql") { + if (!args.join(" ").includes("... on " + process.env.EXPECTED_NODE_TYPE)) process.exit(2); + console.log(JSON.stringify({ data: { node: { userContentEdits: { totalCount: 3 } } } })); +} else { + process.stdout.write(fs.readFileSync(process.env.CONTENT_FIXTURE, "utf8")); +} +`, + { mode: 0o755 }, + ); + const output = execFileSync( + process.execPath, + [ + scriptPath, + "fetch-content", + JSON.stringify({ type, details: { [`${type}_url`]: "https://api.github.com/fixture" } }), + ], + { + encoding: "utf8", + env: { + ...process.env, + TMPDIR: tempDir, + OPENCLAW_GH_BIN: ghPath, + PATH: `${tempDir}${path.delimiter}${process.env.PATH ?? ""}`, + EXPECTED_NODE_TYPE: nodeType, + CONTENT_FIXTURE: fixturePath, + }, + }, + ); + const result = JSON.parse(output); + expect(result).toEqual({ + type, + ...(type === "pull_request_review_comment" ? { comment_id: 456 } : {}), + [numberKey]: type === "pull_request_review_comment" ? "123" : 123, + node_id: "fixture-node", + author: "contributor", + ...(type === "pull_request_body" ? { merged: false, state: "open" } : {}), + html_url: htmlUrl, + edit_history_count: 3, + body_file: expect.any(String), + }); + expect(output).not.toContain(body.trim()); + expect(fs.readFileSync(result.body_file, "utf8")).toBe(body); + expect(fs.statSync(result.body_file).mode & 0o777).toBe(0o600); + }, + ); + it("marks body alerts as not requiring notification when redaction is unchanged", () => { const tempDir = createTempDir("openclaw-secret-scan-"); const currentBody = path.join(tempDir, "current.md");