mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
ci: batch compiler planning and start guards after preflight (#163196)
Read complete compiler memberships from one native snapshot while retaining canonical graph ownership checks and conservative full selection. Keep the serial CLI boundary owner and provide OPENCLAW_CI_TYPE_PLAN_SERIAL=true (or 1) as an operator fallback; unset uses snapshot discovery. No repository setting is created. Move existing narrow-PR guards into the preflight-ready additional matrix, reusing their commands, setup, runner and comparison base. Preserve full and frozen layouts, coercion deduplication and the versioned observer count. Four fresh Linux PR-input replays preserved every output field. Compiler planning fell from 77-91s to 17-21s; complete materialization fell from 85-101s to 25-32s. The materializer peaked at 14.8 GiB RSS, so the existing eligible hybrid planner uses the 16-class. Hosted and trust fallbacks stay unchanged; no jobs or permissions are added.
This commit is contained in:
Peter Steinberger
GitHub
parent
ed71fa44a4
commit
7fc0cfb379
8 changed files with 256 additions and 75 deletions
14
.github/workflows/ci.yml
vendored
14
.github/workflows/ci.yml
vendored
|
|
@ -3424,7 +3424,7 @@ jobs:
|
|||
name: check-plan
|
||||
needs: [preflight]
|
||||
if: needs.preflight.outputs.run_check_plan == 'true'
|
||||
runs-on: ${{ (github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '') != '' && fromJSON(format('{{"group":{0},"labels":{1}}}', toJSON((github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '')), toJSON((needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'hybrid' && needs.preflight.outputs.node_runner_backend != 'runson' && github.run_attempt == 1 && (github.event_name != 'workflow_dispatch' || needs.preflight.outputs.ci_qualification == 'true') && needs.preflight.outputs.frozen_target != 'true' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04'))) || ((needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'hybrid' && needs.preflight.outputs.node_runner_backend != 'runson' && github.run_attempt == 1 && (github.event_name != 'workflow_dispatch' || needs.preflight.outputs.ci_qualification == 'true') && needs.preflight.outputs.frozen_target != 'true' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04') }}
|
||||
runs-on: ${{ (github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '') != '' && fromJSON(format('{{"group":{0},"labels":{1}}}', toJSON((github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '')), toJSON((needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'hybrid' && needs.preflight.outputs.node_runner_backend != 'runson' && github.run_attempt == 1 && (github.event_name != 'workflow_dispatch' || needs.preflight.outputs.ci_qualification == 'true') && needs.preflight.outputs.frozen_target != 'true' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04'))) || ((needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'hybrid' && needs.preflight.outputs.node_runner_backend != 'runson' && github.run_attempt == 1 && (github.event_name != 'workflow_dispatch' || needs.preflight.outputs.ci_qualification == 'true') && needs.preflight.outputs.frozen_target != 'true' && github.repository == 'openclaw/openclaw' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && 'blacksmith-16vcpu-ubuntu-2404' || 'ubuntu-24.04') }}
|
||||
timeout-minutes: 20
|
||||
outputs:
|
||||
check_matrix: ${{ steps.plan.outputs.check_matrix }}
|
||||
|
|
@ -3459,6 +3459,7 @@ jobs:
|
|||
id: plan
|
||||
env:
|
||||
OPENCLAW_CI_CHECK_PLAN_INPUT_JSON: ${{ needs.preflight.outputs.check_plan_input_json }}
|
||||
OPENCLAW_CI_TYPE_PLAN_SERIAL: ${{ vars.OPENCLAW_CI_TYPE_PLAN_SERIAL }}
|
||||
run: node scripts/ci-check-plan.mts
|
||||
# The failure observer depends on this step name.
|
||||
- name: "CI check job count v1: ${{ steps.plan.outputs.check_job_count }}"
|
||||
|
|
@ -4163,6 +4164,8 @@ jobs:
|
|||
if: ${{ !cancelled() && always() && needs.preflight.outputs.run_check_additional == 'true' }}
|
||||
runs-on: ${{ (github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '') != '' && fromJSON(format('{{"group":{0},"labels":{1}}}', toJSON((github.event_name == 'workflow_dispatch' && startsWith(inputs.dispatch_id, 'full-release-validation-') && vars.OPENCLAW_RELEASE_RUNNER_GROUP || '')), toJSON((contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && github.run_attempt == 1 && needs.preflight.outputs.hybrid_hosted_checks == 'true' && contains(fromJSON('["runtime-topology-architecture","dependencies"]'), matrix.group)) && 'ubuntu-24.04' || (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'github' && 'ubuntu-24.04' || (contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && (github.run_attempt > 1 || !contains(fromJSON('["extension-package-boundary","runtime-topology-architecture","plugin-sdk-api-diff","dependencies"]'), matrix.group))) && 'ubuntu-24.04' || ((github.event_name == 'workflow_dispatch' && ((needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') || github.run_attempt != 1)) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || 'ubuntu-24.04')))) || ((contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && github.run_attempt == 1 && needs.preflight.outputs.hybrid_hosted_checks == 'true' && contains(fromJSON('["runtime-topology-architecture","dependencies"]'), matrix.group)) && 'ubuntu-24.04' || (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND) == 'github' && 'ubuntu-24.04' || (contains(fromJSON('["hybrid","runson"]'), (needs.preflight.outputs.ci_qualification == 'true' && (github.run_attempt == 1 && needs.preflight.outputs.qualification_runner_backend || 'github') || vars.OPENCLAW_CI_RUNNER_BACKEND)) && (github.run_attempt > 1 || !contains(fromJSON('["extension-package-boundary","runtime-topology-architecture","plugin-sdk-api-diff","dependencies"]'), matrix.group))) && 'ubuntu-24.04' || ((github.event_name == 'workflow_dispatch' && ((needs.preflight.outputs.node_runner_backend != 'runson' && needs.preflight.outputs.ci_qualification != 'true') || github.run_attempt != 1)) || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && github.run_attempt > 1)) && 'ubuntu-24.04' || (github.repository == 'openclaw/openclaw' && (matrix.runner || 'blacksmith-4vcpu-ubuntu-2404') || 'ubuntu-24.04')) }}
|
||||
timeout-minutes: ${{ matrix.group == 'extension-package-boundary' && 30 || 20 }}
|
||||
env:
|
||||
CHECKOUT_BASE_SHA: ${{ matrix.group == 'guards' && (github.event_name == 'pull_request' || needs.preflight.outputs.node_runner_backend == 'runson' || needs.preflight.outputs.ci_qualification == 'true') && needs.preflight.outputs.diff_base_revision || '' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 12
|
||||
|
|
@ -4243,7 +4246,7 @@ jobs:
|
|||
with: *sdk_main_restore_inputs
|
||||
|
||||
- name: Run additional check shard
|
||||
if: matrix.group != 'dependencies'
|
||||
if: matrix.group != 'guards' && matrix.group != 'dependencies'
|
||||
env:
|
||||
SDK_DIFF_BASE: ${{ needs.preflight.outputs.diff_base_revision }}
|
||||
SDK_DIFF_HEAD: ${{ needs.preflight.outputs.diff_head_revision }}
|
||||
|
|
@ -4260,13 +4263,14 @@ jobs:
|
|||
shell: bash
|
||||
run: bash .ci-harness/scripts/ci-additional-checks.sh
|
||||
|
||||
- name: Run dependency checks
|
||||
if: matrix.group == 'dependencies'
|
||||
- name: Run independent checks
|
||||
if: matrix.group == 'guards' || matrix.group == 'dependencies'
|
||||
env:
|
||||
TASK: dependencies
|
||||
TASK: ${{ matrix.group }}
|
||||
OPENCLAW_LOCAL_CHECK: "0"
|
||||
FROZEN_TARGET: *frozen_target
|
||||
HISTORICAL_TARGET: *historical_target
|
||||
NARROW_CHECK_PATHS_JSON: ${{ needs.preflight.outputs.narrow_check_paths_json }}
|
||||
shell: bash
|
||||
run: *run_check_shard
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,9 @@ The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Bl
|
|||
|
||||
Full GitHub and hybrid type checks run the five core stripes independently, retaining two compiler children per job. The last four rows then each run one root-test partition serially, leaving extension tests and scripts in the central row. Narrow plans reuse four already-selected rows when available; smaller selections retain central root checking. This adds no jobs or compiler overlap. Current hybrid full runs use three hosted extension-lint jobs; targeted layouts retain six stripe identities. Trusted hybrid first attempts place both packed core-lint rows on the Blacksmith 16-class and the final gate on the 4-class to avoid serial hosted assignment delays. Frozen targets keep their earlier layout; see [static checks](/ci/runners#runner-backend-modes).
|
||||
|
||||
Additional checks and narrow-PR dependency scans start directly after preflight. Known full compiler selections skip discovery while retaining the core graph boundary in an existing required owner; see [pipeline ordering](/ci/pipeline#fail-fast-order).
|
||||
Additional checks and narrow-PR guards and dependency scans start directly after preflight. Guards retain the exact comparison base and shared check commands; compiler and lint rows wait for their selected graphs. Known full compiler selections skip discovery while retaining the core graph boundary in an existing required owner; see [pipeline ordering](/ci/pipeline#fail-fast-order).
|
||||
|
||||
Changed compiler planning reads every selected program from one native compiler snapshot. With `OPENCLAW_CI_TYPE_PLAN_SERIAL` unset, this avoids serial compiler discovery without changing graph membership or full fallback. Cold Linux replays reduced compiler planning from 77–91 seconds to 17–21 seconds on four available CPUs; the complete materializer reached 14.8 GiB peak RSS. Eligible hybrid `check-plan` jobs therefore use the 16-class. Hosted fallback, fork, retry, and frozen routing remain unchanged. Set the repository variable to `true` or `1` to restore serial queries.
|
||||
|
||||
The extension package boundary row has a 30-minute job budget for SDK preparation,
|
||||
all selected plugin compiles, input-receipt validation, the required negative
|
||||
|
|
|
|||
|
|
@ -752,7 +752,7 @@ automation account, and SecOps-owned-path cases before declaring enforcement act
|
|||
## Fail-fast order
|
||||
|
||||
1. `preflight` decides which lanes exist at all. The `docs-scope` and `changed-scope` logic are steps inside this job, not standalone jobs. Canonical `main` starts immediately in one of two parity slots; each slot admits one complete run and coalesces later pushes into its newest pending tip. Downstream jobs wait for the manifest, then eligible Blacksmith jobs restore exact dependencies from the trusted warmer or fall back to the ordinary pnpm-store cache on a miss. Pushes, pull requests, and manual runs targeting the workflow revision run preflight with native Node and skip dependency setup. Manual runs targeting a different revision install dependencies and retain that target's `tsx` tooling.
|
||||
2. `security-fast`, `check-*`, `check-additional-*`, `check-docs`, and `skills-python` fail quickly without waiting on the heavier artifact and platform matrix jobs. Additional checks start directly after preflight. Narrow PRs with additional checks also place the existing `check-dependencies` row there: its complete dependency, unused-file, and export scans do not consume the installed compiler/lint plan. Full selections and runs without that family retain the central row, and the aggregate requires the selected owner. Extension-only compiler inputs can be planned from the four noncore graphs when an already selected additional boundary row owns the full core graph check. This requires existing regular source files without symlink aliases; mixed source changes retain complete discovery, and missing or uncertain inputs retain full checking. Known full selections skip discovery and retain boundary proof in an already selected additional boundary row or the required planner. No extra boundary row is admitted for that optimization. The production dependency audit sends one complete graph with up to four attempts and a four-minute total request budget, including retries and response reading. Timeouts, native fetch failures, HTTP 429, and 5xx responses retry with exponential backoff; retryable HTTP responses honor `Retry-After`. Attempts and recovery are logged. Persistent unavailability, vulnerability findings, invalid inputs, malformed advisory data, oversized responses, and permanent HTTP failures block CI. An unavailable audit is incomplete coverage, not a clean result. CI dispatched by Full Release Validation or release publication records a failing audit as a warning instead, because advisories never block a release. Local pre-commit and release dependency audits use the same bounded request owner and fail on unavailability; release dependency evidence blocks only on known malware.
|
||||
2. `security-fast`, `check-*`, `check-additional-*`, `check-docs`, and `skills-python` fail quickly without waiting on the heavier artifact and platform matrix jobs. Additional checks start directly after preflight. Narrow PRs with additional checks also place the existing `check-guards` and `check-dependencies` rows there: their complete guard, dependency, unused-file, and export scans do not consume the installed compiler/lint plan. Guards use the same commands and fetch the exact comparison base during checkout. Full selections and runs without that family retain the central rows, and the aggregate requires each selected owner. Extension-only compiler inputs can be planned from the four noncore graphs when an already selected additional boundary row owns the full core graph check. This requires existing regular source files without symlink aliases; mixed source changes retain complete discovery, and missing or uncertain inputs retain full checking. Known full selections skip discovery and retain boundary proof in an already selected additional boundary row or the required planner. No extra boundary row is admitted for that optimization. The production dependency audit sends one complete graph with up to four attempts and a four-minute total request budget, including retries and response reading. Timeouts, native fetch failures, HTTP 429, and 5xx responses retry with exponential backoff; retryable HTTP responses honor `Retry-After`. Attempts and recovery are logged. Persistent unavailability, vulnerability findings, invalid inputs, malformed advisory data, oversized responses, and permanent HTTP failures block CI. An unavailable audit is incomplete coverage, not a clean result. CI dispatched by Full Release Validation or release publication records a failing audit as a warning instead, because advisories never block a release. Local pre-commit and release dependency audits use the same bounded request owner and fail on unavailability; release dependency evidence blocks only on known malware.
|
||||
3. `build-artifacts` and the locale checks overlap with the fast Linux lanes. Control UI and native app source PRs exclude generated locale snapshots/resources; their serialized refresh workflows repair and auto-merge isolated generated PRs in the background. Source CI still blocks stale source inventories and unsafe localization calls. Generated PRs, manual CI, and release prep enforce full translated/platform-generated parity. Canonical `release/YYYY.M.PATCH` branches may include release-prep locale repairs with the other generated release output.
|
||||
4. Baseline ratchets and selected Node test shards start independently after preflight. Node rows consume the manifest, not ratchet outputs. `ci-gate` still requires every selected ratchet to pass, and the PR failure monitor still cancels remaining work after a ratchet failure. Frozen targets retain their existing ratchet selection.
|
||||
5. Current plans with guards run `check:coercion-helpers` there once; fast-only plans retain its standalone row. Other platform and runtime lanes fan out independently: `checks-fast-core` (including startup corpus), `checks-fast-contracts-plugins`, `checks-fast-contracts-channels`, `checks-windows`, `macos-node`, `macos-swift`, `ios-build`, the screenshot shards, and `android`.
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ Runner choice does not depend on the pull request author. Same-repository PRs us
|
|||
|
||||
The table lists default placement. On eligible hybrid first attempts, the [hosted budget](/ci/capacity#bounded-hybrid-hosted-offload) can move `security-fast`, all three `checks-ui` rows, and only the browser-extension E2E row to `ubuntu-24.04`; the default Blacksmith routes apply when optional admission is closed.
|
||||
|
||||
Baseline ratchets and Node shards start independently after preflight; the final gate still requires the selected ratchets to pass. Standalone ratchets and `check-plan` use the existing Blacksmith 4-class on same-repository hybrid first attempts, automatic main runs, and admitted qualification dispatches. The GitHub override, hybrid retries, ordinary manual and frozen targets, and noncanonical contexts retain hosted routing. Narrowed check rows still wait for their complete compiler/lint plan. See [admission measurements and cost](/ci/routing-costs#ratchet-admission-and-node-tests).
|
||||
Baseline ratchets and Node shards start independently after preflight; the final gate still requires the selected ratchets to pass. Standalone ratchets use the Blacksmith 4-class; `check-plan` uses the 16-class for shared compiler snapshot memory. These routes apply to same-repository hybrid first attempts, automatic main runs, and admitted qualification dispatches. The GitHub override, hybrid retries, ordinary manual and frozen targets, and noncanonical contexts retain hosted routing. Compiler and lint rows still wait for their complete plan; eligible guards and dependency rows start after preflight. See [admission measurements and cost](/ci/routing-costs#ratchet-admission-and-node-tests).
|
||||
|
||||
RunsOn retains its hosted placement for standalone ratchets and check planning, including qualification dispatches.
|
||||
|
||||
|
|
|
|||
|
|
@ -116,19 +116,36 @@ export type CoreTsgoGraph = {
|
|||
files: readonly string[];
|
||||
};
|
||||
|
||||
type GraphQuery = {
|
||||
config: (config: string, cwd: string) => ReturnType<typeof readGraphConfig>;
|
||||
files: (config: string, name: string, cwd: string) => Promise<readonly string[]>;
|
||||
};
|
||||
const compilerQueries: GraphQuery = {
|
||||
config: readGraphConfig,
|
||||
async files(config, name, cwd) {
|
||||
return (await runTsgoQuery(config, "--listFilesOnly", `${name} file listing`, cwd))
|
||||
.split(/\r?\n/u)
|
||||
.filter(Boolean);
|
||||
},
|
||||
};
|
||||
|
||||
/** Validates all boundaries and returns this invocation's compiler-resolved inputs. */
|
||||
export async function checkCoreTsgoGraphBoundary(
|
||||
options: { cwd?: string } = {},
|
||||
): Promise<CoreTsgoGraph[]> {
|
||||
const cwd = realpathSync(options.cwd ?? repoRoot);
|
||||
return validateCoreGraphs(cwd, compilerQueries);
|
||||
}
|
||||
|
||||
async function validateCoreGraphs(cwd: string, query: GraphQuery): Promise<CoreTsgoGraph[]> {
|
||||
const normalize = (file: string) => normalizeFilePath(file, cwd);
|
||||
const testRootPattern = /\.test\.(?:ts|tsx)$/u;
|
||||
const canonicalRoots = ((await readGraphConfig(canonicalCoreTestConfig, cwd)).files ?? [])
|
||||
const canonicalRoots = ((await query.config(canonicalCoreTestConfig, cwd)).files ?? [])
|
||||
.map(normalize)
|
||||
.filter((file) => testRootPattern.test(file));
|
||||
const shardConfigs = [];
|
||||
for (const shard of TSGO_CORE_TEST_SHARDS) {
|
||||
shardConfigs.push({ ...shard, expanded: await readGraphConfig(shard.config, cwd) });
|
||||
shardConfigs.push({ ...shard, expanded: await query.config(shard.config, cwd) });
|
||||
}
|
||||
const shardRoots = shardConfigs.map((shard) => ({
|
||||
name: shard.name,
|
||||
|
|
@ -177,12 +194,7 @@ export async function checkCoreTsgoGraphBoundary(
|
|||
const violations: string[] = [];
|
||||
const graphs: CoreTsgoGraph[] = [];
|
||||
for (const graph of TSGO_CORE_GRAPHS) {
|
||||
const files = (
|
||||
await runTsgoQuery(graph.config, "--listFilesOnly", `${graph.name} file listing`, cwd)
|
||||
)
|
||||
.split(/\r?\n/u)
|
||||
.map(normalize)
|
||||
.filter(Boolean);
|
||||
const files = (await query.files(graph.config, graph.name, cwd)).map(normalize);
|
||||
graphs.push({
|
||||
...graph,
|
||||
files,
|
||||
|
|
@ -214,17 +226,68 @@ export async function inspectCiTsgoCheckGraphs(
|
|||
options: { cwd?: string; scope?: "all" | "noncore" } = {},
|
||||
): Promise<CoreTsgoGraph[]> {
|
||||
const cwd = realpathSync(options.cwd ?? repoRoot);
|
||||
const graphs = options.scope === "noncore" ? [] : await checkCoreTsgoGraphBoundary({ cwd });
|
||||
for (const graph of TSGO_CI_ADDITIONAL_GRAPHS) {
|
||||
const files = (
|
||||
await runTsgoQuery(graph.config, "--listFilesOnly", `${graph.name} file listing`, cwd)
|
||||
)
|
||||
.split(/\r?\n/u)
|
||||
.map((file) => normalizeFilePath(file, cwd))
|
||||
.filter(Boolean);
|
||||
graphs.push({ ...graph, files, roots: [] });
|
||||
const inspect = async (query: GraphQuery) => {
|
||||
const graphs = options.scope === "noncore" ? [] : await validateCoreGraphs(cwd, query);
|
||||
for (const graph of TSGO_CI_ADDITIONAL_GRAPHS) {
|
||||
const files = (await query.files(graph.config, graph.name, cwd)).map((file) =>
|
||||
normalizeFilePath(file, cwd),
|
||||
);
|
||||
graphs.push({ ...graph, files, roots: [] });
|
||||
}
|
||||
return graphs;
|
||||
};
|
||||
if (["1", "true"].includes(process.env.OPENCLAW_CI_TYPE_PLAN_SERIAL ?? "")) {
|
||||
return inspect(compilerQueries);
|
||||
}
|
||||
// One immutable compiler snapshot shares parsing and resolution across projects.
|
||||
// Membership still comes from each complete compiler program, including type-only edges.
|
||||
const { API } = await import("typescript/unstable/async");
|
||||
const api = new API({ cwd });
|
||||
try {
|
||||
const configs = [
|
||||
...(options.scope === "noncore" ? [] : TSGO_CORE_GRAPHS),
|
||||
...TSGO_CI_ADDITIONAL_GRAPHS,
|
||||
];
|
||||
const snapshot = await api.createSnapshot({
|
||||
openProjects: configs.map(({ config }) => path.resolve(cwd, config)),
|
||||
ensurePrograms: true,
|
||||
});
|
||||
const project = (config: string) => {
|
||||
const opened = snapshot.getConfiguredProject(path.resolve(cwd, config));
|
||||
if (!opened) {
|
||||
throw new Error(`Native TypeScript did not open ${config}`);
|
||||
}
|
||||
return opened;
|
||||
};
|
||||
for (const { config } of configs) {
|
||||
const diagnostics = await project(config).program.getConfigFileParsingDiagnostics();
|
||||
if (diagnostics.length) {
|
||||
throw new Error(`Invalid TypeScript config ${config}: ${JSON.stringify(diagnostics)}`);
|
||||
}
|
||||
}
|
||||
return await inspect({
|
||||
async config(config) {
|
||||
const absolute = path.resolve(cwd, config);
|
||||
const parsed =
|
||||
config === canonicalCoreTestConfig
|
||||
? await api.parseConfigFile(absolute)
|
||||
: project(config).parsedCommandLine;
|
||||
const syntax = await api.readConfigFile(absolute);
|
||||
if (parsed.errors.length || syntax.error) {
|
||||
throw new Error(`Invalid TypeScript config ${config}`);
|
||||
}
|
||||
return {
|
||||
compilerOptions: { tsBuildInfoFile: parsed.options.tsBuildInfoFile },
|
||||
files: parsed.fileNames.map((file) => path.relative(path.dirname(absolute), file)),
|
||||
};
|
||||
},
|
||||
async files(config) {
|
||||
return project(config).program.getSourceFileNames();
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await api.close();
|
||||
}
|
||||
return graphs;
|
||||
}
|
||||
|
||||
if (isDirectRunUrl(process.argv[1], import.meta.url)) {
|
||||
|
|
|
|||
|
|
@ -1332,15 +1332,6 @@ const checkTasks = [
|
|||
: narrowCheckScope.checkTasks.includes(row.task);
|
||||
});
|
||||
|
||||
// Move dependencies only when the preflight-only family is admitted.
|
||||
if (runCheckPlan && runNodeFull && !releaseFastLane) {
|
||||
const index = checkTasks.findIndex(({ task }) => task === "dependencies");
|
||||
if (index >= 0) {
|
||||
const { task, ...row } = checkTasks.splice(index, 1)[0];
|
||||
additionalChecks.push({ ...row, group: task });
|
||||
}
|
||||
}
|
||||
|
||||
// The selected guards row owns the same coercion scan; fast-only plans retain its row.
|
||||
if (
|
||||
!frozenTarget &&
|
||||
|
|
@ -1354,6 +1345,17 @@ if (
|
|||
}
|
||||
}
|
||||
|
||||
// These rows need no compiler plan; retain their existing full-check placement.
|
||||
if (runCheckPlan && runNodeFull && !releaseFastLane) {
|
||||
for (const task of ["guards", "dependencies"]) {
|
||||
const index = checkTasks.findIndex((row) => row.task === task);
|
||||
if (index >= 0) {
|
||||
const { task: group, ...row } = checkTasks.splice(index, 1)[0];
|
||||
additionalChecks.push({ ...row, group });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const manifest = {
|
||||
release_scope: releaseScope,
|
||||
release_fast_lane: releaseFastLane,
|
||||
|
|
|
|||
|
|
@ -496,6 +496,7 @@ function runCheckShardFixture(options: {
|
|||
frozenTarget: boolean;
|
||||
scripts: string[];
|
||||
task?: "guards" | "npm-lock" | "prod-types" | "test-types";
|
||||
earlyGuards?: boolean;
|
||||
checkoutBase?: string;
|
||||
types?: {
|
||||
compose?: boolean;
|
||||
|
|
@ -603,9 +604,13 @@ appendFileSync(process.env.TYPE_CALLS, [process.env.TYPE_ROW, process.env.OPENCL
|
|||
'if [ "$*" = "${FAIL_PACKAGE_SCRIPT:-}" ]; then exit 17; fi',
|
||||
]);
|
||||
const workflow = readCiWorkflow();
|
||||
const checkShardStep = workflow.jobs["check-shard"].steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run check shard",
|
||||
);
|
||||
const checkShardStep = options.earlyGuards
|
||||
? workflow.jobs["check-additional-shard"].steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run independent checks",
|
||||
)
|
||||
: workflow.jobs["check-shard"].steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run check shard",
|
||||
);
|
||||
const context: Parameters<typeof evaluateWorkflowExpression>[1] = {
|
||||
eventName:
|
||||
options.types?.eventName ?? (options.frozenTarget ? "workflow_dispatch" : "pull_request"),
|
||||
|
|
@ -622,7 +627,8 @@ appendFileSync(process.env.TYPE_CALLS, [process.env.TYPE_ROW, process.env.OPENCL
|
|||
compatibility_target: String(options.frozenTarget),
|
||||
run_format_check: "false",
|
||||
changed_core_test_paths_json: options.types?.changedPathsJson ?? "",
|
||||
narrow_check_paths_json: options.types?.narrowPathsJson ?? "",
|
||||
narrow_check_paths_json:
|
||||
options.types?.narrowPathsJson ?? (options.earlyGuards ? '["src/shared/runtime.ts"]' : ""),
|
||||
...(options.types?.compose
|
||||
? {
|
||||
core_type_matrix: expectDefined(
|
||||
|
|
@ -651,7 +657,11 @@ appendFileSync(process.env.TYPE_CALLS, [process.env.TYPE_ROW, process.env.OPENCL
|
|||
rows.push({
|
||||
name: "central",
|
||||
step: checkShardStep,
|
||||
matrix: { task: options.task ?? "guards", ...options.types?.matrix },
|
||||
matrix: {
|
||||
task: options.task ?? "guards",
|
||||
...(options.earlyGuards ? { group: "guards" } : {}),
|
||||
...options.types?.matrix,
|
||||
},
|
||||
});
|
||||
if (options.types?.boundary) {
|
||||
rows.push({
|
||||
|
|
@ -688,7 +698,7 @@ appendFileSync(process.env.TYPE_CALLS, [process.env.TYPE_ROW, process.env.OPENCL
|
|||
PATH: `${fakeBin}:${process.env.PATH ?? ""}`,
|
||||
PNPM_CALLS: callsPath,
|
||||
TASK: options.task ?? "guards",
|
||||
...(typeCheck
|
||||
...(typeCheck || options.earlyGuards
|
||||
? {
|
||||
OPENCLAW_LOCAL_CHECK: undefined,
|
||||
TYPE_ROW: row.name,
|
||||
|
|
@ -773,7 +783,7 @@ function runDependencyCheckFixture(options: {
|
|||
'printf "%s\\n" "$*" >> "$PNPM_CALLS"',
|
||||
]);
|
||||
const checkShardRun = readCiWorkflow().jobs["check-additional-shard"].steps.find(
|
||||
(step: WorkflowStep) => step.name === "Run dependency checks",
|
||||
(step: WorkflowStep) => step.name === "Run independent checks",
|
||||
).run;
|
||||
const run = spawnSync("bash", ["-c", checkShardRun], {
|
||||
cwd: root,
|
||||
|
|
@ -1403,29 +1413,36 @@ describe("ci workflow guards", () => {
|
|||
workflow.jobs["check-shard"].strategy.matrix,
|
||||
context,
|
||||
).include.map((row: { task: string }) => row.task),
|
||||
).toEqual(nodeDataOnly ? tasks : tasks.filter((task) => task !== "dependencies"));
|
||||
).toEqual(
|
||||
nodeDataOnly ? tasks : tasks.filter((task) => !["guards", "dependencies"].includes(task)),
|
||||
);
|
||||
const additional = workflow.jobs["check-additional-shard"];
|
||||
const dependencyRows = evaluateWorkflowExpression(
|
||||
const independentRows = evaluateWorkflowExpression(
|
||||
additional.strategy.matrix,
|
||||
context,
|
||||
).include.filter((row: { group: string }) => row.group === "dependencies");
|
||||
expect(dependencyRows).toEqual(
|
||||
tasks.includes("dependencies") && !nodeDataOnly
|
||||
? [
|
||||
{
|
||||
check_name: "check-dependencies",
|
||||
group: "dependencies",
|
||||
runner: "blacksmith-16vcpu-ubuntu-2404",
|
||||
},
|
||||
]
|
||||
: [],
|
||||
).include.filter((row: { group: string }) =>
|
||||
["guards", "dependencies"].includes(row.group),
|
||||
);
|
||||
expect(independentRows).toEqual(
|
||||
nodeDataOnly
|
||||
? []
|
||||
: tasks
|
||||
.filter((task) => ["guards", "dependencies"].includes(task))
|
||||
.map((group) => ({
|
||||
check_name: `check-${group}`,
|
||||
group,
|
||||
runner:
|
||||
group === "guards"
|
||||
? "blacksmith-4vcpu-ubuntu-2404"
|
||||
: "blacksmith-16vcpu-ubuntu-2404",
|
||||
})),
|
||||
);
|
||||
if (nodeDataOnly) {
|
||||
expect(manifest.outputs.run_check).toBe("true");
|
||||
expect(manifest.outputs.run_check_additional).toBe("false");
|
||||
}
|
||||
if (dependencyRows.length) {
|
||||
// The dependency gate must start while the installed compiler planner is pending.
|
||||
for (const matrix of independentRows) {
|
||||
// These gates must start while the installed compiler planner is pending.
|
||||
expect(additional.needs).toEqual(["preflight"]);
|
||||
expect(
|
||||
evaluateWorkflowExpression(additional.if, {
|
||||
|
|
@ -1436,9 +1453,29 @@ describe("ci workflow guards", () => {
|
|||
expect(
|
||||
evaluateWorkflowExpression(additional["runs-on"], {
|
||||
...context,
|
||||
matrix: dependencyRows[0],
|
||||
matrix,
|
||||
}),
|
||||
).toBe("blacksmith-16vcpu-ubuntu-2404");
|
||||
).toBe(matrix.runner);
|
||||
const independentStep = additional.steps.find(
|
||||
(candidate: WorkflowStep) => candidate.name === "Run independent checks",
|
||||
);
|
||||
const rowContext = { ...context, matrix };
|
||||
expect(evaluateWorkflowExpression(`\${{ ${independentStep.if} }}`, rowContext)).toBe(
|
||||
true,
|
||||
);
|
||||
expect(evaluateWorkflowExpression(independentStep.env.TASK, rowContext)).toBe(
|
||||
matrix.group,
|
||||
);
|
||||
expect(
|
||||
evaluateWorkflowExpression(independentStep.env.NARROW_CHECK_PATHS_JSON, rowContext),
|
||||
).toBe(JSON.stringify(paths));
|
||||
const baseSha = "a".repeat(40);
|
||||
expect(
|
||||
evaluateWorkflowExpression(additional.env.CHECKOUT_BASE_SHA, {
|
||||
...rowContext,
|
||||
preflightOutputs: { ...manifest.outputs, diff_base_revision: baseSha },
|
||||
}),
|
||||
).toBe(matrix.group === "guards" ? baseSha : "");
|
||||
}
|
||||
expect(
|
||||
JSON.parse(
|
||||
|
|
@ -5561,6 +5598,50 @@ describe("ci workflow guards", () => {
|
|||
const workflow = readCiWorkflow();
|
||||
const buildArtifactsTestbox = readBuildArtifactsTestboxWorkflow();
|
||||
const source = readFileSync(".github/workflows/ci.yml", "utf8");
|
||||
const plannerRunner = workflow.jobs["check-plan"]["runs-on"];
|
||||
const plannerContext = {
|
||||
eventName: "pull_request" as const,
|
||||
repository: "openclaw/openclaw",
|
||||
runAttempt: 1,
|
||||
runnerBackend: "hybrid" as const,
|
||||
runnerProfile: "hybrid" as const,
|
||||
};
|
||||
for (const eventName of ["pull_request", "push", "schedule"] as const) {
|
||||
expect(evaluateWorkflowExpression(plannerRunner, { ...plannerContext, eventName })).toBe(
|
||||
"blacksmith-16vcpu-ubuntu-2404",
|
||||
);
|
||||
}
|
||||
const qualification = {
|
||||
...plannerContext,
|
||||
eventName: "workflow_dispatch" as const,
|
||||
preflightOutputs: { ci_qualification: "true", qualification_runner_backend: "hybrid" },
|
||||
};
|
||||
expect(evaluateWorkflowExpression(plannerRunner, qualification)).toBe(
|
||||
"blacksmith-16vcpu-ubuntu-2404",
|
||||
);
|
||||
expect(
|
||||
evaluateWorkflowExpression(plannerRunner, {
|
||||
...qualification,
|
||||
dispatchId: "full-release-validation-test",
|
||||
releaseRunnerGroup: "test-group",
|
||||
}),
|
||||
).toEqual({ group: "test-group", labels: "blacksmith-16vcpu-ubuntu-2404" });
|
||||
for (const override of [
|
||||
{ runnerBackend: "github" },
|
||||
{ runnerBackend: "blacksmith" },
|
||||
{ runnerBackend: "runson" },
|
||||
{ preflightOutputs: { node_runner_backend: "runson" } },
|
||||
{ headRepository: "contributor/openclaw" },
|
||||
{ repository: "contributor/openclaw" },
|
||||
{ runAttempt: 2 },
|
||||
{ frozenTarget: true },
|
||||
{ eventName: "workflow_dispatch" },
|
||||
] as const) {
|
||||
expect(
|
||||
evaluateWorkflowExpression(plannerRunner, { ...plannerContext, ...override }),
|
||||
JSON.stringify(override),
|
||||
).toBe("ubuntu-24.04");
|
||||
}
|
||||
|
||||
expect(readFileSync("scripts/ci-build-manifest.mjs", "utf8")).toContain(
|
||||
"createNodeTestShardBundles",
|
||||
|
|
@ -6559,6 +6640,7 @@ describe("ci workflow guards", () => {
|
|||
];
|
||||
const current = runCheckShardFixture({
|
||||
frozenTarget: false,
|
||||
earlyGuards: true,
|
||||
scripts: [...requiredScripts, "check:temp-path-guardrails"],
|
||||
});
|
||||
expect(current.status, current.output).toBe(0);
|
||||
|
|
@ -6566,6 +6648,19 @@ describe("ci workflow guards", () => {
|
|||
expect(current.calls.indexOf("check:temp-path-guardrails")).toBeLessThan(
|
||||
current.calls.indexOf("dup:check"),
|
||||
);
|
||||
expect(current.calls).toEqual([
|
||||
"tool-display:check",
|
||||
"check:host-env-policy:swift",
|
||||
"check:browser-inspect-script:swift",
|
||||
"check:temp-path-guardrails",
|
||||
"dup:check",
|
||||
"check:coercion-helpers",
|
||||
"deps:patches:check",
|
||||
"lint:webhook:no-low-level-body-read",
|
||||
"lint:auth:no-pairing-store-group",
|
||||
"lint:auth:pairing-account-scope",
|
||||
"check:import-cycles",
|
||||
]);
|
||||
|
||||
const frozenMissing = runCheckShardFixture({
|
||||
frozenTarget: true,
|
||||
|
|
@ -6580,6 +6675,7 @@ describe("ci workflow guards", () => {
|
|||
|
||||
const currentMissing = runCheckShardFixture({
|
||||
frozenTarget: false,
|
||||
earlyGuards: true,
|
||||
scripts: requiredScripts,
|
||||
});
|
||||
expect(currentMissing.status).toBe(1);
|
||||
|
|
|
|||
|
|
@ -682,14 +682,14 @@ import { checkCoreTsgoGraphBoundary } from "./check-tsgo-core-boundary.mts";
|
|||
if (process.argv[2] === "boundary") {
|
||||
await checkCoreTsgoGraphBoundary();
|
||||
} else {
|
||||
const plan = await createChangedCiTypeCheckPlan([${JSON.stringify(extension)}], {
|
||||
const plan = await createChangedCiTypeCheckPlan([process.argv[2] === "core-plan" ? ${JSON.stringify(helper)} : ${JSON.stringify(extension)}], {
|
||||
cwd: process.cwd(), coreBoundaryOwner: "additional-checks",
|
||||
});
|
||||
console.log(JSON.stringify({ mode: plan.mode, names: plan.graphs.map(({ name }) => name) }));
|
||||
}
|
||||
`,
|
||||
);
|
||||
const inspectExtension = async (mode: "plan" | "boundary") => {
|
||||
const inspectExtension = async (mode: "plan" | "core-plan" | "boundary", serial = "") => {
|
||||
write("compiler-events.jsonl", "");
|
||||
return await lifetime.track(
|
||||
runNodeScript(
|
||||
|
|
@ -699,25 +699,35 @@ if (process.argv[2] === "boundary") {
|
|||
plannerDriver,
|
||||
mode,
|
||||
],
|
||||
env,
|
||||
{ ...env, OPENCLAW_CI_TYPE_PLAN_SERIAL: serial },
|
||||
undefined,
|
||||
{ cwd: root, signal, requireProcessTreeExit: true },
|
||||
),
|
||||
);
|
||||
};
|
||||
const extensionPlan = await inspectExtension("plan");
|
||||
expect(extensionPlan.status, extensionPlan.stderr).toBe(0);
|
||||
expect(JSON.parse(extensionPlan.stdout.trim())).toEqual({
|
||||
mode: "changed",
|
||||
names: ["extensions", "extensions-test", "scripts", "test-root"],
|
||||
});
|
||||
const discovery = fs
|
||||
.readFileSync(path.join(root, "compiler-events.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line) as string[]);
|
||||
expect(discovery).toHaveLength(4);
|
||||
expect(discovery.every((args) => args.includes("--listFilesOnly"))).toBe(true);
|
||||
for (const serial of ["", "1"]) {
|
||||
const extensionPlan = await inspectExtension("plan", serial);
|
||||
expect(extensionPlan.status, extensionPlan.stderr).toBe(0);
|
||||
expect(JSON.parse(extensionPlan.stdout.trim())).toEqual({
|
||||
mode: "changed",
|
||||
names: ["extensions", "extensions-test", "scripts", "test-root"],
|
||||
});
|
||||
const discovery = fs
|
||||
.readFileSync(path.join(root, "compiler-events.jsonl"), "utf8")
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line) as string[]);
|
||||
expect(discovery).toHaveLength(serial ? 4 : 1);
|
||||
expect(discovery.every((args) => args.includes(serial ? "--listFilesOnly" : "--api"))).toBe(
|
||||
true,
|
||||
);
|
||||
const corePlan = await inspectExtension("core-plan", serial);
|
||||
expect(corePlan.status, corePlan.stderr).toBe(0);
|
||||
expect(JSON.parse(corePlan.stdout.trim())).toEqual({
|
||||
mode: "changed",
|
||||
names: ["core-test-agents-other"],
|
||||
});
|
||||
}
|
||||
// Its parallel owner must still reject a type-only edge into an extension.
|
||||
write(
|
||||
consumer,
|
||||
|
|
@ -729,6 +739,10 @@ if (process.argv[2] === "boundary") {
|
|||
"Core tsgo graphs include bundled extension files",
|
||||
);
|
||||
expect(extensionBoundary.stderr).toContain(extension);
|
||||
const invalidCorePlan = await inspectExtension("core-plan");
|
||||
expect(invalidCorePlan.status).not.toBe(0);
|
||||
expect(invalidCorePlan.stderr).toContain("Core tsgo graphs include bundled extension files");
|
||||
expect(invalidCorePlan.stderr).toContain(extension);
|
||||
write(
|
||||
consumer,
|
||||
"import type {Value} from '../nested/leaf.test.js';\nconst value: Value = 1;\n",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue