diff --git a/.agents/skills/crabbox/SKILL.md b/.agents/skills/crabbox/SKILL.md index 9fcdedd96854..aa5537d760d8 100644 --- a/.agents/skills/crabbox/SKILL.md +++ b/.agents/skills/crabbox/SKILL.md @@ -61,6 +61,14 @@ Source trust determines which providers are allowed. It does not select one. - Never untrusted code on credential-hydrated Testbox. - Never run untrusted repo wrapper/config locally. - No speculative warmup. Acquire when first heavy command ready. Reuse id. Stop. +- Use the consumer's smallest proven remote profile. Large/high-memory profiles + are explicit exceptions for a named command with measured memory need or a + controlled total-cost benefit. Record the reason before allocation; a generic + failure, queue delay, or timeout is not a reason to upsize. Do not raise worker + counts to compensate for a smaller machine. +- For delegated Testbox sizing, select a documented workflow through + `--blacksmith-workflow`; direct-provider `--class`/`--type` flags do not choose + the Testbox runner. Change profiles with a fresh lease. Test size, expected duration, and hydration failure do not authorize a provider override. Omit `--provider` for normal work. Add it only when the user requests diff --git a/.agents/skills/openclaw-ci-limits/SKILL.md b/.agents/skills/openclaw-ci-limits/SKILL.md index 7e4a0952cf99..676104db8f9e 100644 --- a/.agents/skills/openclaw-ci-limits/SKILL.md +++ b/.agents/skills/openclaw-ci-limits/SKILL.md @@ -29,6 +29,27 @@ availability, Blacksmith control-plane health, and downstream queue drains. scans should stay on GitHub-hosted runners unless measured evidence says Blacksmith is required. +## Runner Cost Policy + +- Use the smallest runner that completes the required workload reliably. Keep + short control jobs hosted and ordinary trusted development proof local. +- Treat the 32-class as an exception, not a default or generic retry. Record the + command and measured peak memory, a smaller-runner OOM, or a controlled + comparison showing lower total billed cost. Low CPU use alone does not prove + a memory-heavy job can move down. Preserve resource-based worker limits. +- Routine OpenClaw Testboxes use the 16-class with a 60-minute total-job + default, including hydration. Keep the 15-minute idle ceiling. Shorter + deadlines are welcome for known short commands; do not request four hours + automatically or upsize only to obtain more time. Select + `.github/workflows/ci-check-high-memory-testbox.yml` explicitly only for a + named memory-heavy command; see `docs/reference/test/remote-proof.md`. + That workflow has at most four concurrent leases inside the shared 32-slot + Testbox pool. All Testbox profiles cap idle time at 15 minutes. +- Do not promote an entire workflow family because one command needs more RAM. + Keep proven high-memory CI rows scoped to their owning planner and evidence; + remeasure before changing their allocation. A 32-class label is not proof of + 32 available CPUs. Compare observed resources and total billed job cost. + ## Rejected Experiments - **Boundary asynchronous input preparation (2026-09-26):** Adding the existing diff --git a/.crabbox.yaml b/.crabbox.yaml index 458b96a9f16a..7645d3729592 100644 --- a/.crabbox.yaml +++ b/.crabbox.yaml @@ -28,6 +28,7 @@ actions: ephemeral: true blacksmith: org: openclaw + # Routine remote proof uses 16-class; high-memory proof opts in by workflow. workflow: .github/workflows/ci-check-testbox.yml job: check ref: main @@ -126,7 +127,7 @@ jobs: testbox-changed: provider: blacksmith-testbox target: linux - idleTimeout: 90m + idleTimeout: 15m hydrate: actions: false actions: diff --git a/.github/workflows/ci-build-artifacts-testbox.yml b/.github/workflows/ci-build-artifacts-testbox.yml index 9650ecac3107..ee2faebad6b9 100644 --- a/.github/workflows/ci-build-artifacts-testbox.yml +++ b/.github/workflows/ci-build-artifacts-testbox.yml @@ -10,6 +10,7 @@ on: pull_request: types: [opened, reopened, synchronize, ready_for_review] paths: + - "scripts/ci-testbox-budget.mjs" - ".github/workflows/ci-build-artifacts-testbox.yml" - ".github/actions/prepare-testbox-shell/**" - ".github/actions/setup-node-env/**" @@ -62,16 +63,47 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: + admission: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 3 + permissions: + contents: read + actions: read + outputs: + group: ${{ steps.budget.outputs.group }} + runner: ${{ steps.budget.outputs.runner }} + minutes: ${{ steps.budget.outputs.minutes }} + expires_at: ${{ steps.budget.outputs.expires_at }} + steps: + - name: Checkout admission policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ci-testbox-budget.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Admit Testbox within the shared budget + id: budget + env: + GH_TOKEN: ${{ github.token }} + TESTBOX_PROFILE: build + TESTBOX_ID: ${{ inputs.testbox_id }} + run: node scripts/ci-testbox-budget.mjs admit + build-artifacts: - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + needs: admission + if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }} + concurrency: + group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: false permissions: contents: read name: "build-artifacts" # Pull requests only validate the artifact build; real Testbox leases # arrive via dispatch. Hosted PR runs keep this landing gate satisfiable # during Blacksmith outages (mirrors ci-check-testbox.yml). - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }} - timeout-minutes: 35 + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '35') }} steps: # Testbox lifecycle actions require Blacksmith VM metadata; PRs validate the build only. - name: Begin Testbox @@ -80,12 +112,30 @@ jobs: with: testbox_id: ${{ inputs.testbox_id }} + - name: Reject expired Testbox admission + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: | + if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] || + (( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then + echo "Testbox waited more than 10 minutes; request a fresh lease." >&2 + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }} persist-credentials: false + - name: Bound Testbox idle lifetime + if: github.event_name == 'workflow_dispatch' + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: node scripts/ci-testbox-budget.mjs configure + - name: Setup Node environment id: setup-node-env uses: ./.github/actions/setup-node-env @@ -257,3 +307,23 @@ jobs: if: github.event_name == 'workflow_dispatch' && always() env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + + - name: Close Testbox SSH sessions + if: github.event_name == 'workflow_dispatch' && always() + shell: bash + run: | + set -euo pipefail + + # Testbox state stores Blacksmith's external forwarded port. Resolve + # sshd's VM-local listener because that is the sport visible to ss. + runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')" + if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] || + (( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then + echo "No valid local SSH listener port found; skipping session cleanup" + exit 0 + fi + + # run-testbox has no post hook. Close only Testbox client sockets so + # Blacksmith's runner teardown does not wait for its 290-second grace. + timeout --signal=KILL 5s sudo ss -K state established \ + "( sport = :${runner_ssh_local_port} )" || true diff --git a/.github/workflows/ci-check-arm-testbox.yml b/.github/workflows/ci-check-arm-testbox.yml index 5d92b783dfe8..ae0fa5913e49 100644 --- a/.github/workflows/ci-check-arm-testbox.yml +++ b/.github/workflows/ci-check-arm-testbox.yml @@ -9,6 +9,7 @@ on: pull_request: types: [opened, reopened, synchronize, ready_for_review] paths: + - "scripts/ci-testbox-budget.mjs" - ".github/workflows/ci-check-arm-testbox.yml" - ".github/actions/prepare-testbox-shell/**" - ".github/actions/setup-node-env/**" @@ -56,18 +57,62 @@ env: PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false" jobs: + admission: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 3 + permissions: + contents: read + actions: read + outputs: + group: ${{ steps.budget.outputs.group }} + runner: ${{ steps.budget.outputs.runner }} + minutes: ${{ steps.budget.outputs.minutes }} + expires_at: ${{ steps.budget.outputs.expires_at }} + steps: + - name: Checkout admission policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ci-testbox-budget.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Admit Testbox within the shared budget + id: budget + env: + GH_TOKEN: ${{ github.token }} + TESTBOX_PROFILE: arm + TESTBOX_ID: ${{ inputs.testbox_id }} + run: node scripts/ci-testbox-budget.mjs admit + check-arm: - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + needs: admission + if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }} + concurrency: + group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: false permissions: contents: read name: "check-arm" - runs-on: blacksmith-16vcpu-ubuntu-2404-arm - timeout-minutes: 120 + runs-on: ${{ github.event_name == 'pull_request' && 'blacksmith-16vcpu-ubuntu-2404-arm' || needs.admission.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '120') }} steps: - name: Begin Testbox + if: github.event_name == 'workflow_dispatch' uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2 with: testbox_id: ${{ inputs.testbox_id }} + - name: Reject expired Testbox admission + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: | + if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] || + (( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then + echo "Testbox waited more than 10 minutes; request a fresh lease." >&2 + exit 1 + fi + - name: Verify ARM runner shell: bash run: | @@ -88,6 +133,12 @@ jobs: with: fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }} persist-credentials: false + - name: Bound Testbox idle lifetime + if: github.event_name == 'workflow_dispatch' + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: node scripts/ci-testbox-budget.mjs configure + - name: Setup Node environment uses: ./.github/actions/setup-node-env with: @@ -155,6 +206,26 @@ jobs: # Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15 # Return to useblacksmith/run-testbox after the fix lands upstream. uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4 - if: always() + if: github.event_name == 'workflow_dispatch' && always() env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + + - name: Close Testbox SSH sessions + if: github.event_name == 'workflow_dispatch' && always() + shell: bash + run: | + set -euo pipefail + + # Testbox state stores Blacksmith's external forwarded port. Resolve + # sshd's VM-local listener because that is the sport visible to ss. + runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')" + if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] || + (( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then + echo "No valid local SSH listener port found; skipping session cleanup" + exit 0 + fi + + # run-testbox has no post hook. Close only Testbox client sockets so + # Blacksmith's runner teardown does not wait for its 290-second grace. + timeout --signal=KILL 5s sudo ss -K state established \ + "( sport = :${runner_ssh_local_port} )" || true diff --git a/.github/workflows/ci-check-high-memory-testbox.yml b/.github/workflows/ci-check-high-memory-testbox.yml new file mode 100644 index 000000000000..831b1aa4c999 --- /dev/null +++ b/.github/workflows/ci-check-high-memory-testbox.yml @@ -0,0 +1,183 @@ +name: Blacksmith High-Memory Testbox +on: + workflow_dispatch: + inputs: + testbox_id: + type: string + description: "Testbox session ID" + required: true + timeout_minutes: + type: number + description: "Maximum GitHub job runtime for long Testbox commands" + default: 240 + +permissions: + contents: read + +# Explicit memory-heavy exception; ordinary proof uses ci-check-testbox.yml. +concurrency: + group: ${{ github.workflow }}-manual-v1-${{ github.run_id }} + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false" + +jobs: + admission: + runs-on: ubuntu-24.04 + timeout-minutes: 3 + permissions: + contents: read + actions: read + outputs: + group: ${{ steps.budget.outputs.group }} + runner: ${{ steps.budget.outputs.runner }} + minutes: ${{ steps.budget.outputs.minutes }} + expires_at: ${{ steps.budget.outputs.expires_at }} + steps: + - name: Checkout admission policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ci-testbox-budget.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Admit Testbox within the shared budget + id: budget + env: + GH_TOKEN: ${{ github.token }} + TESTBOX_PROFILE: check-memory + TESTBOX_ID: ${{ inputs.testbox_id }} + TESTBOX_MINUTES: ${{ inputs.timeout_minutes }} + run: node scripts/ci-testbox-budget.mjs admit + + check: + needs: admission + if: ${{ !cancelled() && needs.admission.result == 'success' }} + concurrency: + group: ${{ needs.admission.outputs.group }} + cancel-in-progress: false + permissions: + contents: read + name: "check" + runs-on: ${{ needs.admission.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '240') }} + steps: + - name: Begin Testbox + uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 + with: + testbox_id: ${{ inputs.testbox_id }} + - name: Reject expired Testbox admission + shell: bash + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: | + if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] || + (( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then + echo "Testbox waited more than 10 minutes; request a fresh lease." >&2 + exit 1 + fi + + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 1 + persist-credentials: false + - name: Bound Testbox idle lifetime + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: node scripts/ci-testbox-budget.mjs configure + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + semantic-checks: "true" + cache-mode: restore + install-bun: "false" + # Testbox hydration uses the ordinary pnpm store cache. Canonical CI + # owns the exact dependency archive and never delegates here. + - name: Prepare Testbox shell + uses: ./.github/actions/prepare-testbox-shell + with: + base-ref: HEAD + go-version: "1.27.1" + + - name: Hydrate Testbox provider env helper + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }} + ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} + BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }} + OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }} + OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }} + OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }} + OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }} + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + FAL_KEY: ${{ secrets.FAL_KEY }} + RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }} + DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} + Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }} + BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }} + BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }} + OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }} + OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }} + OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} + OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + run: bash scripts/ci-hydrate-testbox-env.sh + + - name: Run Testbox + # Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15 + # Return to useblacksmith/run-testbox after the fix lands upstream. + uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4 + if: always() + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + + - name: Close Testbox SSH sessions + if: always() + shell: bash + run: | + set -euo pipefail + + # Testbox state stores Blacksmith's external forwarded port. Resolve + # sshd's VM-local listener because that is the sport visible to ss. + runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')" + if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] || + (( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then + echo "No valid local SSH listener port found; skipping session cleanup" + exit 0 + fi + + # run-testbox has no post hook. Close only Testbox client sockets so + # Blacksmith's runner teardown does not wait for its 290-second grace. + timeout --signal=KILL 5s sudo ss -K state established \ + "( sport = :${runner_ssh_local_port} )" || true diff --git a/.github/workflows/ci-check-testbox.yml b/.github/workflows/ci-check-testbox.yml index ab410d40664b..20817b18f300 100644 --- a/.github/workflows/ci-check-testbox.yml +++ b/.github/workflows/ci-check-testbox.yml @@ -8,11 +8,12 @@ on: required: true timeout_minutes: type: number - description: "Maximum GitHub job runtime for long Testbox commands" - default: 240 + description: "GitHub job runtime; explicit long-proof requests may use up to 240 minutes" + default: 60 pull_request: types: [opened, reopened, synchronize, ready_for_review] paths: + - "scripts/ci-testbox-budget.mjs" - ".github/workflows/ci-check-testbox.yml" - ".github/actions/prepare-testbox-shell/**" - ".github/actions/setup-node-env/**" @@ -59,14 +60,46 @@ env: PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false" jobs: + admission: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 3 + permissions: + contents: read + actions: read + outputs: + group: ${{ steps.budget.outputs.group }} + runner: ${{ steps.budget.outputs.runner }} + minutes: ${{ steps.budget.outputs.minutes }} + expires_at: ${{ steps.budget.outputs.expires_at }} + steps: + - name: Checkout admission policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ci-testbox-budget.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Admit Testbox within the shared budget + id: budget + env: + GH_TOKEN: ${{ github.token }} + TESTBOX_PROFILE: check + TESTBOX_ID: ${{ inputs.testbox_id }} + TESTBOX_MINUTES: ${{ inputs.timeout_minutes }} + run: node scripts/ci-testbox-budget.mjs admit + check: - if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }} + needs: admission + if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }} + concurrency: + group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: false permissions: contents: read name: "check" # Pull requests only validate the hydration steps; real Testbox leases arrive via dispatch. - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} - timeout-minutes: ${{ fromJSON(inputs.timeout_minutes || '240') }} + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '60') }} steps: # Testbox lifecycle actions require Blacksmith VM metadata; PRs validate our setup steps only. - name: Begin Testbox @@ -74,6 +107,18 @@ jobs: uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 with: testbox_id: ${{ inputs.testbox_id }} + - name: Reject expired Testbox admission + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: | + if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] || + (( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then + echo "Testbox waited more than 10 minutes; request a fresh lease." >&2 + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -81,6 +126,12 @@ jobs: # reconstructs its exact base and final tree. PR validation keeps both commits. fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '1' }} persist-credentials: false + - name: Bound Testbox idle lifetime + if: github.event_name == 'workflow_dispatch' + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: node scripts/ci-testbox-budget.mjs configure + - name: Setup Node environment uses: ./.github/actions/setup-node-env with: diff --git a/.github/workflows/windows-blacksmith-testbox.yml b/.github/workflows/windows-blacksmith-testbox.yml index 1a742d92ca87..a95db2ac9442 100644 --- a/.github/workflows/windows-blacksmith-testbox.yml +++ b/.github/workflows/windows-blacksmith-testbox.yml @@ -8,7 +8,10 @@ on: description: "Testbox session ID" required: true runner_label: - type: string + type: choice + options: + - blacksmith-8vcpu-windows-2025 + - blacksmith-16vcpu-windows-2025 description: "Windows runner label" required: false default: "blacksmith-16vcpu-windows-2025" @@ -20,10 +23,43 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" jobs: + admission: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-24.04 + timeout-minutes: 3 + permissions: + contents: read + actions: read + outputs: + group: ${{ steps.budget.outputs.group }} + runner: ${{ steps.budget.outputs.runner }} + minutes: ${{ steps.budget.outputs.minutes }} + expires_at: ${{ steps.budget.outputs.expires_at }} + steps: + - name: Checkout admission policy + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: scripts/ci-testbox-budget.mjs + sparse-checkout-cone-mode: false + persist-credentials: false + - name: Admit Testbox within the shared budget + id: budget + env: + GH_TOKEN: ${{ github.token }} + TESTBOX_PROFILE: windows + TESTBOX_ID: ${{ inputs.testbox_id }} + TESTBOX_RUNNER: ${{ inputs.runner_label }} + run: node scripts/ci-testbox-budget.mjs admit + windows: + needs: admission + if: ${{ !cancelled() && needs.admission.result == 'success' }} + concurrency: + group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }} + cancel-in-progress: false name: windows - runs-on: ${{ inputs.runner_label }} - timeout-minutes: 75 + runs-on: ${{ needs.admission.outputs.runner }} + timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes) }} defaults: run: shell: pwsh @@ -138,6 +174,18 @@ jobs: printf 'public_key_path=%s\n' "$public_key_path" >> "$GITHUB_OUTPUT" printf 'bash_os=%s\nbash_user=%s\n' "$(uname -s)" "$(id -un)" + - name: Reject expired Testbox admission + if: github.event_name == 'workflow_dispatch' + shell: bash + env: + TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }} + run: | + if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] || + (( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then + echo "Testbox waited more than 10 minutes; request a fresh lease." >&2 + exit 1 + fi + - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -217,6 +265,11 @@ jobs: auth_token="$(cat "$state/auth_token")" idle_timeout="$(cat "$state/idle_timeout" 2>/dev/null || true)" idle_timeout="${idle_timeout:-10}" + if [[ ! "$idle_timeout" =~ ^[0-9]+$ ]] || (( idle_timeout < 1 )); then + echo "Invalid Testbox idle timeout" >&2 + exit 1 + fi + if (( idle_timeout > 15 )); then idle_timeout=15; fi api_url="$(cat "$state/api_url")" runner_host="$(cat "$state/runner_host")" runner_ssh_port="$(cat "$state/runner_ssh_port")" diff --git a/docs/ci.md b/docs/ci.md index 6d99301937cc..e6ac607c072f 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -62,7 +62,7 @@ file, so large PRs do not lose test-planning inputs to Actions output or environ size limits. Frozen targets that predate this transport retain their bounded JSON output contract. Missing or invalid inputs still reject current PR Node planning. -The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 32-class for dispatched proof and defaults to a four-hour outer job budget. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged. +The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 16-class for routine dispatched proof, with a 60-minute total-job deadline including hydration. The explicit high-memory 32-class workflow retains 240 minutes for memory-heavy full-suite gates. The outer GitHub deadline can terminate active SSH commands; the separate 15-minute idle limit does not extend it. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged. Full GitHub and hybrid type checks run the five core stripes independently, retaining two compiler children per job. The last four rows then each run one root-test partition serially, leaving extension tests and scripts in the central row. Narrow plans reuse four already-selected rows when available; smaller selections retain central root checking. This adds no jobs or compiler overlap. Current hybrid full runs use three hosted extension-lint jobs; targeted layouts retain six stripe identities. Trusted hybrid first attempts place both packed core-lint rows on the Blacksmith 16-class and the final gate on the 4-class to avoid serial hosted assignment delays. Frozen targets keep their earlier layout; see [static checks](/ci/runners#runner-backend-modes). diff --git a/docs/ci/local-proof.md b/docs/ci/local-proof.md index 64e4ffe93c22..aa98da74f5c1 100644 --- a/docs/ci/local-proof.md +++ b/docs/ci/local-proof.md @@ -369,15 +369,26 @@ Provider readiness and broker authentication still determine which configured backend can run the proof. The check workflow hydrates its pinned dispatch commit with a depth-1 checkout; the changed gate later reconstructs the exact merge base and synced final tree. -Dispatched check leases request `blacksmith-32vcpu-ubuntu-2404`. A native capacity -probe measured eight CPUs and 30.95 GiB of memory on that class, compared with -15.42 GiB on the previous 16-class. This supplies headroom for isolated runtime -validation without increasing the number of jobs or workers. Workloads still -admit work from observed resources; the runner label is not a capacity guarantee. -PR hydration checks remain on `ubuntu-24.04`. -Its outer GitHub job defaults to 240 minutes. Manual dispatches can override -`timeout_minutes`. Testbox idle timeouts and individual test deadlines remain -separate limits. +Routine dispatched check leases request `blacksmith-16vcpu-ubuntu-2404` through +`ci-check-testbox.yml`, with a 60-minute total GitHub job deadline including +hydration. The explicit `ci-check-high-memory-testbox.yml` workflow requests +`blacksmith-32vcpu-ubuntu-2404` and retains 240 minutes for memory-heavy full-suite +gates. Select it only for a justified memory need, not merely for more time; see +[Testbox runner sizing](/reference/test/remote-proof#testbox-runner-sizing). +A native capacity probe measured eight CPUs and 30.95 GiB of memory on the +32-class, compared with 15.42 GiB on the 16-class. This supplies headroom for +isolated runtime validation without increasing the number of jobs or workers. +Workloads still admit work from observed resources; the runner label is not a +capacity guarantee. PR hydration checks remain on `ubuntu-24.04`. + +The outer GitHub deadline can terminate active SSH commands. Both profiles have +a separate 15-minute idle limit; active SSH prevents idle expiry, not the outer +job deadline. Individual test deadlines also remain separate limits. The standard +workflow accepts an explicit `timeout_minutes` input up to 240 minutes, but +managed Crabbox 0.69.0 does not forward arbitrary workflow inputs, including +`timeout_minutes`, and `--ttl` does not extend a Testbox job. Plan routine proof +within its total-job budget rather than treating TTL or a larger runner as a +deadline override. Sanitized AWS runs set `CRABBOX_ENV_ALLOW=CI`, pass `--no-hydrate`, and use a fresh temporary remote `HOME`; this prevents the repo `OPENCLAW_*` allowlist and existing auth profiles from reaching untrusted code. diff --git a/docs/ci/runners.md b/docs/ci/runners.md index 1d906919cd02..b10d1854cdbf 100644 --- a/docs/ci/runners.md +++ b/docs/ci/runners.md @@ -8,6 +8,79 @@ read_when: ## Runners +### Testbox spending limits + +The general, high-memory, ARM, build-artifact, and Windows Testbox workflows share 32 +concurrency slots. A GitHub-hosted admission job validates the lease ID, runner, +and runtime before the Blacksmith job becomes eligible. Dispatches wait in their +assigned slot without canceling an active lease. Each slot retains one pending +request; a newer request can cancel and replace that pending request. Hash +collisions can leave other slots unused; this is a ceiling, not a promise of +32 busy machines. + +Admission expires ten minutes after the workflow was created. A request that +waits longer fails before checkout or hydration when its runner starts; it can +still incur runner startup cost. This does not remove the queued job immediately. +Stop an abandoned lease by its exact ID instead of leaving a warmup pending. +Do not retry in a loop when the pool is full. + +Idle requests are capped at 15 minutes. The existing Testbox monitor continues +to protect active SSH commands. Stop retained leases when their task finishes; +the idle limit is not a substitute for caller cleanup. + +| Testbox | Allowed runner | Default runtime | Maximum requested runtime | +| --------------------- | --------------------- | --------------- | ------------------------- | +| General runtime proof | 16-vCPU Ubuntu x64 | 60 minutes | 240 minutes | +| High-memory exception | 32-vCPU Ubuntu x64 | 240 minutes | 240 minutes | +| ARM proof | 16-vCPU Ubuntu ARM | 120 minutes | 120 minutes | +| Build artifacts | 16-vCPU Ubuntu x64 | 35 minutes | 35 minutes | +| Windows | 8- or 16-vCPU Windows | 75 minutes | 75 minutes | + +Routine remote proof uses the 16-class. The explicit high-memory workflow can +use only four of the shared 32 slots; it does not add four more slots. Normal +leases can also occupy those slots, so hash collisions may queue high-memory +work below its ceiling. Stop owned leases when done rather than requesting new +IDs to evade a busy slot. + +Use the 32-class only for a named command with measured memory need, a verified +smaller-runner OOM, or a controlled comparison showing lower total billed cost. +Record the evidence before allocation. The explicit full-suite Testbox PR gate +retains this exception; ordinary remote commands and changed gates use 16-class. +See [remote proof](/reference/test/remote-proof#testbox-runner-sizing) for selection. +A failed test, queue delay, or generic timeout does not justify promotion. + +General proof defaults to 60 minutes including hydration. The existing +`timeout_minutes` input can request a shorter deadline or explicitly extend a +known long proof up to 240 minutes; larger values fail before allocation. Native Blacksmith warmup does not expose arbitrary +workflow inputs, and Crabbox `--ttl` does not enforce a Testbox lifetime. +The GitHub job timeout is the wall-clock limit. + +The October 1 sizing audit favors 16-class over a blanket 8-class default. +Across the prior seven days, 5,521 32-class runs had a 31-minute median lifetime +and a 108-minute p95. The memory sampler returned 4,436 eligible runs; sampled +[median](https://github.com/openclaw/openclaw/actions/runs/36588824988/job/109476238445) +and [p75](https://github.com/openclaw/openclaw/actions/runs/36593802388/job/109493424473) +peaks were 6.2 and 12.8 GiB against the 8-class's observed +7.66 GiB. At those sampled peaks, total RAM minus available RAM was 6.8 and +13.4 GiB, so reclaimable cache does not explain away the smaller-class risk. +The separate 6,199-run 16-class population also included 25 OOM-affected jobs; +these populations are not a controlled size comparison. + +Job duration includes hydration, commands, and idle retention. The sampled +[31-minute job](https://github.com/openclaw/openclaw/actions/runs/36630648805/job/109618719716) +spent 30 minutes waiting in the Testbox monitor with a 30-minute +idle setting. Do not treat lease percentiles as active-command percentiles or +claim a measured timeout rate for the new policy. The 60-minute default is an +operating choice; remeasure after the 15-minute idle cap is deployed before +making 30 minutes or 8-class the general default. + +These controls cover dispatches using the updated workflows in this repository. +Historical refs, other repositories, alternate workflows, and Windows probe +workflows are outside the shared pool. Organization-wide concurrency, per-token +admission, SKU restrictions, and a hard spending stop require provider controls. + +### CI runner routing + Hosted Node 24 setup honors the workflow's existing `NODE_VERSION` pin when the setup input is `24.x`. This prevents runner-image refreshes from silently choosing a different patch version. Explicit compatibility versions retain their own diff --git a/docs/reference/test/remote-proof.md b/docs/reference/test/remote-proof.md index 5d6a6707e153..b925fe735656 100644 --- a/docs/reference/test/remote-proof.md +++ b/docs/reference/test/remote-proof.md @@ -112,6 +112,35 @@ Unset all `CRABBOX_TAILSCALE*` overrides, force `--network public --tailscale=false`, clear exit-node/LAN flags, and require `crabbox inspect` to report public networking with no Tailscale state before uploading any script. +## Testbox runner sizing + +Use the default 16-class workflow for routine remote proof, with a 60-minute +total-job deadline including hydration. Keep the 32-class +rare: record the command and its measured memory need, a smaller-runner OOM, +or a controlled comparison showing lower total billed cost before selecting it. +Existing memory-heavy full-suite Testbox PR gates use this exception. A generic +failure, queue delay, or timeout is not a sizing signal. Keep resource-based +worker limits; do not force higher parallelism on the smaller machine. + +Select the exception explicitly with a fresh lease: + +```bash +node scripts/crabbox-wrapper.mjs run \ + --blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \ + --blacksmith-job check --idle-timeout 15m \ + --label --timing-json -- +``` + +The high-memory profile uses at most four of the shared 32 Testbox concurrency +slots. Both profiles cap idle time at 15 minutes. Routine proof defaults to +60 minutes; the explicit high-memory workflow retains its four-hour deadline +for known heavy gates. The standard workflow accepts an explicit +`timeout_minutes` input up to 240 minutes, but Crabbox does not forward arbitrary +workflow inputs and `--ttl` does not extend a Testbox job. Do not select a larger +runner merely for more time. Direct-provider `--class` and `--type` flags do not size Testboxes; +workflow selection owns the runner. A profile change needs a fresh lease. +See [runner limits](/ci/runners#testbox-spending-limits) for queue behavior. + ## Crabbox repository setup The shared [Crabbox skill](https://github.com/openclaw/agent-skills/tree/main/skills/crabbox) @@ -163,7 +192,7 @@ For a selected trusted Testbox lane: ```bash node scripts/crabbox-wrapper.mjs run --timing-json -- \ CI=1 NODE_OPTIONS=--max-old-space-size=4096 \ - OPENCLAW_TEST_PROJECTS_PARALLEL=6 OPENCLAW_VITEST_MAX_WORKERS=1 \ + OPENCLAW_VITEST_MAX_WORKERS=1 \ OPENCLAW_TESTBOX=1 OPENCLAW_TESTBOX_REMOTE_RUN=1 \ pnpm test ``` diff --git a/scripts/ci-testbox-budget.mjs b/scripts/ci-testbox-budget.mjs new file mode 100644 index 000000000000..452e38a2c065 --- /dev/null +++ b/scripts/ci-testbox-budget.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { appendFileSync, readFileSync, writeFileSync } from "node:fs"; +import { pathToFileURL } from "node:url"; + +const PROFILES = { + check: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 240 }, + "check-memory": { runner: "blacksmith-32vcpu-ubuntu-2404", minutes: 240 }, + arm: { runner: "blacksmith-16vcpu-ubuntu-2404-arm", minutes: 120 }, + build: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 35 }, + windows: { runner: "blacksmith-16vcpu-windows-2025", minutes: 75 }, +}; +const WINDOWS_RUNNERS = new Set([ + "blacksmith-8vcpu-windows-2025", + "blacksmith-16vcpu-windows-2025", +]); +const ADMISSION_AGE_MS = 10 * 60_000; +const CONCURRENT_LEASES = 32; +const MAX_IDLE_MINUTES = 15; +const DEFAULT_STANDARD_MINUTES = 60; +const HIGH_MEMORY_LEASES = 4; + +export function assertFreshTestboxAdmission(expiresAt, now = Date.now()) { + const deadline = Number(expiresAt); + if (!Number.isSafeInteger(deadline) || deadline <= now) { + throw new Error("Testbox admission expired after 10 minutes; request a fresh lease."); + } +} + +/** + * @param {{ profile: string, id: string, createdAt: string, runner?: string, minutes?: string | number }} request + */ +export function planTestboxAdmission( + { profile, id, runner, minutes, createdAt }, + now = Date.now(), +) { + const policy = PROFILES[profile]; + if (!policy || !/^tbx_[a-zA-Z0-9_-]+$/.test(id ?? "")) { + throw new Error("Testbox admission requires a known profile and a tbx_ lease ID."); + } + const selectedRunner = runner || policy.runner; + if ( + profile === "windows" ? !WINDOWS_RUNNERS.has(selectedRunner) : selectedRunner !== policy.runner + ) { + throw new Error(`Runner ${selectedRunner} is not allowed for the ${profile} Testbox profile.`); + } + const defaultMinutes = profile === "check" ? DEFAULT_STANDARD_MINUTES : policy.minutes; + const timeout = minutes === undefined || minutes === "" ? defaultMinutes : Number(minutes); + if (!Number.isInteger(timeout) || timeout < 1 || timeout > policy.minutes) { + throw new Error(`Testbox runtime must be an integer from 1 to ${policy.minutes} minutes.`); + } + const created = Date.parse(createdAt); + const expiresAt = created + ADMISSION_AGE_MS; + if (!Number.isFinite(created) || created > now) { + throw new Error("Testbox workflow creation time is invalid."); + } + assertFreshTestboxAdmission(expiresAt, now); + // The shared finite namespace is enforced atomically by GitHub concurrency. + // Do not include a workflow, branch, actor, or runner in this group name. + // High-memory work shares four of the global slots, never a separate pool. + const slots = profile === "check-memory" ? HIGH_MEMORY_LEASES : CONCURRENT_LEASES; + const slot = createHash("sha256").update(id).digest().readUInt32BE(0) % slots; + return { + group: `openclaw-testbox-budget-v1-${slot}`, + runner: selectedRunner, + minutes: timeout, + expires_at: expiresAt, + }; +} + +export function boundedTestboxIdleMinutes(value) { + const minutes = Number(value.trim()); + if (!Number.isInteger(minutes) || minutes < 1) { + throw new Error("Testbox provider returned an invalid idle timeout."); + } + return Math.min(minutes, MAX_IDLE_MINUTES); +} + +async function main() { + if (process.argv[2] === "configure") { + assertFreshTestboxAdmission(process.env.TESTBOX_EXPIRES_AT); + const path = "/tmp/.testbox/idle_timeout"; + const idle = boundedTestboxIdleMinutes(readFileSync(path, "utf8")); + writeFileSync(path, `${idle}\n`); + console.log( + `Testbox idle timeout capped at ${idle} minutes; active SSH work remains protected.`, + ); + return; + } + if (process.argv[2] !== "admit") { + throw new Error("Usage: node scripts/ci-testbox-budget.mjs admit|configure"); + } + const response = await fetch( + `${process.env.GITHUB_API_URL}/repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`, + { + headers: { + Authorization: `Bearer ${process.env.GH_TOKEN}`, + Accept: "application/vnd.github+json", + }, + signal: AbortSignal.timeout(15_000), + }, + ); + if (!response.ok) { + throw new Error(`Cannot establish Testbox dispatch age: GitHub returned ${response.status}.`); + } + const run = await response.json(); + const plan = planTestboxAdmission({ + profile: process.env.TESTBOX_PROFILE, + id: process.env.TESTBOX_ID, + runner: process.env.TESTBOX_RUNNER, + minutes: process.env.TESTBOX_MINUTES, + createdAt: run.created_at, + }); + appendFileSync( + process.env.GITHUB_OUTPUT, + Object.entries(plan) + .map(([key, value]) => `${key}=${value}\n`) + .join(""), + ); + console.log( + `Testbox ${plan.group}: ${plan.runner}, at most ${plan.minutes} minutes, idle ceiling ${MAX_IDLE_MINUTES} minutes.`, + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main().catch((/** @type {unknown} */ error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/scripts/crabbox-wrapper.mts b/scripts/crabbox-wrapper.mts index 29276620d108..f7590ad3272e 100755 --- a/scripts/crabbox-wrapper.mts +++ b/scripts/crabbox-wrapper.mts @@ -1236,7 +1236,7 @@ function enforceCrabboxOwnedBlacksmithLease(commandArgs: string[]) { console.error( [ `[crabbox] provider=blacksmith-testbox --id ${id} has no Crabbox SSH key at ${userDisplayPath(keyPath)}.`, - "[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 90m", + "[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 15m", "[crabbox] direct `blacksmith testbox warmup` leases can be used with `blacksmith testbox run`, but Crabbox cannot sync or run them by id.", ].join("\n"), ); diff --git a/scripts/pr-lib/gates.sh b/scripts/pr-lib/gates.sh index 079f9f93aaa2..f9ea53898321 100644 --- a/scripts/pr-lib/gates.sh +++ b/scripts/pr-lib/gates.sh @@ -190,17 +190,16 @@ run_remote_testbox_full_test_gate() { ' "$script_parent_dir" "${!name}" "$name") || return 2 [ -z "$value" ] || remote_env+=("$name=$value") done - # Same Blacksmith Testbox delegation shape check:changed uses; the worktree's - # own wrapper syncs this prep tree (the canonical copy would sync the primary - # checkout instead). + # Explicit full-suite proof retains the measured high-memory allocation. + # The worktree wrapper syncs this prep tree, not the canonical checkout. run_quiet_logged "$label" "$log_file" \ node scripts/crabbox-wrapper.mjs run \ --provider blacksmith-testbox \ --blacksmith-org openclaw \ - --blacksmith-workflow .github/workflows/ci-check-testbox.yml \ + --blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \ --blacksmith-job check \ --blacksmith-ref main \ - --idle-timeout 90m \ + --idle-timeout 15m \ --ttl 240m \ --timing-json \ --label "$lease_label" \ diff --git a/test/scripts/ancillary-workflow-concurrency.test.ts b/test/scripts/ancillary-workflow-concurrency.test.ts index a9e6c43b794f..449db8d1d20e 100644 --- a/test/scripts/ancillary-workflow-concurrency.test.ts +++ b/test/scripts/ancillary-workflow-concurrency.test.ts @@ -14,6 +14,7 @@ const WORKFLOWS: { file: string; prGroup: string; manual: ManualPolicy; + manualAdmission?: string; push?: { group: string; cancel: boolean }; convertToDraft?: true; }[] = [ @@ -21,16 +22,19 @@ const WORKFLOWS: { file: ".github/workflows/ci-check-testbox.yml", prGroup: "Blacksmith Testbox-pr-v1-123", manual: { mode: "isolated per-run", group: "Blacksmith Testbox-manual-v1-201" }, + manualAdmission: "admission", }, { file: ".github/workflows/ci-check-arm-testbox.yml", prGroup: "Blacksmith ARM Testbox-pr-v1-123", manual: { mode: "isolated per-run", group: "Blacksmith ARM Testbox-manual-v1-201" }, + manualAdmission: "admission", }, { file: ".github/workflows/ci-build-artifacts-testbox.yml", prGroup: "Blacksmith Build Artifacts Testbox-pr-v1-123", manual: { mode: "isolated per-run", group: "Blacksmith Build Artifacts Testbox-manual-v1-201" }, + manualAdmission: "admission", }, { file: ".github/workflows/ios-periphery.yml", @@ -435,7 +439,7 @@ describe.each(WORKFLOWS)("ancillary admission: $file", (policy) => { expect(delayed.group).not.toBe(ready.group); for (const [id, eligible] of Object.entries(ready.eligibility!.jobs)) { if (id !== "scope") { - expect(eligible, id).toBe(true); + expect(eligible, id).toBe(id !== policy.manualAdmission); } } }, diff --git a/test/scripts/ci-testbox-budget.test.ts b/test/scripts/ci-testbox-budget.test.ts new file mode 100644 index 000000000000..563352de173b --- /dev/null +++ b/test/scripts/ci-testbox-budget.test.ts @@ -0,0 +1,135 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { + assertFreshTestboxAdmission, + boundedTestboxIdleMinutes, + planTestboxAdmission, +} from "../../scripts/ci-testbox-budget.mjs"; + +const now = Date.parse("2026-10-01T12:00:00Z"); +const request = { + profile: "check", + id: "tbx_example", + createdAt: "2026-10-01T11:55:00Z", +}; + +describe("Testbox spending admission", () => { + it("shares one bounded pool across profiles without changing a lease's slot", () => { + const groups = new Set(); + for (let index = 0; index < 256; index++) { + const lease = { ...request, id: `tbx_example_${index}` }; + const group = planTestboxAdmission(lease, now).group; + expect(group).toMatch(/^openclaw-testbox-budget-v1-(?:[0-9]|[12][0-9]|3[01])$/); + for (const profile of ["arm", "build", "windows"]) { + expect(planTestboxAdmission({ ...lease, profile }, now).group).toBe(group); + } + groups.add(group); + } + expect(groups.size).toBe(32); + }); + + it("defaults routine proof to 16-class and requires the high-memory profile for 32-class", () => { + expect(planTestboxAdmission(request, now).runner).toBe("blacksmith-16vcpu-ubuntu-2404"); + expect(() => + planTestboxAdmission({ ...request, runner: "blacksmith-32vcpu-ubuntu-2404" }, now), + ).toThrow(/not allowed/); + const groups = new Set(); + for (let index = 0; index < 128; index++) { + const plan = planTestboxAdmission( + { ...request, profile: "check-memory", id: `tbx_memory_${index}` }, + now, + ); + expect(plan.runner).toBe("blacksmith-32vcpu-ubuntu-2404"); + expect(plan.group).toMatch(/^openclaw-testbox-budget-v1-[0-3]$/); + groups.add(plan.group); + } + expect(groups.size).toBe(4); + }); + + it.each([0, -1, 241, 1.5, "invalid"])("rejects an unbounded runtime: %s", (minutes) => { + expect(() => planTestboxAdmission({ ...request, minutes }, now)).toThrow(/runtime/); + }); + + it("defaults routine proof to one hour and preserves explicit long-proof requests", () => { + const workflow = parse(readFileSync(".github/workflows/ci-check-testbox.yml", "utf8")); + const dispatchDefault = workflow.on.workflow_dispatch.inputs.timeout_minutes.default; + expect(planTestboxAdmission({ ...request, minutes: dispatchDefault }, now).minutes).toBe(60); + expect(planTestboxAdmission(request, now).minutes).toBe(60); + expect(planTestboxAdmission({ ...request, minutes: "" }, now).minutes).toBe(60); + expect(planTestboxAdmission({ ...request, minutes: 30 }, now).minutes).toBe(30); + expect(planTestboxAdmission({ ...request, profile: "check-memory" }, now).minutes).toBe(240); + expect(planTestboxAdmission({ ...request, minutes: 240 }, now).minutes).toBe(240); + expect(() => planTestboxAdmission({ ...request, profile: "build", minutes: 36 }, now)).toThrow( + /1 to 35/, + ); + expect(() => planTestboxAdmission({ ...request, profile: "arm", minutes: 121 }, now)).toThrow( + /1 to 120/, + ); + }); + + it("allows approved Windows sizes and refuses arbitrary dispatch labels", () => { + for (const size of [8, 16]) { + const runner = `blacksmith-${size}vcpu-windows-2025`; + expect(planTestboxAdmission({ ...request, profile: "windows", runner }, now).runner).toBe( + runner, + ); + } + for (const runner of ["self-hosted", "blacksmith-32vcpu-windows-2025"]) { + expect(() => planTestboxAdmission({ ...request, profile: "windows", runner }, now)).toThrow( + /not allowed/, + ); + } + }); + + it("expires abandoned queue entries before hydration, including the deadline itself", () => { + const plan = planTestboxAdmission(request, now); + expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at - 1)).not.toThrow(); + expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at)).toThrow(/expired/); + expect(() => planTestboxAdmission(request, plan.expires_at)).toThrow(/expired/); + expect(() => planTestboxAdmission({ ...request, createdAt: "unknown" }, now)).toThrow( + /invalid/, + ); + }); + + it("caps idle requests while retaining shorter provider deadlines", () => { + expect(boundedTestboxIdleMinutes("90\n")).toBe(15); + expect(boundedTestboxIdleMinutes("5\n")).toBe(5); + expect(() => boundedTestboxIdleMinutes("0")).toThrow(/invalid/); + }); + + it.each([ + ["ci-check-testbox.yml", "check", "check"], + ["ci-check-high-memory-testbox.yml", "check", "check-memory"], + ["ci-check-arm-testbox.yml", "check-arm", "arm"], + ["ci-build-artifacts-testbox.yml", "build-artifacts", "build"], + ["windows-blacksmith-testbox.yml", "windows", "windows"], + ])("enforces admission before allocating %s", (file, jobName, profile) => { + const workflow = parse(readFileSync(`.github/workflows/${file}`, "utf8")); + const admission = workflow.jobs.admission; + const job = workflow.jobs[jobName]; + expect(admission["runs-on"]).toBe("ubuntu-24.04"); + const budget = admission.steps.find((step: { id?: string }) => step.id === "budget"); + expect(budget.run).toBe("node scripts/ci-testbox-budget.mjs admit"); + expect(budget.env.TESTBOX_PROFILE).toBe(profile); + expect(job.needs).toBe("admission"); + expect(job.if).toContain("needs.admission.result == 'success'"); + expect(job["runs-on"]).toContain("needs.admission.outputs.runner"); + expect(job.concurrency.group).toContain("needs.admission.outputs.group"); + expect(job.concurrency.queue).toBeUndefined(); + expect(job.concurrency["cancel-in-progress"]).toBe(false); + const names = job.steps.map((step: { name: string }) => step.name); + expect(names.indexOf("Reject expired Testbox admission")).toBeGreaterThan( + names.indexOf("Begin Testbox"), + ); + expect(names.indexOf("Reject expired Testbox admission")).toBeLessThan( + names.indexOf("Checkout"), + ); + if (profile !== "windows") { + expect(names.indexOf("Bound Testbox idle lifetime")).toBeLessThan( + names.indexOf("Setup Node environment"), + ); + expect(names).toContain("Close Testbox SSH sessions"); + } + }); +}); diff --git a/test/scripts/ci-workflow-guards.test.ts b/test/scripts/ci-workflow-guards.test.ts index 36dc3ece5de8..9054fd22c6af 100644 --- a/test/scripts/ci-workflow-guards.test.ts +++ b/test/scripts/ci-workflow-guards.test.ts @@ -873,9 +873,26 @@ AFTER_CD ]); expect(workflow.on.pull_request.paths).toContain(workflowPath); expect(workflow.on.pull_request.paths).not.toContain(".github/workflows/**"); - expect(workflow.jobs[jobName].if).toBe( - "${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}", - ); + for (const [eventName, draft, result, cancelled, admitted] of [ + ["pull_request", false, "skipped", false, true], + ["pull_request", true, "skipped", false, false], + ["pull_request", false, "skipped", true, false], + ["workflow_dispatch", false, "success", false, true], + ["workflow_dispatch", false, "failure", false, false], + ["workflow_dispatch", false, "success", true, false], + ] as const) { + expect( + evaluateWorkflowExpression(workflow.jobs[jobName].if, { + eventName, + draft, + cancelled, + additionalNeeds: { admission: { outputs: {}, result } }, + repository: "openclaw/openclaw", + runAttempt: 1, + }), + `${workflowPath}: ${eventName}, admission=${result}, cancelled=${cancelled}`, + ).toBe(admitted); + } } }); diff --git a/test/scripts/ci-workflow-planning.test.ts b/test/scripts/ci-workflow-planning.test.ts index b885b4124726..875930b90300 100644 --- a/test/scripts/ci-workflow-planning.test.ts +++ b/test/scripts/ci-workflow-planning.test.ts @@ -5494,9 +5494,21 @@ describe("ci workflow guards", () => { // PR events validate the artifact build on hosted runners (landing gate // stays satisfiable during Blacksmith outages); Testbox leases are // dispatch-only, mirroring ci-check-testbox.yml. - expect(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"]).toBe( - "${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }}", - ); + for (const [eventName, expected] of [ + ["pull_request", "ubuntu-24.04"], + ["workflow_dispatch", "blacksmith-16vcpu-ubuntu-2404"], + ] as const) { + expect( + evaluateWorkflowExpression(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"], { + eventName, + repository: "openclaw/openclaw", + runAttempt: 1, + additionalNeeds: { + admission: { outputs: { runner: "blacksmith-16vcpu-ubuntu-2404" }, result: "success" }, + }, + }), + ).toBe(expected); + } for (const stepName of ["Begin Testbox", "Run Testbox"]) { expect( buildArtifactsTestbox.jobs["build-artifacts"].steps.find( diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 9fb076411978..30c11ab87ebd 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -11226,9 +11226,19 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`, expect(setupNodeWith).not.toHaveProperty("dependency-cache"); expect(setupNodeWith).not.toHaveProperty("sticky-disk"); expect(setupNodeWith["cache-mode"]).toBe("restore"); - expect(checkTestboxJob["timeout-minutes"]).toBe( - "${{ fromJSON(inputs.timeout_minutes || '240') }}", - ); + for (const [minutes, expected] of [ + ["45", 45], + ["", 60], + ] as const) { + expect( + evaluateWorkflowExpression(checkTestboxJob["timeout-minutes"], { + eventName: "workflow_dispatch", + repository: "openclaw/openclaw", + runAttempt: 1, + additionalNeeds: { admission: { outputs: { minutes }, result: "success" } }, + }), + ).toBe(expected); + } for (const step of [runTestboxStep, runArmTestboxStep, runBuildArtifactsTestboxStep]) { expect(step.uses).toBe(RUN_TESTBOX_WITH_FAILURE_REPORTING); } @@ -11236,8 +11246,27 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`, "useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d", ); expect(windowsTestboxActionMarker.if).toBe("${{ false }}"); - expect(runTestboxStep.if).toBe("github.event_name == 'workflow_dispatch' && always()"); - expect(closeTestboxSshStep.if).toBe("github.event_name == 'workflow_dispatch' && always()"); + for (const step of [ + runTestboxStep, + runArmTestboxStep, + runBuildArtifactsTestboxStep, + closeTestboxSshStep, + ]) { + for (const [eventName, expected] of [ + ["workflow_dispatch", true], + ["pull_request", false], + ] as const) { + expect( + evaluateWorkflowExpression(`\${{ ${step.if} }}`, { + eventName, + repository: "openclaw/openclaw", + runAttempt: 1, + failed: true, + cancelled: true, + }), + ).toBe(expected); + } + } expect(closeTestboxSshStep.run).toContain( `sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }'`, ); @@ -11247,10 +11276,6 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`, expect(checkTestboxSteps.indexOf(closeTestboxSshStep)).toBe( checkTestboxSteps.indexOf(runTestboxStep) + 1, ); - expect(runArmTestboxStep.if).toBe("always()"); - expect(runBuildArtifactsTestboxStep.if).toBe( - "github.event_name == 'workflow_dispatch' && always()", - ); expect(runWindowsTestboxStep.if).toBe("always()"); expect(runWindowsTestboxStep.env?.JOB_STATUS).toBe("${{ job.status }}"); expect(runWindowsTestboxStep.env?.NATIVE_SSH_USER).toBe( diff --git a/test/scripts/pr-prepare-gates.test.ts b/test/scripts/pr-prepare-gates.test.ts index c402cb07ace8..550eead6fb49 100644 --- a/test/scripts/pr-prepare-gates.test.ts +++ b/test/scripts/pr-prepare-gates.test.ts @@ -518,13 +518,13 @@ describe("remote testbox gate delegation", () => { "--blacksmith-org", "openclaw", "--blacksmith-workflow", - ".github/workflows/ci-check-testbox.yml", + ".github/workflows/ci-check-high-memory-testbox.yml", "--blacksmith-job", "check", "--blacksmith-ref", "main", "--idle-timeout", - "90m", + "15m", "--ttl", "240m", "--timing-json", diff --git a/test/scripts/testbox-base.test.ts b/test/scripts/testbox-base.test.ts index 513e7d2fa6f3..1a055a390313 100644 --- a/test/scripts/testbox-base.test.ts +++ b/test/scripts/testbox-base.test.ts @@ -70,7 +70,7 @@ function runBasePreparation( eventName: string, ) { const workflow = parse(fs.readFileSync(workflowName, "utf8")); - const job = Object.values(workflow.jobs)[0] as { steps: Step[] }; + const jobs = Object.values(workflow.jobs) as { steps: Step[] }[]; const values = new Map([ ["github.event.pull_request.base.sha", base], ["github.event.pull_request.base.ref", "main"], @@ -99,7 +99,9 @@ function runBasePreparation( '#!/bin/sh\nif [ "$1" = tee ]; then cat >/dev/null; fi\n', ); fs.chmodSync(path.join(bin, "sudo"), 0o755); - const step = job.steps.find((entry) => entry.name === "Prepare Testbox shell"); + const step = jobs + .flatMap((job) => job.steps) + .find((entry) => entry.uses === "./.github/actions/prepare-testbox-shell"); if (!step?.uses) { throw new Error("Missing Testbox preparation action"); }