fix(ci): cap Testbox concurrency and idle spend (#162678)

* fix(ci): cap Testbox concurrency and idle spend

* test(ci): align workflow guards with Testbox admission

* fix(ci): declare optional Testbox admission inputs

* fix(ci): narrow Testbox CLI rejection errors

* test(ci): verify admitted Testbox workflow behavior

* fix(ci): reserve large Testboxes for memory-heavy proof

* fix(ci): default routine Testboxes to one hour

* docs(ci): clarify Testbox profiles and total job deadlines

Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
Vincent Koc 2026-10-02 02:16:05 +07:00 • committed by GitHub
parent 4359fb6ca9
commit 32dc6c861d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
22 changed files with 948 additions and 53 deletions

View file

@ -61,6 +61,14 @@ Source trust determines which providers are allowed. It does not select one.
- Never untrusted code on credential-hydrated Testbox.
- Never run untrusted repo wrapper/config locally.
- No speculative warmup. Acquire when first heavy command ready. Reuse id. Stop.
- Use the consumer's smallest proven remote profile. Large/high-memory profiles
are explicit exceptions for a named command with measured memory need or a
controlled total-cost benefit. Record the reason before allocation; a generic
failure, queue delay, or timeout is not a reason to upsize. Do not raise worker
counts to compensate for a smaller machine.
- For delegated Testbox sizing, select a documented workflow through
`--blacksmith-workflow`; direct-provider `--class`/`--type` flags do not choose
the Testbox runner. Change profiles with a fresh lease.
Test size, expected duration, and hydration failure do not authorize a provider
override. Omit `--provider` for normal work. Add it only when the user requests

View file

@ -29,6 +29,27 @@ availability, Blacksmith control-plane health, and downstream queue drains.
scans should stay on GitHub-hosted runners unless measured evidence says
Blacksmith is required.
## Runner Cost Policy
- Use the smallest runner that completes the required workload reliably. Keep
short control jobs hosted and ordinary trusted development proof local.
- Treat the 32-class as an exception, not a default or generic retry. Record the
command and measured peak memory, a smaller-runner OOM, or a controlled
comparison showing lower total billed cost. Low CPU use alone does not prove
a memory-heavy job can move down. Preserve resource-based worker limits.
- Routine OpenClaw Testboxes use the 16-class with a 60-minute total-job
default, including hydration. Keep the 15-minute idle ceiling. Shorter
deadlines are welcome for known short commands; do not request four hours
automatically or upsize only to obtain more time. Select
`.github/workflows/ci-check-high-memory-testbox.yml` explicitly only for a
named memory-heavy command; see `docs/reference/test/remote-proof.md`.
That workflow has at most four concurrent leases inside the shared 32-slot
Testbox pool. All Testbox profiles cap idle time at 15 minutes.
- Do not promote an entire workflow family because one command needs more RAM.
Keep proven high-memory CI rows scoped to their owning planner and evidence;
remeasure before changing their allocation. A 32-class label is not proof of
32 available CPUs. Compare observed resources and total billed job cost.
## Rejected Experiments
- **Boundary asynchronous input preparation (2026-09-26):** Adding the existing

View file

@ -28,6 +28,7 @@ actions:
ephemeral: true
blacksmith:
org: openclaw
# Routine remote proof uses 16-class; high-memory proof opts in by workflow.
workflow: .github/workflows/ci-check-testbox.yml
job: check
ref: main
@ -126,7 +127,7 @@ jobs:
testbox-changed:
provider: blacksmith-testbox
target: linux
idleTimeout: 90m
idleTimeout: 15m
hydrate:
actions: false
actions:

View file

@ -10,6 +10,7 @@ on:
pull_request:
types: [opened, reopened, synchronize, ready_for_review]
paths:
- "scripts/ci-testbox-budget.mjs"
- ".github/workflows/ci-build-artifacts-testbox.yml"
- ".github/actions/prepare-testbox-shell/**"
- ".github/actions/setup-node-env/**"
@ -62,16 +63,47 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
admission:
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-24.04
timeout-minutes: 3
permissions:
contents: read
actions: read
outputs:
group: ${{ steps.budget.outputs.group }}
runner: ${{ steps.budget.outputs.runner }}
minutes: ${{ steps.budget.outputs.minutes }}
expires_at: ${{ steps.budget.outputs.expires_at }}
steps:
- name: Checkout admission policy
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts/ci-testbox-budget.mjs
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Admit Testbox within the shared budget
id: budget
env:
GH_TOKEN: ${{ github.token }}
TESTBOX_PROFILE: build
TESTBOX_ID: ${{ inputs.testbox_id }}
run: node scripts/ci-testbox-budget.mjs admit
build-artifacts:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
needs: admission
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
concurrency:
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
cancel-in-progress: false
permissions:
contents: read
name: "build-artifacts"
# Pull requests only validate the artifact build; real Testbox leases
# arrive via dispatch. Hosted PR runs keep this landing gate satisfiable
# during Blacksmith outages (mirrors ci-check-testbox.yml).
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }}
timeout-minutes: 35
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }}
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '35') }}
steps:
# Testbox lifecycle actions require Blacksmith VM metadata; PRs validate the build only.
- name: Begin Testbox
@ -80,12 +112,30 @@ jobs:
with:
testbox_id: ${{ inputs.testbox_id }}
- name: Reject expired Testbox admission
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: |
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
exit 1
fi
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }}
persist-credentials: false
- name: Bound Testbox idle lifetime
if: github.event_name == 'workflow_dispatch'
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: node scripts/ci-testbox-budget.mjs configure
- name: Setup Node environment
id: setup-node-env
uses: ./.github/actions/setup-node-env
@ -257,3 +307,23 @@ jobs:
if: github.event_name == 'workflow_dispatch' && always()
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
- name: Close Testbox SSH sessions
if: github.event_name == 'workflow_dispatch' && always()
shell: bash
run: |
set -euo pipefail
# Testbox state stores Blacksmith's external forwarded port. Resolve
# sshd's VM-local listener because that is the sport visible to ss.
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
echo "No valid local SSH listener port found; skipping session cleanup"
exit 0
fi
# run-testbox has no post hook. Close only Testbox client sockets so
# Blacksmith's runner teardown does not wait for its 290-second grace.
timeout --signal=KILL 5s sudo ss -K state established \
"( sport = :${runner_ssh_local_port} )" || true

View file

@ -9,6 +9,7 @@ on:
pull_request:
types: [opened, reopened, synchronize, ready_for_review]
paths:
- "scripts/ci-testbox-budget.mjs"
- ".github/workflows/ci-check-arm-testbox.yml"
- ".github/actions/prepare-testbox-shell/**"
- ".github/actions/setup-node-env/**"
@ -56,18 +57,62 @@ env:
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
jobs:
admission:
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-24.04
timeout-minutes: 3
permissions:
contents: read
actions: read
outputs:
group: ${{ steps.budget.outputs.group }}
runner: ${{ steps.budget.outputs.runner }}
minutes: ${{ steps.budget.outputs.minutes }}
expires_at: ${{ steps.budget.outputs.expires_at }}
steps:
- name: Checkout admission policy
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts/ci-testbox-budget.mjs
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Admit Testbox within the shared budget
id: budget
env:
GH_TOKEN: ${{ github.token }}
TESTBOX_PROFILE: arm
TESTBOX_ID: ${{ inputs.testbox_id }}
run: node scripts/ci-testbox-budget.mjs admit
check-arm:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
needs: admission
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
concurrency:
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
cancel-in-progress: false
permissions:
contents: read
name: "check-arm"
runs-on: blacksmith-16vcpu-ubuntu-2404-arm
timeout-minutes: 120
runs-on: ${{ github.event_name == 'pull_request' && 'blacksmith-16vcpu-ubuntu-2404-arm' || needs.admission.outputs.runner }}
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '120') }}
steps:
- name: Begin Testbox
if: github.event_name == 'workflow_dispatch'
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2
with:
testbox_id: ${{ inputs.testbox_id }}
- name: Reject expired Testbox admission
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: |
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
exit 1
fi
- name: Verify ARM runner
shell: bash
run: |
@ -88,6 +133,12 @@ jobs:
with:
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }}
persist-credentials: false
- name: Bound Testbox idle lifetime
if: github.event_name == 'workflow_dispatch'
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: node scripts/ci-testbox-budget.mjs configure
- name: Setup Node environment
uses: ./.github/actions/setup-node-env
with:
@ -155,6 +206,26 @@ jobs:
# Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15
# Return to useblacksmith/run-testbox after the fix lands upstream.
uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4
if: always()
if: github.event_name == 'workflow_dispatch' && always()
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
- name: Close Testbox SSH sessions
if: github.event_name == 'workflow_dispatch' && always()
shell: bash
run: |
set -euo pipefail
# Testbox state stores Blacksmith's external forwarded port. Resolve
# sshd's VM-local listener because that is the sport visible to ss.
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
echo "No valid local SSH listener port found; skipping session cleanup"
exit 0
fi
# run-testbox has no post hook. Close only Testbox client sockets so
# Blacksmith's runner teardown does not wait for its 290-second grace.
timeout --signal=KILL 5s sudo ss -K state established \
"( sport = :${runner_ssh_local_port} )" || true

View file

@ -0,0 +1,183 @@
name: Blacksmith High-Memory Testbox
on:
workflow_dispatch:
inputs:
testbox_id:
type: string
description: "Testbox session ID"
required: true
timeout_minutes:
type: number
description: "Maximum GitHub job runtime for long Testbox commands"
default: 240
permissions:
contents: read
# Explicit memory-heavy exception; ordinary proof uses ci-check-testbox.yml.
concurrency:
group: ${{ github.workflow }}-manual-v1-${{ github.run_id }}
cancel-in-progress: false
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
jobs:
admission:
runs-on: ubuntu-24.04
timeout-minutes: 3
permissions:
contents: read
actions: read
outputs:
group: ${{ steps.budget.outputs.group }}
runner: ${{ steps.budget.outputs.runner }}
minutes: ${{ steps.budget.outputs.minutes }}
expires_at: ${{ steps.budget.outputs.expires_at }}
steps:
- name: Checkout admission policy
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts/ci-testbox-budget.mjs
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Admit Testbox within the shared budget
id: budget
env:
GH_TOKEN: ${{ github.token }}
TESTBOX_PROFILE: check-memory
TESTBOX_ID: ${{ inputs.testbox_id }}
TESTBOX_MINUTES: ${{ inputs.timeout_minutes }}
run: node scripts/ci-testbox-budget.mjs admit
check:
needs: admission
if: ${{ !cancelled() && needs.admission.result == 'success' }}
concurrency:
group: ${{ needs.admission.outputs.group }}
cancel-in-progress: false
permissions:
contents: read
name: "check"
runs-on: ${{ needs.admission.outputs.runner }}
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '240') }}
steps:
- name: Begin Testbox
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043
with:
testbox_id: ${{ inputs.testbox_id }}
- name: Reject expired Testbox admission
shell: bash
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: |
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
exit 1
fi
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 1
persist-credentials: false
- name: Bound Testbox idle lifetime
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: node scripts/ci-testbox-budget.mjs configure
- name: Setup Node environment
uses: ./.github/actions/setup-node-env
with:
semantic-checks: "true"
cache-mode: restore
install-bun: "false"
# Testbox hydration uses the ordinary pnpm store cache. Canonical CI
# owns the exact dependency archive and never delegates here.
- name: Prepare Testbox shell
uses: ./.github/actions/prepare-testbox-shell
with:
base-ref: HEAD
go-version: "1.27.1"
- name: Hydrate Testbox provider env helper
shell: bash
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }}
ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }}
BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }}
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }}
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }}
MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }}
MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }}
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }}
OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }}
OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }}
OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }}
OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }}
OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}
FAL_KEY: ${{ secrets.FAL_KEY }}
RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }}
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }}
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }}
Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }}
BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }}
BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }}
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }}
OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }}
OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }}
OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }}
OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }}
OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }}
OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }}
run: bash scripts/ci-hydrate-testbox-env.sh
- name: Run Testbox
# Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15
# Return to useblacksmith/run-testbox after the fix lands upstream.
uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4
if: always()
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
- name: Close Testbox SSH sessions
if: always()
shell: bash
run: |
set -euo pipefail
# Testbox state stores Blacksmith's external forwarded port. Resolve
# sshd's VM-local listener because that is the sport visible to ss.
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
echo "No valid local SSH listener port found; skipping session cleanup"
exit 0
fi
# run-testbox has no post hook. Close only Testbox client sockets so
# Blacksmith's runner teardown does not wait for its 290-second grace.
timeout --signal=KILL 5s sudo ss -K state established \
"( sport = :${runner_ssh_local_port} )" || true

View file

@ -8,11 +8,12 @@ on:
required: true
timeout_minutes:
type: number
description: "Maximum GitHub job runtime for long Testbox commands"
default: 240
description: "GitHub job runtime; explicit long-proof requests may use up to 240 minutes"
default: 60
pull_request:
types: [opened, reopened, synchronize, ready_for_review]
paths:
- "scripts/ci-testbox-budget.mjs"
- ".github/workflows/ci-check-testbox.yml"
- ".github/actions/prepare-testbox-shell/**"
- ".github/actions/setup-node-env/**"
@ -59,14 +60,46 @@ env:
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
jobs:
admission:
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-24.04
timeout-minutes: 3
permissions:
contents: read
actions: read
outputs:
group: ${{ steps.budget.outputs.group }}
runner: ${{ steps.budget.outputs.runner }}
minutes: ${{ steps.budget.outputs.minutes }}
expires_at: ${{ steps.budget.outputs.expires_at }}
steps:
- name: Checkout admission policy
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts/ci-testbox-budget.mjs
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Admit Testbox within the shared budget
id: budget
env:
GH_TOKEN: ${{ github.token }}
TESTBOX_PROFILE: check
TESTBOX_ID: ${{ inputs.testbox_id }}
TESTBOX_MINUTES: ${{ inputs.timeout_minutes }}
run: node scripts/ci-testbox-budget.mjs admit
check:
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
needs: admission
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
concurrency:
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
cancel-in-progress: false
permissions:
contents: read
name: "check"
# Pull requests only validate the hydration steps; real Testbox leases arrive via dispatch.
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }}
timeout-minutes: ${{ fromJSON(inputs.timeout_minutes || '240') }}
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }}
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '60') }}
steps:
# Testbox lifecycle actions require Blacksmith VM metadata; PRs validate our setup steps only.
- name: Begin Testbox
@ -74,6 +107,18 @@ jobs:
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043
with:
testbox_id: ${{ inputs.testbox_id }}
- name: Reject expired Testbox admission
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: |
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
exit 1
fi
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@ -81,6 +126,12 @@ jobs:
# reconstructs its exact base and final tree. PR validation keeps both commits.
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '1' }}
persist-credentials: false
- name: Bound Testbox idle lifetime
if: github.event_name == 'workflow_dispatch'
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: node scripts/ci-testbox-budget.mjs configure
- name: Setup Node environment
uses: ./.github/actions/setup-node-env
with:

View file

@ -8,7 +8,10 @@ on:
description: "Testbox session ID"
required: true
runner_label:
type: string
type: choice
options:
- blacksmith-8vcpu-windows-2025
- blacksmith-16vcpu-windows-2025
description: "Windows runner label"
required: false
default: "blacksmith-16vcpu-windows-2025"
@ -20,10 +23,43 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
jobs:
admission:
if: github.event_name == 'workflow_dispatch'
runs-on: ubuntu-24.04
timeout-minutes: 3
permissions:
contents: read
actions: read
outputs:
group: ${{ steps.budget.outputs.group }}
runner: ${{ steps.budget.outputs.runner }}
minutes: ${{ steps.budget.outputs.minutes }}
expires_at: ${{ steps.budget.outputs.expires_at }}
steps:
- name: Checkout admission policy
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
sparse-checkout: scripts/ci-testbox-budget.mjs
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Admit Testbox within the shared budget
id: budget
env:
GH_TOKEN: ${{ github.token }}
TESTBOX_PROFILE: windows
TESTBOX_ID: ${{ inputs.testbox_id }}
TESTBOX_RUNNER: ${{ inputs.runner_label }}
run: node scripts/ci-testbox-budget.mjs admit
windows:
needs: admission
if: ${{ !cancelled() && needs.admission.result == 'success' }}
concurrency:
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
cancel-in-progress: false
name: windows
runs-on: ${{ inputs.runner_label }}
timeout-minutes: 75
runs-on: ${{ needs.admission.outputs.runner }}
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes) }}
defaults:
run:
shell: pwsh
@ -138,6 +174,18 @@ jobs:
printf 'public_key_path=%s\n' "$public_key_path" >> "$GITHUB_OUTPUT"
printf 'bash_os=%s\nbash_user=%s\n' "$(uname -s)" "$(id -un)"
- name: Reject expired Testbox admission
if: github.event_name == 'workflow_dispatch'
shell: bash
env:
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
run: |
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
exit 1
fi
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@ -217,6 +265,11 @@ jobs:
auth_token="$(cat "$state/auth_token")"
idle_timeout="$(cat "$state/idle_timeout" 2>/dev/null || true)"
idle_timeout="${idle_timeout:-10}"
if [[ ! "$idle_timeout" =~ ^[0-9]+$ ]] || (( idle_timeout < 1 )); then
echo "Invalid Testbox idle timeout" >&2
exit 1
fi
if (( idle_timeout > 15 )); then idle_timeout=15; fi
api_url="$(cat "$state/api_url")"
runner_host="$(cat "$state/runner_host")"
runner_ssh_port="$(cat "$state/runner_ssh_port")"

View file

@ -62,7 +62,7 @@ file, so large PRs do not lose test-planning inputs to Actions output or environ
size limits. Frozen targets that predate this transport retain their bounded JSON
output contract. Missing or invalid inputs still reject current PR Node planning.
The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 32-class for dispatched proof and defaults to a four-hour outer job budget. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged.
The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 16-class for routine dispatched proof, with a 60-minute total-job deadline including hydration. The explicit high-memory 32-class workflow retains 240 minutes for memory-heavy full-suite gates. The outer GitHub deadline can terminate active SSH commands; the separate 15-minute idle limit does not extend it. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged.
Full GitHub and hybrid type checks run the five core stripes independently, retaining two compiler children per job. The last four rows then each run one root-test partition serially, leaving extension tests and scripts in the central row. Narrow plans reuse four already-selected rows when available; smaller selections retain central root checking. This adds no jobs or compiler overlap. Current hybrid full runs use three hosted extension-lint jobs; targeted layouts retain six stripe identities. Trusted hybrid first attempts place both packed core-lint rows on the Blacksmith 16-class and the final gate on the 4-class to avoid serial hosted assignment delays. Frozen targets keep their earlier layout; see [static checks](/ci/runners#runner-backend-modes).

View file

@ -369,15 +369,26 @@ Provider readiness and broker authentication still determine
which configured backend can run the proof.
The check workflow hydrates its pinned dispatch commit with a depth-1 checkout;
the changed gate later reconstructs the exact merge base and synced final tree.
Dispatched check leases request `blacksmith-32vcpu-ubuntu-2404`. A native capacity
probe measured eight CPUs and 30.95 GiB of memory on that class, compared with
15.42 GiB on the previous 16-class. This supplies headroom for isolated runtime
validation without increasing the number of jobs or workers. Workloads still
admit work from observed resources; the runner label is not a capacity guarantee.
PR hydration checks remain on `ubuntu-24.04`.
Its outer GitHub job defaults to 240 minutes. Manual dispatches can override
`timeout_minutes`. Testbox idle timeouts and individual test deadlines remain
separate limits.
Routine dispatched check leases request `blacksmith-16vcpu-ubuntu-2404` through
`ci-check-testbox.yml`, with a 60-minute total GitHub job deadline including
hydration. The explicit `ci-check-high-memory-testbox.yml` workflow requests
`blacksmith-32vcpu-ubuntu-2404` and retains 240 minutes for memory-heavy full-suite
gates. Select it only for a justified memory need, not merely for more time; see
[Testbox runner sizing](/reference/test/remote-proof#testbox-runner-sizing).
A native capacity probe measured eight CPUs and 30.95 GiB of memory on the
32-class, compared with 15.42 GiB on the 16-class. This supplies headroom for
isolated runtime validation without increasing the number of jobs or workers.
Workloads still admit work from observed resources; the runner label is not a
capacity guarantee. PR hydration checks remain on `ubuntu-24.04`.
The outer GitHub deadline can terminate active SSH commands. Both profiles have
a separate 15-minute idle limit; active SSH prevents idle expiry, not the outer
job deadline. Individual test deadlines also remain separate limits. The standard
workflow accepts an explicit `timeout_minutes` input up to 240 minutes, but
managed Crabbox 0.69.0 does not forward arbitrary workflow inputs, including
`timeout_minutes`, and `--ttl` does not extend a Testbox job. Plan routine proof
within its total-job budget rather than treating TTL or a larger runner as a
deadline override.
Sanitized AWS runs set `CRABBOX_ENV_ALLOW=CI`, pass
`--no-hydrate`, and use a fresh temporary remote `HOME`; this prevents the repo
`OPENCLAW_*` allowlist and existing auth profiles from reaching untrusted code.

View file

@ -8,6 +8,79 @@ read_when:
## Runners
### Testbox spending limits
The general, high-memory, ARM, build-artifact, and Windows Testbox workflows share 32
concurrency slots. A GitHub-hosted admission job validates the lease ID, runner,
and runtime before the Blacksmith job becomes eligible. Dispatches wait in their
assigned slot without canceling an active lease. Each slot retains one pending
request; a newer request can cancel and replace that pending request. Hash
collisions can leave other slots unused; this is a ceiling, not a promise of
32 busy machines.
Admission expires ten minutes after the workflow was created. A request that
waits longer fails before checkout or hydration when its runner starts; it can
still incur runner startup cost. This does not remove the queued job immediately.
Stop an abandoned lease by its exact ID instead of leaving a warmup pending.
Do not retry in a loop when the pool is full.
Idle requests are capped at 15 minutes. The existing Testbox monitor continues
to protect active SSH commands. Stop retained leases when their task finishes;
the idle limit is not a substitute for caller cleanup.
| Testbox | Allowed runner | Default runtime | Maximum requested runtime |
| --------------------- | --------------------- | --------------- | ------------------------- |
| General runtime proof | 16-vCPU Ubuntu x64 | 60 minutes | 240 minutes |
| High-memory exception | 32-vCPU Ubuntu x64 | 240 minutes | 240 minutes |
| ARM proof | 16-vCPU Ubuntu ARM | 120 minutes | 120 minutes |
| Build artifacts | 16-vCPU Ubuntu x64 | 35 minutes | 35 minutes |
| Windows | 8- or 16-vCPU Windows | 75 minutes | 75 minutes |
Routine remote proof uses the 16-class. The explicit high-memory workflow can
use only four of the shared 32 slots; it does not add four more slots. Normal
leases can also occupy those slots, so hash collisions may queue high-memory
work below its ceiling. Stop owned leases when done rather than requesting new
IDs to evade a busy slot.
Use the 32-class only for a named command with measured memory need, a verified
smaller-runner OOM, or a controlled comparison showing lower total billed cost.
Record the evidence before allocation. The explicit full-suite Testbox PR gate
retains this exception; ordinary remote commands and changed gates use 16-class.
See [remote proof](/reference/test/remote-proof#testbox-runner-sizing) for selection.
A failed test, queue delay, or generic timeout does not justify promotion.
General proof defaults to 60 minutes including hydration. The existing
`timeout_minutes` input can request a shorter deadline or explicitly extend a
known long proof up to 240 minutes; larger values fail before allocation. Native Blacksmith warmup does not expose arbitrary
workflow inputs, and Crabbox `--ttl` does not enforce a Testbox lifetime.
The GitHub job timeout is the wall-clock limit.
The October 1 sizing audit favors 16-class over a blanket 8-class default.
Across the prior seven days, 5,521 32-class runs had a 31-minute median lifetime
and a 108-minute p95. The memory sampler returned 4,436 eligible runs; sampled
[median](https://github.com/openclaw/openclaw/actions/runs/36588824988/job/109476238445)
and [p75](https://github.com/openclaw/openclaw/actions/runs/36593802388/job/109493424473)
peaks were 6.2 and 12.8 GiB against the 8-class's observed
7.66 GiB. At those sampled peaks, total RAM minus available RAM was 6.8 and
13.4 GiB, so reclaimable cache does not explain away the smaller-class risk.
The separate 6,199-run 16-class population also included 25 OOM-affected jobs;
these populations are not a controlled size comparison.
Job duration includes hydration, commands, and idle retention. The sampled
[31-minute job](https://github.com/openclaw/openclaw/actions/runs/36630648805/job/109618719716)
spent 30 minutes waiting in the Testbox monitor with a 30-minute
idle setting. Do not treat lease percentiles as active-command percentiles or
claim a measured timeout rate for the new policy. The 60-minute default is an
operating choice; remeasure after the 15-minute idle cap is deployed before
making 30 minutes or 8-class the general default.
These controls cover dispatches using the updated workflows in this repository.
Historical refs, other repositories, alternate workflows, and Windows probe
workflows are outside the shared pool. Organization-wide concurrency, per-token
admission, SKU restrictions, and a hard spending stop require provider controls.
### CI runner routing
Hosted Node 24 setup honors the workflow's existing `NODE_VERSION` pin when the
setup input is `24.x`. This prevents runner-image refreshes from silently choosing
a different patch version. Explicit compatibility versions retain their own

View file

@ -112,6 +112,35 @@ Unset all `CRABBOX_TAILSCALE*` overrides, force `--network public
--tailscale=false`, clear exit-node/LAN flags, and require `crabbox inspect` to
report public networking with no Tailscale state before uploading any script.
## Testbox runner sizing
Use the default 16-class workflow for routine remote proof, with a 60-minute
total-job deadline including hydration. Keep the 32-class
rare: record the command and its measured memory need, a smaller-runner OOM,
or a controlled comparison showing lower total billed cost before selecting it.
Existing memory-heavy full-suite Testbox PR gates use this exception. A generic
failure, queue delay, or timeout is not a sizing signal. Keep resource-based
worker limits; do not force higher parallelism on the smaller machine.
Select the exception explicitly with a fresh lease:
```bash
node scripts/crabbox-wrapper.mjs run \
--blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \
--blacksmith-job check --idle-timeout 15m \
--label <task-and-memory-reason> --timing-json -- <command>
```
The high-memory profile uses at most four of the shared 32 Testbox concurrency
slots. Both profiles cap idle time at 15 minutes. Routine proof defaults to
60 minutes; the explicit high-memory workflow retains its four-hour deadline
for known heavy gates. The standard workflow accepts an explicit
`timeout_minutes` input up to 240 minutes, but Crabbox does not forward arbitrary
workflow inputs and `--ttl` does not extend a Testbox job. Do not select a larger
runner merely for more time. Direct-provider `--class` and `--type` flags do not size Testboxes;
workflow selection owns the runner. A profile change needs a fresh lease.
See [runner limits](/ci/runners#testbox-spending-limits) for queue behavior.
## Crabbox repository setup
The shared [Crabbox skill](https://github.com/openclaw/agent-skills/tree/main/skills/crabbox)
@ -163,7 +192,7 @@ For a selected trusted Testbox lane:
```bash
node scripts/crabbox-wrapper.mjs run --timing-json -- \
CI=1 NODE_OPTIONS=--max-old-space-size=4096 \
OPENCLAW_TEST_PROJECTS_PARALLEL=6 OPENCLAW_VITEST_MAX_WORKERS=1 \
OPENCLAW_VITEST_MAX_WORKERS=1 \
OPENCLAW_TESTBOX=1 OPENCLAW_TESTBOX_REMOTE_RUN=1 \
pnpm test <path-or-filter>
```

View file

@ -0,0 +1,130 @@
#!/usr/bin/env node
import { createHash } from "node:crypto";
import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
import { pathToFileURL } from "node:url";
const PROFILES = {
check: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 240 },
"check-memory": { runner: "blacksmith-32vcpu-ubuntu-2404", minutes: 240 },
arm: { runner: "blacksmith-16vcpu-ubuntu-2404-arm", minutes: 120 },
build: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 35 },
windows: { runner: "blacksmith-16vcpu-windows-2025", minutes: 75 },
};
const WINDOWS_RUNNERS = new Set([
"blacksmith-8vcpu-windows-2025",
"blacksmith-16vcpu-windows-2025",
]);
const ADMISSION_AGE_MS = 10 * 60_000;
const CONCURRENT_LEASES = 32;
const MAX_IDLE_MINUTES = 15;
const DEFAULT_STANDARD_MINUTES = 60;
const HIGH_MEMORY_LEASES = 4;
export function assertFreshTestboxAdmission(expiresAt, now = Date.now()) {
const deadline = Number(expiresAt);
if (!Number.isSafeInteger(deadline) || deadline <= now) {
throw new Error("Testbox admission expired after 10 minutes; request a fresh lease.");
}
}
/**
* @param {{ profile: string, id: string, createdAt: string, runner?: string, minutes?: string | number }} request
*/
export function planTestboxAdmission(
{ profile, id, runner, minutes, createdAt },
now = Date.now(),
) {
const policy = PROFILES[profile];
if (!policy || !/^tbx_[a-zA-Z0-9_-]+$/.test(id ?? "")) {
throw new Error("Testbox admission requires a known profile and a tbx_ lease ID.");
}
const selectedRunner = runner || policy.runner;
if (
profile === "windows" ? !WINDOWS_RUNNERS.has(selectedRunner) : selectedRunner !== policy.runner
) {
throw new Error(`Runner ${selectedRunner} is not allowed for the ${profile} Testbox profile.`);
}
const defaultMinutes = profile === "check" ? DEFAULT_STANDARD_MINUTES : policy.minutes;
const timeout = minutes === undefined || minutes === "" ? defaultMinutes : Number(minutes);
if (!Number.isInteger(timeout) || timeout < 1 || timeout > policy.minutes) {
throw new Error(`Testbox runtime must be an integer from 1 to ${policy.minutes} minutes.`);
}
const created = Date.parse(createdAt);
const expiresAt = created + ADMISSION_AGE_MS;
if (!Number.isFinite(created) || created > now) {
throw new Error("Testbox workflow creation time is invalid.");
}
assertFreshTestboxAdmission(expiresAt, now);
// The shared finite namespace is enforced atomically by GitHub concurrency.
// Do not include a workflow, branch, actor, or runner in this group name.
// High-memory work shares four of the global slots, never a separate pool.
const slots = profile === "check-memory" ? HIGH_MEMORY_LEASES : CONCURRENT_LEASES;
const slot = createHash("sha256").update(id).digest().readUInt32BE(0) % slots;
return {
group: `openclaw-testbox-budget-v1-${slot}`,
runner: selectedRunner,
minutes: timeout,
expires_at: expiresAt,
};
}
export function boundedTestboxIdleMinutes(value) {
const minutes = Number(value.trim());
if (!Number.isInteger(minutes) || minutes < 1) {
throw new Error("Testbox provider returned an invalid idle timeout.");
}
return Math.min(minutes, MAX_IDLE_MINUTES);
}
async function main() {
if (process.argv[2] === "configure") {
assertFreshTestboxAdmission(process.env.TESTBOX_EXPIRES_AT);
const path = "/tmp/.testbox/idle_timeout";
const idle = boundedTestboxIdleMinutes(readFileSync(path, "utf8"));
writeFileSync(path, `${idle}\n`);
console.log(
`Testbox idle timeout capped at ${idle} minutes; active SSH work remains protected.`,
);
return;
}
if (process.argv[2] !== "admit") {
throw new Error("Usage: node scripts/ci-testbox-budget.mjs admit|configure");
}
const response = await fetch(
`${process.env.GITHUB_API_URL}/repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
{
headers: {
Authorization: `Bearer ${process.env.GH_TOKEN}`,
Accept: "application/vnd.github+json",
},
signal: AbortSignal.timeout(15_000),
},
);
if (!response.ok) {
throw new Error(`Cannot establish Testbox dispatch age: GitHub returned ${response.status}.`);
}
const run = await response.json();
const plan = planTestboxAdmission({
profile: process.env.TESTBOX_PROFILE,
id: process.env.TESTBOX_ID,
runner: process.env.TESTBOX_RUNNER,
minutes: process.env.TESTBOX_MINUTES,
createdAt: run.created_at,
});
appendFileSync(
process.env.GITHUB_OUTPUT,
Object.entries(plan)
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
console.log(
`Testbox ${plan.group}: ${plan.runner}, at most ${plan.minutes} minutes, idle ceiling ${MAX_IDLE_MINUTES} minutes.`,
);
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
main().catch((/** @type {unknown} */ error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
});
}

View file

@ -1236,7 +1236,7 @@ function enforceCrabboxOwnedBlacksmithLease(commandArgs: string[]) {
console.error(
[
`[crabbox] provider=blacksmith-testbox --id ${id} has no Crabbox SSH key at ${userDisplayPath(keyPath)}.`,
"[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 90m",
"[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 15m",
"[crabbox] direct `blacksmith testbox warmup` leases can be used with `blacksmith testbox run`, but Crabbox cannot sync or run them by id.",
].join("\n"),
);

View file

@ -190,17 +190,16 @@ run_remote_testbox_full_test_gate() {
' "$script_parent_dir" "${!name}" "$name") || return 2
[ -z "$value" ] || remote_env+=("$name=$value")
done
# Same Blacksmith Testbox delegation shape check:changed uses; the worktree's
# own wrapper syncs this prep tree (the canonical copy would sync the primary
# checkout instead).
# Explicit full-suite proof retains the measured high-memory allocation.
# The worktree wrapper syncs this prep tree, not the canonical checkout.
run_quiet_logged "$label" "$log_file" \
node scripts/crabbox-wrapper.mjs run \
--provider blacksmith-testbox \
--blacksmith-org openclaw \
--blacksmith-workflow .github/workflows/ci-check-testbox.yml \
--blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \
--blacksmith-job check \
--blacksmith-ref main \
--idle-timeout 90m \
--idle-timeout 15m \
--ttl 240m \
--timing-json \
--label "$lease_label" \

View file

@ -14,6 +14,7 @@ const WORKFLOWS: {
file: string;
prGroup: string;
manual: ManualPolicy;
manualAdmission?: string;
push?: { group: string; cancel: boolean };
convertToDraft?: true;
}[] = [
@ -21,16 +22,19 @@ const WORKFLOWS: {
file: ".github/workflows/ci-check-testbox.yml",
prGroup: "Blacksmith Testbox-pr-v1-123",
manual: { mode: "isolated per-run", group: "Blacksmith Testbox-manual-v1-201" },
manualAdmission: "admission",
},
{
file: ".github/workflows/ci-check-arm-testbox.yml",
prGroup: "Blacksmith ARM Testbox-pr-v1-123",
manual: { mode: "isolated per-run", group: "Blacksmith ARM Testbox-manual-v1-201" },
manualAdmission: "admission",
},
{
file: ".github/workflows/ci-build-artifacts-testbox.yml",
prGroup: "Blacksmith Build Artifacts Testbox-pr-v1-123",
manual: { mode: "isolated per-run", group: "Blacksmith Build Artifacts Testbox-manual-v1-201" },
manualAdmission: "admission",
},
{
file: ".github/workflows/ios-periphery.yml",
@ -435,7 +439,7 @@ describe.each(WORKFLOWS)("ancillary admission: $file", (policy) => {
expect(delayed.group).not.toBe(ready.group);
for (const [id, eligible] of Object.entries(ready.eligibility!.jobs)) {
if (id !== "scope") {
expect(eligible, id).toBe(true);
expect(eligible, id).toBe(id !== policy.manualAdmission);
}
}
},

View file

@ -0,0 +1,135 @@
import { readFileSync } from "node:fs";
import { describe, expect, it } from "vitest";
import { parse } from "yaml";
import {
assertFreshTestboxAdmission,
boundedTestboxIdleMinutes,
planTestboxAdmission,
} from "../../scripts/ci-testbox-budget.mjs";
const now = Date.parse("2026-10-01T12:00:00Z");
const request = {
profile: "check",
id: "tbx_example",
createdAt: "2026-10-01T11:55:00Z",
};
describe("Testbox spending admission", () => {
it("shares one bounded pool across profiles without changing a lease's slot", () => {
const groups = new Set<string>();
for (let index = 0; index < 256; index++) {
const lease = { ...request, id: `tbx_example_${index}` };
const group = planTestboxAdmission(lease, now).group;
expect(group).toMatch(/^openclaw-testbox-budget-v1-(?:[0-9]|[12][0-9]|3[01])$/);
for (const profile of ["arm", "build", "windows"]) {
expect(planTestboxAdmission({ ...lease, profile }, now).group).toBe(group);
}
groups.add(group);
}
expect(groups.size).toBe(32);
});
it("defaults routine proof to 16-class and requires the high-memory profile for 32-class", () => {
expect(planTestboxAdmission(request, now).runner).toBe("blacksmith-16vcpu-ubuntu-2404");
expect(() =>
planTestboxAdmission({ ...request, runner: "blacksmith-32vcpu-ubuntu-2404" }, now),
).toThrow(/not allowed/);
const groups = new Set<string>();
for (let index = 0; index < 128; index++) {
const plan = planTestboxAdmission(
{ ...request, profile: "check-memory", id: `tbx_memory_${index}` },
now,
);
expect(plan.runner).toBe("blacksmith-32vcpu-ubuntu-2404");
expect(plan.group).toMatch(/^openclaw-testbox-budget-v1-[0-3]$/);
groups.add(plan.group);
}
expect(groups.size).toBe(4);
});
it.each([0, -1, 241, 1.5, "invalid"])("rejects an unbounded runtime: %s", (minutes) => {
expect(() => planTestboxAdmission({ ...request, minutes }, now)).toThrow(/runtime/);
});
it("defaults routine proof to one hour and preserves explicit long-proof requests", () => {
const workflow = parse(readFileSync(".github/workflows/ci-check-testbox.yml", "utf8"));
const dispatchDefault = workflow.on.workflow_dispatch.inputs.timeout_minutes.default;
expect(planTestboxAdmission({ ...request, minutes: dispatchDefault }, now).minutes).toBe(60);
expect(planTestboxAdmission(request, now).minutes).toBe(60);
expect(planTestboxAdmission({ ...request, minutes: "" }, now).minutes).toBe(60);
expect(planTestboxAdmission({ ...request, minutes: 30 }, now).minutes).toBe(30);
expect(planTestboxAdmission({ ...request, profile: "check-memory" }, now).minutes).toBe(240);
expect(planTestboxAdmission({ ...request, minutes: 240 }, now).minutes).toBe(240);
expect(() => planTestboxAdmission({ ...request, profile: "build", minutes: 36 }, now)).toThrow(
/1 to 35/,
);
expect(() => planTestboxAdmission({ ...request, profile: "arm", minutes: 121 }, now)).toThrow(
/1 to 120/,
);
});
it("allows approved Windows sizes and refuses arbitrary dispatch labels", () => {
for (const size of [8, 16]) {
const runner = `blacksmith-${size}vcpu-windows-2025`;
expect(planTestboxAdmission({ ...request, profile: "windows", runner }, now).runner).toBe(
runner,
);
}
for (const runner of ["self-hosted", "blacksmith-32vcpu-windows-2025"]) {
expect(() => planTestboxAdmission({ ...request, profile: "windows", runner }, now)).toThrow(
/not allowed/,
);
}
});
it("expires abandoned queue entries before hydration, including the deadline itself", () => {
const plan = planTestboxAdmission(request, now);
expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at - 1)).not.toThrow();
expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at)).toThrow(/expired/);
expect(() => planTestboxAdmission(request, plan.expires_at)).toThrow(/expired/);
expect(() => planTestboxAdmission({ ...request, createdAt: "unknown" }, now)).toThrow(
/invalid/,
);
});
it("caps idle requests while retaining shorter provider deadlines", () => {
expect(boundedTestboxIdleMinutes("90\n")).toBe(15);
expect(boundedTestboxIdleMinutes("5\n")).toBe(5);
expect(() => boundedTestboxIdleMinutes("0")).toThrow(/invalid/);
});
it.each([
["ci-check-testbox.yml", "check", "check"],
["ci-check-high-memory-testbox.yml", "check", "check-memory"],
["ci-check-arm-testbox.yml", "check-arm", "arm"],
["ci-build-artifacts-testbox.yml", "build-artifacts", "build"],
["windows-blacksmith-testbox.yml", "windows", "windows"],
])("enforces admission before allocating %s", (file, jobName, profile) => {
const workflow = parse(readFileSync(`.github/workflows/${file}`, "utf8"));
const admission = workflow.jobs.admission;
const job = workflow.jobs[jobName];
expect(admission["runs-on"]).toBe("ubuntu-24.04");
const budget = admission.steps.find((step: { id?: string }) => step.id === "budget");
expect(budget.run).toBe("node scripts/ci-testbox-budget.mjs admit");
expect(budget.env.TESTBOX_PROFILE).toBe(profile);
expect(job.needs).toBe("admission");
expect(job.if).toContain("needs.admission.result == 'success'");
expect(job["runs-on"]).toContain("needs.admission.outputs.runner");
expect(job.concurrency.group).toContain("needs.admission.outputs.group");
expect(job.concurrency.queue).toBeUndefined();
expect(job.concurrency["cancel-in-progress"]).toBe(false);
const names = job.steps.map((step: { name: string }) => step.name);
expect(names.indexOf("Reject expired Testbox admission")).toBeGreaterThan(
names.indexOf("Begin Testbox"),
);
expect(names.indexOf("Reject expired Testbox admission")).toBeLessThan(
names.indexOf("Checkout"),
);
if (profile !== "windows") {
expect(names.indexOf("Bound Testbox idle lifetime")).toBeLessThan(
names.indexOf("Setup Node environment"),
);
expect(names).toContain("Close Testbox SSH sessions");
}
});
});

View file

@ -873,9 +873,26 @@ AFTER_CD
]);
expect(workflow.on.pull_request.paths).toContain(workflowPath);
expect(workflow.on.pull_request.paths).not.toContain(".github/workflows/**");
expect(workflow.jobs[jobName].if).toBe(
"${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}",
);
for (const [eventName, draft, result, cancelled, admitted] of [
["pull_request", false, "skipped", false, true],
["pull_request", true, "skipped", false, false],
["pull_request", false, "skipped", true, false],
["workflow_dispatch", false, "success", false, true],
["workflow_dispatch", false, "failure", false, false],
["workflow_dispatch", false, "success", true, false],
] as const) {
expect(
evaluateWorkflowExpression(workflow.jobs[jobName].if, {
eventName,
draft,
cancelled,
additionalNeeds: { admission: { outputs: {}, result } },
repository: "openclaw/openclaw",
runAttempt: 1,
}),
`${workflowPath}: ${eventName}, admission=${result}, cancelled=${cancelled}`,
).toBe(admitted);
}
}
});

View file

@ -5494,9 +5494,21 @@ describe("ci workflow guards", () => {
// PR events validate the artifact build on hosted runners (landing gate
// stays satisfiable during Blacksmith outages); Testbox leases are
// dispatch-only, mirroring ci-check-testbox.yml.
expect(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"]).toBe(
"${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }}",
);
for (const [eventName, expected] of [
["pull_request", "ubuntu-24.04"],
["workflow_dispatch", "blacksmith-16vcpu-ubuntu-2404"],
] as const) {
expect(
evaluateWorkflowExpression(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"], {
eventName,
repository: "openclaw/openclaw",
runAttempt: 1,
additionalNeeds: {
admission: { outputs: { runner: "blacksmith-16vcpu-ubuntu-2404" }, result: "success" },
},
}),
).toBe(expected);
}
for (const stepName of ["Begin Testbox", "Run Testbox"]) {
expect(
buildArtifactsTestbox.jobs["build-artifacts"].steps.find(

View file

@ -11226,9 +11226,19 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
expect(setupNodeWith).not.toHaveProperty("dependency-cache");
expect(setupNodeWith).not.toHaveProperty("sticky-disk");
expect(setupNodeWith["cache-mode"]).toBe("restore");
expect(checkTestboxJob["timeout-minutes"]).toBe(
"${{ fromJSON(inputs.timeout_minutes || '240') }}",
);
for (const [minutes, expected] of [
["45", 45],
["", 60],
] as const) {
expect(
evaluateWorkflowExpression(checkTestboxJob["timeout-minutes"], {
eventName: "workflow_dispatch",
repository: "openclaw/openclaw",
runAttempt: 1,
additionalNeeds: { admission: { outputs: { minutes }, result: "success" } },
}),
).toBe(expected);
}
for (const step of [runTestboxStep, runArmTestboxStep, runBuildArtifactsTestboxStep]) {
expect(step.uses).toBe(RUN_TESTBOX_WITH_FAILURE_REPORTING);
}
@ -11236,8 +11246,27 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
"useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d",
);
expect(windowsTestboxActionMarker.if).toBe("${{ false }}");
expect(runTestboxStep.if).toBe("github.event_name == 'workflow_dispatch' && always()");
expect(closeTestboxSshStep.if).toBe("github.event_name == 'workflow_dispatch' && always()");
for (const step of [
runTestboxStep,
runArmTestboxStep,
runBuildArtifactsTestboxStep,
closeTestboxSshStep,
]) {
for (const [eventName, expected] of [
["workflow_dispatch", true],
["pull_request", false],
] as const) {
expect(
evaluateWorkflowExpression(`\${{ ${step.if} }}`, {
eventName,
repository: "openclaw/openclaw",
runAttempt: 1,
failed: true,
cancelled: true,
}),
).toBe(expected);
}
}
expect(closeTestboxSshStep.run).toContain(
`sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }'`,
);
@ -11247,10 +11276,6 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
expect(checkTestboxSteps.indexOf(closeTestboxSshStep)).toBe(
checkTestboxSteps.indexOf(runTestboxStep) + 1,
);
expect(runArmTestboxStep.if).toBe("always()");
expect(runBuildArtifactsTestboxStep.if).toBe(
"github.event_name == 'workflow_dispatch' && always()",
);
expect(runWindowsTestboxStep.if).toBe("always()");
expect(runWindowsTestboxStep.env?.JOB_STATUS).toBe("${{ job.status }}");
expect(runWindowsTestboxStep.env?.NATIVE_SSH_USER).toBe(

View file

@ -518,13 +518,13 @@ describe("remote testbox gate delegation", () => {
"--blacksmith-org",
"openclaw",
"--blacksmith-workflow",
".github/workflows/ci-check-testbox.yml",
".github/workflows/ci-check-high-memory-testbox.yml",
"--blacksmith-job",
"check",
"--blacksmith-ref",
"main",
"--idle-timeout",
"90m",
"15m",
"--ttl",
"240m",
"--timing-json",

View file

@ -70,7 +70,7 @@ function runBasePreparation(
eventName: string,
) {
const workflow = parse(fs.readFileSync(workflowName, "utf8"));
const job = Object.values(workflow.jobs)[0] as { steps: Step[] };
const jobs = Object.values(workflow.jobs) as { steps: Step[] }[];
const values = new Map([
["github.event.pull_request.base.sha", base],
["github.event.pull_request.base.ref", "main"],
@ -99,7 +99,9 @@ function runBasePreparation(
'#!/bin/sh\nif [ "$1" = tee ]; then cat >/dev/null; fi\n',
);
fs.chmodSync(path.join(bin, "sudo"), 0o755);
const step = job.steps.find((entry) => entry.name === "Prepare Testbox shell");
const step = jobs
.flatMap((job) => job.steps)
.find((entry) => entry.uses === "./.github/actions/prepare-testbox-shell");
if (!step?.uses) {
throw new Error("Missing Testbox preparation action");
}