mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 01:29:56 +00:00
fix(ci): cap Testbox concurrency and idle spend (#162678)
* fix(ci): cap Testbox concurrency and idle spend * test(ci): align workflow guards with Testbox admission * fix(ci): declare optional Testbox admission inputs * fix(ci): narrow Testbox CLI rejection errors * test(ci): verify admitted Testbox workflow behavior * fix(ci): reserve large Testboxes for memory-heavy proof * fix(ci): default routine Testboxes to one hour * docs(ci): clarify Testbox profiles and total job deadlines Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
parent
4359fb6ca9
commit
32dc6c861d
22 changed files with 948 additions and 53 deletions
|
|
@ -61,6 +61,14 @@ Source trust determines which providers are allowed. It does not select one.
|
|||
- Never untrusted code on credential-hydrated Testbox.
|
||||
- Never run untrusted repo wrapper/config locally.
|
||||
- No speculative warmup. Acquire when first heavy command ready. Reuse id. Stop.
|
||||
- Use the consumer's smallest proven remote profile. Large/high-memory profiles
|
||||
are explicit exceptions for a named command with measured memory need or a
|
||||
controlled total-cost benefit. Record the reason before allocation; a generic
|
||||
failure, queue delay, or timeout is not a reason to upsize. Do not raise worker
|
||||
counts to compensate for a smaller machine.
|
||||
- For delegated Testbox sizing, select a documented workflow through
|
||||
`--blacksmith-workflow`; direct-provider `--class`/`--type` flags do not choose
|
||||
the Testbox runner. Change profiles with a fresh lease.
|
||||
|
||||
Test size, expected duration, and hydration failure do not authorize a provider
|
||||
override. Omit `--provider` for normal work. Add it only when the user requests
|
||||
|
|
|
|||
|
|
@ -29,6 +29,27 @@ availability, Blacksmith control-plane health, and downstream queue drains.
|
|||
scans should stay on GitHub-hosted runners unless measured evidence says
|
||||
Blacksmith is required.
|
||||
|
||||
## Runner Cost Policy
|
||||
|
||||
- Use the smallest runner that completes the required workload reliably. Keep
|
||||
short control jobs hosted and ordinary trusted development proof local.
|
||||
- Treat the 32-class as an exception, not a default or generic retry. Record the
|
||||
command and measured peak memory, a smaller-runner OOM, or a controlled
|
||||
comparison showing lower total billed cost. Low CPU use alone does not prove
|
||||
a memory-heavy job can move down. Preserve resource-based worker limits.
|
||||
- Routine OpenClaw Testboxes use the 16-class with a 60-minute total-job
|
||||
default, including hydration. Keep the 15-minute idle ceiling. Shorter
|
||||
deadlines are welcome for known short commands; do not request four hours
|
||||
automatically or upsize only to obtain more time. Select
|
||||
`.github/workflows/ci-check-high-memory-testbox.yml` explicitly only for a
|
||||
named memory-heavy command; see `docs/reference/test/remote-proof.md`.
|
||||
That workflow has at most four concurrent leases inside the shared 32-slot
|
||||
Testbox pool. All Testbox profiles cap idle time at 15 minutes.
|
||||
- Do not promote an entire workflow family because one command needs more RAM.
|
||||
Keep proven high-memory CI rows scoped to their owning planner and evidence;
|
||||
remeasure before changing their allocation. A 32-class label is not proof of
|
||||
32 available CPUs. Compare observed resources and total billed job cost.
|
||||
|
||||
## Rejected Experiments
|
||||
|
||||
- **Boundary asynchronous input preparation (2026-09-26):** Adding the existing
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ actions:
|
|||
ephemeral: true
|
||||
blacksmith:
|
||||
org: openclaw
|
||||
# Routine remote proof uses 16-class; high-memory proof opts in by workflow.
|
||||
workflow: .github/workflows/ci-check-testbox.yml
|
||||
job: check
|
||||
ref: main
|
||||
|
|
@ -126,7 +127,7 @@ jobs:
|
|||
testbox-changed:
|
||||
provider: blacksmith-testbox
|
||||
target: linux
|
||||
idleTimeout: 90m
|
||||
idleTimeout: 15m
|
||||
hydrate:
|
||||
actions: false
|
||||
actions:
|
||||
|
|
|
|||
76
.github/workflows/ci-build-artifacts-testbox.yml
vendored
76
.github/workflows/ci-build-artifacts-testbox.yml
vendored
|
|
@ -10,6 +10,7 @@ on:
|
|||
pull_request:
|
||||
types: [opened, reopened, synchronize, ready_for_review]
|
||||
paths:
|
||||
- "scripts/ci-testbox-budget.mjs"
|
||||
- ".github/workflows/ci-build-artifacts-testbox.yml"
|
||||
- ".github/actions/prepare-testbox-shell/**"
|
||||
- ".github/actions/setup-node-env/**"
|
||||
|
|
@ -62,16 +63,47 @@ env:
|
|||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
group: ${{ steps.budget.outputs.group }}
|
||||
runner: ${{ steps.budget.outputs.runner }}
|
||||
minutes: ${{ steps.budget.outputs.minutes }}
|
||||
expires_at: ${{ steps.budget.outputs.expires_at }}
|
||||
steps:
|
||||
- name: Checkout admission policy
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: scripts/ci-testbox-budget.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
- name: Admit Testbox within the shared budget
|
||||
id: budget
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TESTBOX_PROFILE: build
|
||||
TESTBOX_ID: ${{ inputs.testbox_id }}
|
||||
run: node scripts/ci-testbox-budget.mjs admit
|
||||
|
||||
build-artifacts:
|
||||
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
|
||||
needs: admission
|
||||
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: read
|
||||
name: "build-artifacts"
|
||||
# Pull requests only validate the artifact build; real Testbox leases
|
||||
# arrive via dispatch. Hosted PR runs keep this landing gate satisfiable
|
||||
# during Blacksmith outages (mirrors ci-check-testbox.yml).
|
||||
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }}
|
||||
timeout-minutes: 35
|
||||
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }}
|
||||
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '35') }}
|
||||
steps:
|
||||
# Testbox lifecycle actions require Blacksmith VM metadata; PRs validate the build only.
|
||||
- name: Begin Testbox
|
||||
|
|
@ -80,12 +112,30 @@ jobs:
|
|||
with:
|
||||
testbox_id: ${{ inputs.testbox_id }}
|
||||
|
||||
- name: Reject expired Testbox admission
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: |
|
||||
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
|
||||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
|
||||
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }}
|
||||
persist-credentials: false
|
||||
|
||||
- name: Bound Testbox idle lifetime
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: node scripts/ci-testbox-budget.mjs configure
|
||||
|
||||
- name: Setup Node environment
|
||||
id: setup-node-env
|
||||
uses: ./.github/actions/setup-node-env
|
||||
|
|
@ -257,3 +307,23 @@ jobs:
|
|||
if: github.event_name == 'workflow_dispatch' && always()
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
- name: Close Testbox SSH sessions
|
||||
if: github.event_name == 'workflow_dispatch' && always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Testbox state stores Blacksmith's external forwarded port. Resolve
|
||||
# sshd's VM-local listener because that is the sport visible to ss.
|
||||
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
|
||||
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
|
||||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
|
||||
echo "No valid local SSH listener port found; skipping session cleanup"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# run-testbox has no post hook. Close only Testbox client sockets so
|
||||
# Blacksmith's runner teardown does not wait for its 290-second grace.
|
||||
timeout --signal=KILL 5s sudo ss -K state established \
|
||||
"( sport = :${runner_ssh_local_port} )" || true
|
||||
|
|
|
|||
79
.github/workflows/ci-check-arm-testbox.yml
vendored
79
.github/workflows/ci-check-arm-testbox.yml
vendored
|
|
@ -9,6 +9,7 @@ on:
|
|||
pull_request:
|
||||
types: [opened, reopened, synchronize, ready_for_review]
|
||||
paths:
|
||||
- "scripts/ci-testbox-budget.mjs"
|
||||
- ".github/workflows/ci-check-arm-testbox.yml"
|
||||
- ".github/actions/prepare-testbox-shell/**"
|
||||
- ".github/actions/setup-node-env/**"
|
||||
|
|
@ -56,18 +57,62 @@ env:
|
|||
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
group: ${{ steps.budget.outputs.group }}
|
||||
runner: ${{ steps.budget.outputs.runner }}
|
||||
minutes: ${{ steps.budget.outputs.minutes }}
|
||||
expires_at: ${{ steps.budget.outputs.expires_at }}
|
||||
steps:
|
||||
- name: Checkout admission policy
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: scripts/ci-testbox-budget.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
- name: Admit Testbox within the shared budget
|
||||
id: budget
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TESTBOX_PROFILE: arm
|
||||
TESTBOX_ID: ${{ inputs.testbox_id }}
|
||||
run: node scripts/ci-testbox-budget.mjs admit
|
||||
|
||||
check-arm:
|
||||
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
|
||||
needs: admission
|
||||
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: read
|
||||
name: "check-arm"
|
||||
runs-on: blacksmith-16vcpu-ubuntu-2404-arm
|
||||
timeout-minutes: 120
|
||||
runs-on: ${{ github.event_name == 'pull_request' && 'blacksmith-16vcpu-ubuntu-2404-arm' || needs.admission.outputs.runner }}
|
||||
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '120') }}
|
||||
steps:
|
||||
- name: Begin Testbox
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043 # v2
|
||||
with:
|
||||
testbox_id: ${{ inputs.testbox_id }}
|
||||
- name: Reject expired Testbox admission
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: |
|
||||
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
|
||||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
|
||||
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify ARM runner
|
||||
shell: bash
|
||||
run: |
|
||||
|
|
@ -88,6 +133,12 @@ jobs:
|
|||
with:
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '0' }}
|
||||
persist-credentials: false
|
||||
- name: Bound Testbox idle lifetime
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: node scripts/ci-testbox-budget.mjs configure
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
|
|
@ -155,6 +206,26 @@ jobs:
|
|||
# Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15
|
||||
# Return to useblacksmith/run-testbox after the fix lands upstream.
|
||||
uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4
|
||||
if: always()
|
||||
if: github.event_name == 'workflow_dispatch' && always()
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
- name: Close Testbox SSH sessions
|
||||
if: github.event_name == 'workflow_dispatch' && always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Testbox state stores Blacksmith's external forwarded port. Resolve
|
||||
# sshd's VM-local listener because that is the sport visible to ss.
|
||||
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
|
||||
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
|
||||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
|
||||
echo "No valid local SSH listener port found; skipping session cleanup"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# run-testbox has no post hook. Close only Testbox client sockets so
|
||||
# Blacksmith's runner teardown does not wait for its 290-second grace.
|
||||
timeout --signal=KILL 5s sudo ss -K state established \
|
||||
"( sport = :${runner_ssh_local_port} )" || true
|
||||
|
|
|
|||
183
.github/workflows/ci-check-high-memory-testbox.yml
vendored
Normal file
183
.github/workflows/ci-check-high-memory-testbox.yml
vendored
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
name: Blacksmith High-Memory Testbox
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
testbox_id:
|
||||
type: string
|
||||
description: "Testbox session ID"
|
||||
required: true
|
||||
timeout_minutes:
|
||||
type: number
|
||||
description: "Maximum GitHub job runtime for long Testbox commands"
|
||||
default: 240
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Explicit memory-heavy exception; ordinary proof uses ci-check-testbox.yml.
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-manual-v1-${{ github.run_id }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
group: ${{ steps.budget.outputs.group }}
|
||||
runner: ${{ steps.budget.outputs.runner }}
|
||||
minutes: ${{ steps.budget.outputs.minutes }}
|
||||
expires_at: ${{ steps.budget.outputs.expires_at }}
|
||||
steps:
|
||||
- name: Checkout admission policy
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: scripts/ci-testbox-budget.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
- name: Admit Testbox within the shared budget
|
||||
id: budget
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TESTBOX_PROFILE: check-memory
|
||||
TESTBOX_ID: ${{ inputs.testbox_id }}
|
||||
TESTBOX_MINUTES: ${{ inputs.timeout_minutes }}
|
||||
run: node scripts/ci-testbox-budget.mjs admit
|
||||
|
||||
check:
|
||||
needs: admission
|
||||
if: ${{ !cancelled() && needs.admission.result == 'success' }}
|
||||
concurrency:
|
||||
group: ${{ needs.admission.outputs.group }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: read
|
||||
name: "check"
|
||||
runs-on: ${{ needs.admission.outputs.runner }}
|
||||
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '240') }}
|
||||
steps:
|
||||
- name: Begin Testbox
|
||||
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043
|
||||
with:
|
||||
testbox_id: ${{ inputs.testbox_id }}
|
||||
- name: Reject expired Testbox admission
|
||||
shell: bash
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: |
|
||||
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
|
||||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
|
||||
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
- name: Bound Testbox idle lifetime
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: node scripts/ci-testbox-budget.mjs configure
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
semantic-checks: "true"
|
||||
cache-mode: restore
|
||||
install-bun: "false"
|
||||
# Testbox hydration uses the ordinary pnpm store cache. Canonical CI
|
||||
# owns the exact dependency archive and never delegates here.
|
||||
- name: Prepare Testbox shell
|
||||
uses: ./.github/actions/prepare-testbox-shell
|
||||
with:
|
||||
base-ref: HEAD
|
||||
go-version: "1.27.1"
|
||||
|
||||
- name: Hydrate Testbox provider env helper
|
||||
shell: bash
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }}
|
||||
ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }}
|
||||
BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }}
|
||||
CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }}
|
||||
DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }}
|
||||
DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
|
||||
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
|
||||
FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}
|
||||
FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
|
||||
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
|
||||
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
|
||||
GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
|
||||
KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }}
|
||||
MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }}
|
||||
MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }}
|
||||
MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
|
||||
MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }}
|
||||
OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }}
|
||||
OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }}
|
||||
OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }}
|
||||
OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }}
|
||||
OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }}
|
||||
OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }}
|
||||
OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }}
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }}
|
||||
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
|
||||
QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }}
|
||||
FAL_KEY: ${{ secrets.FAL_KEY }}
|
||||
RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }}
|
||||
DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
|
||||
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
||||
VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }}
|
||||
XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
|
||||
ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }}
|
||||
Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }}
|
||||
BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }}
|
||||
BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }}
|
||||
CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }}
|
||||
OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }}
|
||||
OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }}
|
||||
OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }}
|
||||
OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }}
|
||||
OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }}
|
||||
OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }}
|
||||
OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }}
|
||||
run: bash scripts/ci-hydrate-testbox-env.sh
|
||||
|
||||
- name: Run Testbox
|
||||
# Temporary local-listener fix: https://github.com/useblacksmith/run-testbox/pull/15
|
||||
# Return to useblacksmith/run-testbox after the fix lands upstream.
|
||||
uses: steipete/run-testbox@2b6b1be536ec7f3c73757fedf5460a27ab4856b4
|
||||
if: always()
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
- name: Close Testbox SSH sessions
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Testbox state stores Blacksmith's external forwarded port. Resolve
|
||||
# sshd's VM-local listener because that is the sport visible to ss.
|
||||
runner_ssh_local_port="$(sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }')"
|
||||
if [[ ! "$runner_ssh_local_port" =~ ^[0-9]+$ ]] ||
|
||||
(( runner_ssh_local_port < 1 || runner_ssh_local_port > 65535 )); then
|
||||
echo "No valid local SSH listener port found; skipping session cleanup"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# run-testbox has no post hook. Close only Testbox client sockets so
|
||||
# Blacksmith's runner teardown does not wait for its 290-second grace.
|
||||
timeout --signal=KILL 5s sudo ss -K state established \
|
||||
"( sport = :${runner_ssh_local_port} )" || true
|
||||
61
.github/workflows/ci-check-testbox.yml
vendored
61
.github/workflows/ci-check-testbox.yml
vendored
|
|
@ -8,11 +8,12 @@ on:
|
|||
required: true
|
||||
timeout_minutes:
|
||||
type: number
|
||||
description: "Maximum GitHub job runtime for long Testbox commands"
|
||||
default: 240
|
||||
description: "GitHub job runtime; explicit long-proof requests may use up to 240 minutes"
|
||||
default: 60
|
||||
pull_request:
|
||||
types: [opened, reopened, synchronize, ready_for_review]
|
||||
paths:
|
||||
- "scripts/ci-testbox-budget.mjs"
|
||||
- ".github/workflows/ci-check-testbox.yml"
|
||||
- ".github/actions/prepare-testbox-shell/**"
|
||||
- ".github/actions/setup-node-env/**"
|
||||
|
|
@ -59,14 +60,46 @@ env:
|
|||
PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false"
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
group: ${{ steps.budget.outputs.group }}
|
||||
runner: ${{ steps.budget.outputs.runner }}
|
||||
minutes: ${{ steps.budget.outputs.minutes }}
|
||||
expires_at: ${{ steps.budget.outputs.expires_at }}
|
||||
steps:
|
||||
- name: Checkout admission policy
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: scripts/ci-testbox-budget.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
- name: Admit Testbox within the shared budget
|
||||
id: budget
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TESTBOX_PROFILE: check
|
||||
TESTBOX_ID: ${{ inputs.testbox_id }}
|
||||
TESTBOX_MINUTES: ${{ inputs.timeout_minutes }}
|
||||
run: node scripts/ci-testbox-budget.mjs admit
|
||||
|
||||
check:
|
||||
if: ${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}
|
||||
needs: admission
|
||||
if: ${{ !cancelled() && ((github.event_name == 'pull_request' && !github.event.pull_request.draft) || needs.admission.result == 'success') }}
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: read
|
||||
name: "check"
|
||||
# Pull requests only validate the hydration steps; real Testbox leases arrive via dispatch.
|
||||
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }}
|
||||
timeout-minutes: ${{ fromJSON(inputs.timeout_minutes || '240') }}
|
||||
runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || needs.admission.outputs.runner }}
|
||||
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes || '60') }}
|
||||
steps:
|
||||
# Testbox lifecycle actions require Blacksmith VM metadata; PRs validate our setup steps only.
|
||||
- name: Begin Testbox
|
||||
|
|
@ -74,6 +107,18 @@ jobs:
|
|||
uses: useblacksmith/begin-testbox@233448af4bfdc6fca509a7f0974411ac6d8a8043
|
||||
with:
|
||||
testbox_id: ${{ inputs.testbox_id }}
|
||||
- name: Reject expired Testbox admission
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: |
|
||||
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
|
||||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
|
||||
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
|
|
@ -81,6 +126,12 @@ jobs:
|
|||
# reconstructs its exact base and final tree. PR validation keeps both commits.
|
||||
fetch-depth: ${{ github.event_name == 'pull_request' && '2' || '1' }}
|
||||
persist-credentials: false
|
||||
- name: Bound Testbox idle lifetime
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: node scripts/ci-testbox-budget.mjs configure
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
|
|
|
|||
59
.github/workflows/windows-blacksmith-testbox.yml
vendored
59
.github/workflows/windows-blacksmith-testbox.yml
vendored
|
|
@ -8,7 +8,10 @@ on:
|
|||
description: "Testbox session ID"
|
||||
required: true
|
||||
runner_label:
|
||||
type: string
|
||||
type: choice
|
||||
options:
|
||||
- blacksmith-8vcpu-windows-2025
|
||||
- blacksmith-16vcpu-windows-2025
|
||||
description: "Windows runner label"
|
||||
required: false
|
||||
default: "blacksmith-16vcpu-windows-2025"
|
||||
|
|
@ -20,10 +23,43 @@ env:
|
|||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 3
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
group: ${{ steps.budget.outputs.group }}
|
||||
runner: ${{ steps.budget.outputs.runner }}
|
||||
minutes: ${{ steps.budget.outputs.minutes }}
|
||||
expires_at: ${{ steps.budget.outputs.expires_at }}
|
||||
steps:
|
||||
- name: Checkout admission policy
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
sparse-checkout: scripts/ci-testbox-budget.mjs
|
||||
sparse-checkout-cone-mode: false
|
||||
persist-credentials: false
|
||||
- name: Admit Testbox within the shared budget
|
||||
id: budget
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TESTBOX_PROFILE: windows
|
||||
TESTBOX_ID: ${{ inputs.testbox_id }}
|
||||
TESTBOX_RUNNER: ${{ inputs.runner_label }}
|
||||
run: node scripts/ci-testbox-budget.mjs admit
|
||||
|
||||
windows:
|
||||
needs: admission
|
||||
if: ${{ !cancelled() && needs.admission.result == 'success' }}
|
||||
concurrency:
|
||||
group: ${{ github.event_name == 'workflow_dispatch' && needs.admission.outputs.group || format('testbox-validation-{0}-{1}', github.workflow, github.run_id) }}
|
||||
cancel-in-progress: false
|
||||
name: windows
|
||||
runs-on: ${{ inputs.runner_label }}
|
||||
timeout-minutes: 75
|
||||
runs-on: ${{ needs.admission.outputs.runner }}
|
||||
timeout-minutes: ${{ fromJSON(needs.admission.outputs.minutes) }}
|
||||
defaults:
|
||||
run:
|
||||
shell: pwsh
|
||||
|
|
@ -138,6 +174,18 @@ jobs:
|
|||
printf 'public_key_path=%s\n' "$public_key_path" >> "$GITHUB_OUTPUT"
|
||||
printf 'bash_os=%s\nbash_user=%s\n' "$(uname -s)" "$(id -un)"
|
||||
|
||||
- name: Reject expired Testbox admission
|
||||
if: github.event_name == 'workflow_dispatch'
|
||||
shell: bash
|
||||
env:
|
||||
TESTBOX_EXPIRES_AT: ${{ needs.admission.outputs.expires_at }}
|
||||
run: |
|
||||
if [[ ! "$TESTBOX_EXPIRES_AT" =~ ^[0-9]+$ ]] ||
|
||||
(( $(date +%s) >= TESTBOX_EXPIRES_AT / 1000 )); then
|
||||
echo "Testbox waited more than 10 minutes; request a fresh lease." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
|
|
@ -217,6 +265,11 @@ jobs:
|
|||
auth_token="$(cat "$state/auth_token")"
|
||||
idle_timeout="$(cat "$state/idle_timeout" 2>/dev/null || true)"
|
||||
idle_timeout="${idle_timeout:-10}"
|
||||
if [[ ! "$idle_timeout" =~ ^[0-9]+$ ]] || (( idle_timeout < 1 )); then
|
||||
echo "Invalid Testbox idle timeout" >&2
|
||||
exit 1
|
||||
fi
|
||||
if (( idle_timeout > 15 )); then idle_timeout=15; fi
|
||||
api_url="$(cat "$state/api_url")"
|
||||
runner_host="$(cat "$state/runner_host")"
|
||||
runner_ssh_port="$(cat "$state/runner_ssh_port")"
|
||||
|
|
|
|||
|
|
@ -62,7 +62,7 @@ file, so large PRs do not lose test-planning inputs to Actions output or environ
|
|||
size limits. Frozen targets that predate this transport retain their bounded JSON
|
||||
output contract. Missing or invalid inputs still reject current PR Node planning.
|
||||
|
||||
The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 32-class for dispatched proof and defaults to a four-hour outer job budget. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged.
|
||||
The [Testbox check workflow](/ci/local-proof#testbox-validation) requests the Blacksmith 16-class for routine dispatched proof, with a 60-minute total-job deadline including hydration. The explicit high-memory 32-class workflow retains 240 minutes for memory-heavy full-suite gates. The outer GitHub deadline can terminate active SSH commands; the separate 15-minute idle limit does not extend it. PR hydration checks stay on hosted Ubuntu; individual test deadlines remain unchanged.
|
||||
|
||||
Full GitHub and hybrid type checks run the five core stripes independently, retaining two compiler children per job. The last four rows then each run one root-test partition serially, leaving extension tests and scripts in the central row. Narrow plans reuse four already-selected rows when available; smaller selections retain central root checking. This adds no jobs or compiler overlap. Current hybrid full runs use three hosted extension-lint jobs; targeted layouts retain six stripe identities. Trusted hybrid first attempts place both packed core-lint rows on the Blacksmith 16-class and the final gate on the 4-class to avoid serial hosted assignment delays. Frozen targets keep their earlier layout; see [static checks](/ci/runners#runner-backend-modes).
|
||||
|
||||
|
|
|
|||
|
|
@ -369,15 +369,26 @@ Provider readiness and broker authentication still determine
|
|||
which configured backend can run the proof.
|
||||
The check workflow hydrates its pinned dispatch commit with a depth-1 checkout;
|
||||
the changed gate later reconstructs the exact merge base and synced final tree.
|
||||
Dispatched check leases request `blacksmith-32vcpu-ubuntu-2404`. A native capacity
|
||||
probe measured eight CPUs and 30.95 GiB of memory on that class, compared with
|
||||
15.42 GiB on the previous 16-class. This supplies headroom for isolated runtime
|
||||
validation without increasing the number of jobs or workers. Workloads still
|
||||
admit work from observed resources; the runner label is not a capacity guarantee.
|
||||
PR hydration checks remain on `ubuntu-24.04`.
|
||||
Its outer GitHub job defaults to 240 minutes. Manual dispatches can override
|
||||
`timeout_minutes`. Testbox idle timeouts and individual test deadlines remain
|
||||
separate limits.
|
||||
Routine dispatched check leases request `blacksmith-16vcpu-ubuntu-2404` through
|
||||
`ci-check-testbox.yml`, with a 60-minute total GitHub job deadline including
|
||||
hydration. The explicit `ci-check-high-memory-testbox.yml` workflow requests
|
||||
`blacksmith-32vcpu-ubuntu-2404` and retains 240 minutes for memory-heavy full-suite
|
||||
gates. Select it only for a justified memory need, not merely for more time; see
|
||||
[Testbox runner sizing](/reference/test/remote-proof#testbox-runner-sizing).
|
||||
A native capacity probe measured eight CPUs and 30.95 GiB of memory on the
|
||||
32-class, compared with 15.42 GiB on the 16-class. This supplies headroom for
|
||||
isolated runtime validation without increasing the number of jobs or workers.
|
||||
Workloads still admit work from observed resources; the runner label is not a
|
||||
capacity guarantee. PR hydration checks remain on `ubuntu-24.04`.
|
||||
|
||||
The outer GitHub deadline can terminate active SSH commands. Both profiles have
|
||||
a separate 15-minute idle limit; active SSH prevents idle expiry, not the outer
|
||||
job deadline. Individual test deadlines also remain separate limits. The standard
|
||||
workflow accepts an explicit `timeout_minutes` input up to 240 minutes, but
|
||||
managed Crabbox 0.69.0 does not forward arbitrary workflow inputs, including
|
||||
`timeout_minutes`, and `--ttl` does not extend a Testbox job. Plan routine proof
|
||||
within its total-job budget rather than treating TTL or a larger runner as a
|
||||
deadline override.
|
||||
Sanitized AWS runs set `CRABBOX_ENV_ALLOW=CI`, pass
|
||||
`--no-hydrate`, and use a fresh temporary remote `HOME`; this prevents the repo
|
||||
`OPENCLAW_*` allowlist and existing auth profiles from reaching untrusted code.
|
||||
|
|
|
|||
|
|
@ -8,6 +8,79 @@ read_when:
|
|||
|
||||
## Runners
|
||||
|
||||
### Testbox spending limits
|
||||
|
||||
The general, high-memory, ARM, build-artifact, and Windows Testbox workflows share 32
|
||||
concurrency slots. A GitHub-hosted admission job validates the lease ID, runner,
|
||||
and runtime before the Blacksmith job becomes eligible. Dispatches wait in their
|
||||
assigned slot without canceling an active lease. Each slot retains one pending
|
||||
request; a newer request can cancel and replace that pending request. Hash
|
||||
collisions can leave other slots unused; this is a ceiling, not a promise of
|
||||
32 busy machines.
|
||||
|
||||
Admission expires ten minutes after the workflow was created. A request that
|
||||
waits longer fails before checkout or hydration when its runner starts; it can
|
||||
still incur runner startup cost. This does not remove the queued job immediately.
|
||||
Stop an abandoned lease by its exact ID instead of leaving a warmup pending.
|
||||
Do not retry in a loop when the pool is full.
|
||||
|
||||
Idle requests are capped at 15 minutes. The existing Testbox monitor continues
|
||||
to protect active SSH commands. Stop retained leases when their task finishes;
|
||||
the idle limit is not a substitute for caller cleanup.
|
||||
|
||||
| Testbox | Allowed runner | Default runtime | Maximum requested runtime |
|
||||
| --------------------- | --------------------- | --------------- | ------------------------- |
|
||||
| General runtime proof | 16-vCPU Ubuntu x64 | 60 minutes | 240 minutes |
|
||||
| High-memory exception | 32-vCPU Ubuntu x64 | 240 minutes | 240 minutes |
|
||||
| ARM proof | 16-vCPU Ubuntu ARM | 120 minutes | 120 minutes |
|
||||
| Build artifacts | 16-vCPU Ubuntu x64 | 35 minutes | 35 minutes |
|
||||
| Windows | 8- or 16-vCPU Windows | 75 minutes | 75 minutes |
|
||||
|
||||
Routine remote proof uses the 16-class. The explicit high-memory workflow can
|
||||
use only four of the shared 32 slots; it does not add four more slots. Normal
|
||||
leases can also occupy those slots, so hash collisions may queue high-memory
|
||||
work below its ceiling. Stop owned leases when done rather than requesting new
|
||||
IDs to evade a busy slot.
|
||||
|
||||
Use the 32-class only for a named command with measured memory need, a verified
|
||||
smaller-runner OOM, or a controlled comparison showing lower total billed cost.
|
||||
Record the evidence before allocation. The explicit full-suite Testbox PR gate
|
||||
retains this exception; ordinary remote commands and changed gates use 16-class.
|
||||
See [remote proof](/reference/test/remote-proof#testbox-runner-sizing) for selection.
|
||||
A failed test, queue delay, or generic timeout does not justify promotion.
|
||||
|
||||
General proof defaults to 60 minutes including hydration. The existing
|
||||
`timeout_minutes` input can request a shorter deadline or explicitly extend a
|
||||
known long proof up to 240 minutes; larger values fail before allocation. Native Blacksmith warmup does not expose arbitrary
|
||||
workflow inputs, and Crabbox `--ttl` does not enforce a Testbox lifetime.
|
||||
The GitHub job timeout is the wall-clock limit.
|
||||
|
||||
The October 1 sizing audit favors 16-class over a blanket 8-class default.
|
||||
Across the prior seven days, 5,521 32-class runs had a 31-minute median lifetime
|
||||
and a 108-minute p95. The memory sampler returned 4,436 eligible runs; sampled
|
||||
[median](https://github.com/openclaw/openclaw/actions/runs/36588824988/job/109476238445)
|
||||
and [p75](https://github.com/openclaw/openclaw/actions/runs/36593802388/job/109493424473)
|
||||
peaks were 6.2 and 12.8 GiB against the 8-class's observed
|
||||
7.66 GiB. At those sampled peaks, total RAM minus available RAM was 6.8 and
|
||||
13.4 GiB, so reclaimable cache does not explain away the smaller-class risk.
|
||||
The separate 6,199-run 16-class population also included 25 OOM-affected jobs;
|
||||
these populations are not a controlled size comparison.
|
||||
|
||||
Job duration includes hydration, commands, and idle retention. The sampled
|
||||
[31-minute job](https://github.com/openclaw/openclaw/actions/runs/36630648805/job/109618719716)
|
||||
spent 30 minutes waiting in the Testbox monitor with a 30-minute
|
||||
idle setting. Do not treat lease percentiles as active-command percentiles or
|
||||
claim a measured timeout rate for the new policy. The 60-minute default is an
|
||||
operating choice; remeasure after the 15-minute idle cap is deployed before
|
||||
making 30 minutes or 8-class the general default.
|
||||
|
||||
These controls cover dispatches using the updated workflows in this repository.
|
||||
Historical refs, other repositories, alternate workflows, and Windows probe
|
||||
workflows are outside the shared pool. Organization-wide concurrency, per-token
|
||||
admission, SKU restrictions, and a hard spending stop require provider controls.
|
||||
|
||||
### CI runner routing
|
||||
|
||||
Hosted Node 24 setup honors the workflow's existing `NODE_VERSION` pin when the
|
||||
setup input is `24.x`. This prevents runner-image refreshes from silently choosing
|
||||
a different patch version. Explicit compatibility versions retain their own
|
||||
|
|
|
|||
|
|
@ -112,6 +112,35 @@ Unset all `CRABBOX_TAILSCALE*` overrides, force `--network public
|
|||
--tailscale=false`, clear exit-node/LAN flags, and require `crabbox inspect` to
|
||||
report public networking with no Tailscale state before uploading any script.
|
||||
|
||||
## Testbox runner sizing
|
||||
|
||||
Use the default 16-class workflow for routine remote proof, with a 60-minute
|
||||
total-job deadline including hydration. Keep the 32-class
|
||||
rare: record the command and its measured memory need, a smaller-runner OOM,
|
||||
or a controlled comparison showing lower total billed cost before selecting it.
|
||||
Existing memory-heavy full-suite Testbox PR gates use this exception. A generic
|
||||
failure, queue delay, or timeout is not a sizing signal. Keep resource-based
|
||||
worker limits; do not force higher parallelism on the smaller machine.
|
||||
|
||||
Select the exception explicitly with a fresh lease:
|
||||
|
||||
```bash
|
||||
node scripts/crabbox-wrapper.mjs run \
|
||||
--blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \
|
||||
--blacksmith-job check --idle-timeout 15m \
|
||||
--label <task-and-memory-reason> --timing-json -- <command>
|
||||
```
|
||||
|
||||
The high-memory profile uses at most four of the shared 32 Testbox concurrency
|
||||
slots. Both profiles cap idle time at 15 minutes. Routine proof defaults to
|
||||
60 minutes; the explicit high-memory workflow retains its four-hour deadline
|
||||
for known heavy gates. The standard workflow accepts an explicit
|
||||
`timeout_minutes` input up to 240 minutes, but Crabbox does not forward arbitrary
|
||||
workflow inputs and `--ttl` does not extend a Testbox job. Do not select a larger
|
||||
runner merely for more time. Direct-provider `--class` and `--type` flags do not size Testboxes;
|
||||
workflow selection owns the runner. A profile change needs a fresh lease.
|
||||
See [runner limits](/ci/runners#testbox-spending-limits) for queue behavior.
|
||||
|
||||
## Crabbox repository setup
|
||||
|
||||
The shared [Crabbox skill](https://github.com/openclaw/agent-skills/tree/main/skills/crabbox)
|
||||
|
|
@ -163,7 +192,7 @@ For a selected trusted Testbox lane:
|
|||
```bash
|
||||
node scripts/crabbox-wrapper.mjs run --timing-json -- \
|
||||
CI=1 NODE_OPTIONS=--max-old-space-size=4096 \
|
||||
OPENCLAW_TEST_PROJECTS_PARALLEL=6 OPENCLAW_VITEST_MAX_WORKERS=1 \
|
||||
OPENCLAW_VITEST_MAX_WORKERS=1 \
|
||||
OPENCLAW_TESTBOX=1 OPENCLAW_TESTBOX_REMOTE_RUN=1 \
|
||||
pnpm test <path-or-filter>
|
||||
```
|
||||
|
|
|
|||
130
scripts/ci-testbox-budget.mjs
Normal file
130
scripts/ci-testbox-budget.mjs
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
#!/usr/bin/env node
|
||||
import { createHash } from "node:crypto";
|
||||
import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const PROFILES = {
|
||||
check: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 240 },
|
||||
"check-memory": { runner: "blacksmith-32vcpu-ubuntu-2404", minutes: 240 },
|
||||
arm: { runner: "blacksmith-16vcpu-ubuntu-2404-arm", minutes: 120 },
|
||||
build: { runner: "blacksmith-16vcpu-ubuntu-2404", minutes: 35 },
|
||||
windows: { runner: "blacksmith-16vcpu-windows-2025", minutes: 75 },
|
||||
};
|
||||
const WINDOWS_RUNNERS = new Set([
|
||||
"blacksmith-8vcpu-windows-2025",
|
||||
"blacksmith-16vcpu-windows-2025",
|
||||
]);
|
||||
const ADMISSION_AGE_MS = 10 * 60_000;
|
||||
const CONCURRENT_LEASES = 32;
|
||||
const MAX_IDLE_MINUTES = 15;
|
||||
const DEFAULT_STANDARD_MINUTES = 60;
|
||||
const HIGH_MEMORY_LEASES = 4;
|
||||
|
||||
export function assertFreshTestboxAdmission(expiresAt, now = Date.now()) {
|
||||
const deadline = Number(expiresAt);
|
||||
if (!Number.isSafeInteger(deadline) || deadline <= now) {
|
||||
throw new Error("Testbox admission expired after 10 minutes; request a fresh lease.");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {{ profile: string, id: string, createdAt: string, runner?: string, minutes?: string | number }} request
|
||||
*/
|
||||
export function planTestboxAdmission(
|
||||
{ profile, id, runner, minutes, createdAt },
|
||||
now = Date.now(),
|
||||
) {
|
||||
const policy = PROFILES[profile];
|
||||
if (!policy || !/^tbx_[a-zA-Z0-9_-]+$/.test(id ?? "")) {
|
||||
throw new Error("Testbox admission requires a known profile and a tbx_ lease ID.");
|
||||
}
|
||||
const selectedRunner = runner || policy.runner;
|
||||
if (
|
||||
profile === "windows" ? !WINDOWS_RUNNERS.has(selectedRunner) : selectedRunner !== policy.runner
|
||||
) {
|
||||
throw new Error(`Runner ${selectedRunner} is not allowed for the ${profile} Testbox profile.`);
|
||||
}
|
||||
const defaultMinutes = profile === "check" ? DEFAULT_STANDARD_MINUTES : policy.minutes;
|
||||
const timeout = minutes === undefined || minutes === "" ? defaultMinutes : Number(minutes);
|
||||
if (!Number.isInteger(timeout) || timeout < 1 || timeout > policy.minutes) {
|
||||
throw new Error(`Testbox runtime must be an integer from 1 to ${policy.minutes} minutes.`);
|
||||
}
|
||||
const created = Date.parse(createdAt);
|
||||
const expiresAt = created + ADMISSION_AGE_MS;
|
||||
if (!Number.isFinite(created) || created > now) {
|
||||
throw new Error("Testbox workflow creation time is invalid.");
|
||||
}
|
||||
assertFreshTestboxAdmission(expiresAt, now);
|
||||
// The shared finite namespace is enforced atomically by GitHub concurrency.
|
||||
// Do not include a workflow, branch, actor, or runner in this group name.
|
||||
// High-memory work shares four of the global slots, never a separate pool.
|
||||
const slots = profile === "check-memory" ? HIGH_MEMORY_LEASES : CONCURRENT_LEASES;
|
||||
const slot = createHash("sha256").update(id).digest().readUInt32BE(0) % slots;
|
||||
return {
|
||||
group: `openclaw-testbox-budget-v1-${slot}`,
|
||||
runner: selectedRunner,
|
||||
minutes: timeout,
|
||||
expires_at: expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
export function boundedTestboxIdleMinutes(value) {
|
||||
const minutes = Number(value.trim());
|
||||
if (!Number.isInteger(minutes) || minutes < 1) {
|
||||
throw new Error("Testbox provider returned an invalid idle timeout.");
|
||||
}
|
||||
return Math.min(minutes, MAX_IDLE_MINUTES);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (process.argv[2] === "configure") {
|
||||
assertFreshTestboxAdmission(process.env.TESTBOX_EXPIRES_AT);
|
||||
const path = "/tmp/.testbox/idle_timeout";
|
||||
const idle = boundedTestboxIdleMinutes(readFileSync(path, "utf8"));
|
||||
writeFileSync(path, `${idle}\n`);
|
||||
console.log(
|
||||
`Testbox idle timeout capped at ${idle} minutes; active SSH work remains protected.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (process.argv[2] !== "admit") {
|
||||
throw new Error("Usage: node scripts/ci-testbox-budget.mjs admit|configure");
|
||||
}
|
||||
const response = await fetch(
|
||||
`${process.env.GITHUB_API_URL}/repos/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`,
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${process.env.GH_TOKEN}`,
|
||||
Accept: "application/vnd.github+json",
|
||||
},
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
},
|
||||
);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Cannot establish Testbox dispatch age: GitHub returned ${response.status}.`);
|
||||
}
|
||||
const run = await response.json();
|
||||
const plan = planTestboxAdmission({
|
||||
profile: process.env.TESTBOX_PROFILE,
|
||||
id: process.env.TESTBOX_ID,
|
||||
runner: process.env.TESTBOX_RUNNER,
|
||||
minutes: process.env.TESTBOX_MINUTES,
|
||||
createdAt: run.created_at,
|
||||
});
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries(plan)
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
console.log(
|
||||
`Testbox ${plan.group}: ${plan.runner}, at most ${plan.minutes} minutes, idle ceiling ${MAX_IDLE_MINUTES} minutes.`,
|
||||
);
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
main().catch((/** @type {unknown} */ error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
}
|
||||
|
|
@ -1236,7 +1236,7 @@ function enforceCrabboxOwnedBlacksmithLease(commandArgs: string[]) {
|
|||
console.error(
|
||||
[
|
||||
`[crabbox] provider=blacksmith-testbox --id ${id} has no Crabbox SSH key at ${userDisplayPath(keyPath)}.`,
|
||||
"[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 90m",
|
||||
"[crabbox] create reusable Testboxes through Crabbox before reusing them: node scripts/crabbox-wrapper.mjs warmup --provider blacksmith-testbox --idle-timeout 15m",
|
||||
"[crabbox] direct `blacksmith testbox warmup` leases can be used with `blacksmith testbox run`, but Crabbox cannot sync or run them by id.",
|
||||
].join("\n"),
|
||||
);
|
||||
|
|
|
|||
|
|
@ -190,17 +190,16 @@ run_remote_testbox_full_test_gate() {
|
|||
' "$script_parent_dir" "${!name}" "$name") || return 2
|
||||
[ -z "$value" ] || remote_env+=("$name=$value")
|
||||
done
|
||||
# Same Blacksmith Testbox delegation shape check:changed uses; the worktree's
|
||||
# own wrapper syncs this prep tree (the canonical copy would sync the primary
|
||||
# checkout instead).
|
||||
# Explicit full-suite proof retains the measured high-memory allocation.
|
||||
# The worktree wrapper syncs this prep tree, not the canonical checkout.
|
||||
run_quiet_logged "$label" "$log_file" \
|
||||
node scripts/crabbox-wrapper.mjs run \
|
||||
--provider blacksmith-testbox \
|
||||
--blacksmith-org openclaw \
|
||||
--blacksmith-workflow .github/workflows/ci-check-testbox.yml \
|
||||
--blacksmith-workflow .github/workflows/ci-check-high-memory-testbox.yml \
|
||||
--blacksmith-job check \
|
||||
--blacksmith-ref main \
|
||||
--idle-timeout 90m \
|
||||
--idle-timeout 15m \
|
||||
--ttl 240m \
|
||||
--timing-json \
|
||||
--label "$lease_label" \
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ const WORKFLOWS: {
|
|||
file: string;
|
||||
prGroup: string;
|
||||
manual: ManualPolicy;
|
||||
manualAdmission?: string;
|
||||
push?: { group: string; cancel: boolean };
|
||||
convertToDraft?: true;
|
||||
}[] = [
|
||||
|
|
@ -21,16 +22,19 @@ const WORKFLOWS: {
|
|||
file: ".github/workflows/ci-check-testbox.yml",
|
||||
prGroup: "Blacksmith Testbox-pr-v1-123",
|
||||
manual: { mode: "isolated per-run", group: "Blacksmith Testbox-manual-v1-201" },
|
||||
manualAdmission: "admission",
|
||||
},
|
||||
{
|
||||
file: ".github/workflows/ci-check-arm-testbox.yml",
|
||||
prGroup: "Blacksmith ARM Testbox-pr-v1-123",
|
||||
manual: { mode: "isolated per-run", group: "Blacksmith ARM Testbox-manual-v1-201" },
|
||||
manualAdmission: "admission",
|
||||
},
|
||||
{
|
||||
file: ".github/workflows/ci-build-artifacts-testbox.yml",
|
||||
prGroup: "Blacksmith Build Artifacts Testbox-pr-v1-123",
|
||||
manual: { mode: "isolated per-run", group: "Blacksmith Build Artifacts Testbox-manual-v1-201" },
|
||||
manualAdmission: "admission",
|
||||
},
|
||||
{
|
||||
file: ".github/workflows/ios-periphery.yml",
|
||||
|
|
@ -435,7 +439,7 @@ describe.each(WORKFLOWS)("ancillary admission: $file", (policy) => {
|
|||
expect(delayed.group).not.toBe(ready.group);
|
||||
for (const [id, eligible] of Object.entries(ready.eligibility!.jobs)) {
|
||||
if (id !== "scope") {
|
||||
expect(eligible, id).toBe(true);
|
||||
expect(eligible, id).toBe(id !== policy.manualAdmission);
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
|
|||
135
test/scripts/ci-testbox-budget.test.ts
Normal file
135
test/scripts/ci-testbox-budget.test.ts
Normal file
|
|
@ -0,0 +1,135 @@
|
|||
import { readFileSync } from "node:fs";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import {
|
||||
assertFreshTestboxAdmission,
|
||||
boundedTestboxIdleMinutes,
|
||||
planTestboxAdmission,
|
||||
} from "../../scripts/ci-testbox-budget.mjs";
|
||||
|
||||
const now = Date.parse("2026-10-01T12:00:00Z");
|
||||
const request = {
|
||||
profile: "check",
|
||||
id: "tbx_example",
|
||||
createdAt: "2026-10-01T11:55:00Z",
|
||||
};
|
||||
|
||||
describe("Testbox spending admission", () => {
|
||||
it("shares one bounded pool across profiles without changing a lease's slot", () => {
|
||||
const groups = new Set<string>();
|
||||
for (let index = 0; index < 256; index++) {
|
||||
const lease = { ...request, id: `tbx_example_${index}` };
|
||||
const group = planTestboxAdmission(lease, now).group;
|
||||
expect(group).toMatch(/^openclaw-testbox-budget-v1-(?:[0-9]|[12][0-9]|3[01])$/);
|
||||
for (const profile of ["arm", "build", "windows"]) {
|
||||
expect(planTestboxAdmission({ ...lease, profile }, now).group).toBe(group);
|
||||
}
|
||||
groups.add(group);
|
||||
}
|
||||
expect(groups.size).toBe(32);
|
||||
});
|
||||
|
||||
it("defaults routine proof to 16-class and requires the high-memory profile for 32-class", () => {
|
||||
expect(planTestboxAdmission(request, now).runner).toBe("blacksmith-16vcpu-ubuntu-2404");
|
||||
expect(() =>
|
||||
planTestboxAdmission({ ...request, runner: "blacksmith-32vcpu-ubuntu-2404" }, now),
|
||||
).toThrow(/not allowed/);
|
||||
const groups = new Set<string>();
|
||||
for (let index = 0; index < 128; index++) {
|
||||
const plan = planTestboxAdmission(
|
||||
{ ...request, profile: "check-memory", id: `tbx_memory_${index}` },
|
||||
now,
|
||||
);
|
||||
expect(plan.runner).toBe("blacksmith-32vcpu-ubuntu-2404");
|
||||
expect(plan.group).toMatch(/^openclaw-testbox-budget-v1-[0-3]$/);
|
||||
groups.add(plan.group);
|
||||
}
|
||||
expect(groups.size).toBe(4);
|
||||
});
|
||||
|
||||
it.each([0, -1, 241, 1.5, "invalid"])("rejects an unbounded runtime: %s", (minutes) => {
|
||||
expect(() => planTestboxAdmission({ ...request, minutes }, now)).toThrow(/runtime/);
|
||||
});
|
||||
|
||||
it("defaults routine proof to one hour and preserves explicit long-proof requests", () => {
|
||||
const workflow = parse(readFileSync(".github/workflows/ci-check-testbox.yml", "utf8"));
|
||||
const dispatchDefault = workflow.on.workflow_dispatch.inputs.timeout_minutes.default;
|
||||
expect(planTestboxAdmission({ ...request, minutes: dispatchDefault }, now).minutes).toBe(60);
|
||||
expect(planTestboxAdmission(request, now).minutes).toBe(60);
|
||||
expect(planTestboxAdmission({ ...request, minutes: "" }, now).minutes).toBe(60);
|
||||
expect(planTestboxAdmission({ ...request, minutes: 30 }, now).minutes).toBe(30);
|
||||
expect(planTestboxAdmission({ ...request, profile: "check-memory" }, now).minutes).toBe(240);
|
||||
expect(planTestboxAdmission({ ...request, minutes: 240 }, now).minutes).toBe(240);
|
||||
expect(() => planTestboxAdmission({ ...request, profile: "build", minutes: 36 }, now)).toThrow(
|
||||
/1 to 35/,
|
||||
);
|
||||
expect(() => planTestboxAdmission({ ...request, profile: "arm", minutes: 121 }, now)).toThrow(
|
||||
/1 to 120/,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows approved Windows sizes and refuses arbitrary dispatch labels", () => {
|
||||
for (const size of [8, 16]) {
|
||||
const runner = `blacksmith-${size}vcpu-windows-2025`;
|
||||
expect(planTestboxAdmission({ ...request, profile: "windows", runner }, now).runner).toBe(
|
||||
runner,
|
||||
);
|
||||
}
|
||||
for (const runner of ["self-hosted", "blacksmith-32vcpu-windows-2025"]) {
|
||||
expect(() => planTestboxAdmission({ ...request, profile: "windows", runner }, now)).toThrow(
|
||||
/not allowed/,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("expires abandoned queue entries before hydration, including the deadline itself", () => {
|
||||
const plan = planTestboxAdmission(request, now);
|
||||
expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at - 1)).not.toThrow();
|
||||
expect(() => assertFreshTestboxAdmission(plan.expires_at, plan.expires_at)).toThrow(/expired/);
|
||||
expect(() => planTestboxAdmission(request, plan.expires_at)).toThrow(/expired/);
|
||||
expect(() => planTestboxAdmission({ ...request, createdAt: "unknown" }, now)).toThrow(
|
||||
/invalid/,
|
||||
);
|
||||
});
|
||||
|
||||
it("caps idle requests while retaining shorter provider deadlines", () => {
|
||||
expect(boundedTestboxIdleMinutes("90\n")).toBe(15);
|
||||
expect(boundedTestboxIdleMinutes("5\n")).toBe(5);
|
||||
expect(() => boundedTestboxIdleMinutes("0")).toThrow(/invalid/);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["ci-check-testbox.yml", "check", "check"],
|
||||
["ci-check-high-memory-testbox.yml", "check", "check-memory"],
|
||||
["ci-check-arm-testbox.yml", "check-arm", "arm"],
|
||||
["ci-build-artifacts-testbox.yml", "build-artifacts", "build"],
|
||||
["windows-blacksmith-testbox.yml", "windows", "windows"],
|
||||
])("enforces admission before allocating %s", (file, jobName, profile) => {
|
||||
const workflow = parse(readFileSync(`.github/workflows/${file}`, "utf8"));
|
||||
const admission = workflow.jobs.admission;
|
||||
const job = workflow.jobs[jobName];
|
||||
expect(admission["runs-on"]).toBe("ubuntu-24.04");
|
||||
const budget = admission.steps.find((step: { id?: string }) => step.id === "budget");
|
||||
expect(budget.run).toBe("node scripts/ci-testbox-budget.mjs admit");
|
||||
expect(budget.env.TESTBOX_PROFILE).toBe(profile);
|
||||
expect(job.needs).toBe("admission");
|
||||
expect(job.if).toContain("needs.admission.result == 'success'");
|
||||
expect(job["runs-on"]).toContain("needs.admission.outputs.runner");
|
||||
expect(job.concurrency.group).toContain("needs.admission.outputs.group");
|
||||
expect(job.concurrency.queue).toBeUndefined();
|
||||
expect(job.concurrency["cancel-in-progress"]).toBe(false);
|
||||
const names = job.steps.map((step: { name: string }) => step.name);
|
||||
expect(names.indexOf("Reject expired Testbox admission")).toBeGreaterThan(
|
||||
names.indexOf("Begin Testbox"),
|
||||
);
|
||||
expect(names.indexOf("Reject expired Testbox admission")).toBeLessThan(
|
||||
names.indexOf("Checkout"),
|
||||
);
|
||||
if (profile !== "windows") {
|
||||
expect(names.indexOf("Bound Testbox idle lifetime")).toBeLessThan(
|
||||
names.indexOf("Setup Node environment"),
|
||||
);
|
||||
expect(names).toContain("Close Testbox SSH sessions");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
@ -873,9 +873,26 @@ AFTER_CD
|
|||
]);
|
||||
expect(workflow.on.pull_request.paths).toContain(workflowPath);
|
||||
expect(workflow.on.pull_request.paths).not.toContain(".github/workflows/**");
|
||||
expect(workflow.jobs[jobName].if).toBe(
|
||||
"${{ github.event_name != 'pull_request' || !github.event.pull_request.draft }}",
|
||||
);
|
||||
for (const [eventName, draft, result, cancelled, admitted] of [
|
||||
["pull_request", false, "skipped", false, true],
|
||||
["pull_request", true, "skipped", false, false],
|
||||
["pull_request", false, "skipped", true, false],
|
||||
["workflow_dispatch", false, "success", false, true],
|
||||
["workflow_dispatch", false, "failure", false, false],
|
||||
["workflow_dispatch", false, "success", true, false],
|
||||
] as const) {
|
||||
expect(
|
||||
evaluateWorkflowExpression(workflow.jobs[jobName].if, {
|
||||
eventName,
|
||||
draft,
|
||||
cancelled,
|
||||
additionalNeeds: { admission: { outputs: {}, result } },
|
||||
repository: "openclaw/openclaw",
|
||||
runAttempt: 1,
|
||||
}),
|
||||
`${workflowPath}: ${eventName}, admission=${result}, cancelled=${cancelled}`,
|
||||
).toBe(admitted);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -5494,9 +5494,21 @@ describe("ci workflow guards", () => {
|
|||
// PR events validate the artifact build on hosted runners (landing gate
|
||||
// stays satisfiable during Blacksmith outages); Testbox leases are
|
||||
// dispatch-only, mirroring ci-check-testbox.yml.
|
||||
expect(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"]).toBe(
|
||||
"${{ github.event_name == 'pull_request' && 'ubuntu-24.04' || 'blacksmith-16vcpu-ubuntu-2404' }}",
|
||||
);
|
||||
for (const [eventName, expected] of [
|
||||
["pull_request", "ubuntu-24.04"],
|
||||
["workflow_dispatch", "blacksmith-16vcpu-ubuntu-2404"],
|
||||
] as const) {
|
||||
expect(
|
||||
evaluateWorkflowExpression(buildArtifactsTestbox.jobs["build-artifacts"]["runs-on"], {
|
||||
eventName,
|
||||
repository: "openclaw/openclaw",
|
||||
runAttempt: 1,
|
||||
additionalNeeds: {
|
||||
admission: { outputs: { runner: "blacksmith-16vcpu-ubuntu-2404" }, result: "success" },
|
||||
},
|
||||
}),
|
||||
).toBe(expected);
|
||||
}
|
||||
for (const stepName of ["Begin Testbox", "Run Testbox"]) {
|
||||
expect(
|
||||
buildArtifactsTestbox.jobs["build-artifacts"].steps.find(
|
||||
|
|
|
|||
|
|
@ -11226,9 +11226,19 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
|
|||
expect(setupNodeWith).not.toHaveProperty("dependency-cache");
|
||||
expect(setupNodeWith).not.toHaveProperty("sticky-disk");
|
||||
expect(setupNodeWith["cache-mode"]).toBe("restore");
|
||||
expect(checkTestboxJob["timeout-minutes"]).toBe(
|
||||
"${{ fromJSON(inputs.timeout_minutes || '240') }}",
|
||||
);
|
||||
for (const [minutes, expected] of [
|
||||
["45", 45],
|
||||
["", 60],
|
||||
] as const) {
|
||||
expect(
|
||||
evaluateWorkflowExpression(checkTestboxJob["timeout-minutes"], {
|
||||
eventName: "workflow_dispatch",
|
||||
repository: "openclaw/openclaw",
|
||||
runAttempt: 1,
|
||||
additionalNeeds: { admission: { outputs: { minutes }, result: "success" } },
|
||||
}),
|
||||
).toBe(expected);
|
||||
}
|
||||
for (const step of [runTestboxStep, runArmTestboxStep, runBuildArtifactsTestboxStep]) {
|
||||
expect(step.uses).toBe(RUN_TESTBOX_WITH_FAILURE_REPORTING);
|
||||
}
|
||||
|
|
@ -11236,8 +11246,27 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
|
|||
"useblacksmith/run-testbox@3f60ff9ceb2c10c3feefa87dc0c6490cffae059d",
|
||||
);
|
||||
expect(windowsTestboxActionMarker.if).toBe("${{ false }}");
|
||||
expect(runTestboxStep.if).toBe("github.event_name == 'workflow_dispatch' && always()");
|
||||
expect(closeTestboxSshStep.if).toBe("github.event_name == 'workflow_dispatch' && always()");
|
||||
for (const step of [
|
||||
runTestboxStep,
|
||||
runArmTestboxStep,
|
||||
runBuildArtifactsTestboxStep,
|
||||
closeTestboxSshStep,
|
||||
]) {
|
||||
for (const [eventName, expected] of [
|
||||
["workflow_dispatch", true],
|
||||
["pull_request", false],
|
||||
] as const) {
|
||||
expect(
|
||||
evaluateWorkflowExpression(`\${{ ${step.if} }}`, {
|
||||
eventName,
|
||||
repository: "openclaw/openclaw",
|
||||
runAttempt: 1,
|
||||
failed: true,
|
||||
cancelled: true,
|
||||
}),
|
||||
).toBe(expected);
|
||||
}
|
||||
}
|
||||
expect(closeTestboxSshStep.run).toContain(
|
||||
`sudo sshd -T 2>/dev/null | awk '$1 == "port" { print $2; exit }'`,
|
||||
);
|
||||
|
|
@ -11247,10 +11276,6 @@ printf '%s\\n' "$DEEPSEEK_API_KEY" "$DEEPINFRA_API_KEY"`,
|
|||
expect(checkTestboxSteps.indexOf(closeTestboxSshStep)).toBe(
|
||||
checkTestboxSteps.indexOf(runTestboxStep) + 1,
|
||||
);
|
||||
expect(runArmTestboxStep.if).toBe("always()");
|
||||
expect(runBuildArtifactsTestboxStep.if).toBe(
|
||||
"github.event_name == 'workflow_dispatch' && always()",
|
||||
);
|
||||
expect(runWindowsTestboxStep.if).toBe("always()");
|
||||
expect(runWindowsTestboxStep.env?.JOB_STATUS).toBe("${{ job.status }}");
|
||||
expect(runWindowsTestboxStep.env?.NATIVE_SSH_USER).toBe(
|
||||
|
|
|
|||
|
|
@ -518,13 +518,13 @@ describe("remote testbox gate delegation", () => {
|
|||
"--blacksmith-org",
|
||||
"openclaw",
|
||||
"--blacksmith-workflow",
|
||||
".github/workflows/ci-check-testbox.yml",
|
||||
".github/workflows/ci-check-high-memory-testbox.yml",
|
||||
"--blacksmith-job",
|
||||
"check",
|
||||
"--blacksmith-ref",
|
||||
"main",
|
||||
"--idle-timeout",
|
||||
"90m",
|
||||
"15m",
|
||||
"--ttl",
|
||||
"240m",
|
||||
"--timing-json",
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ function runBasePreparation(
|
|||
eventName: string,
|
||||
) {
|
||||
const workflow = parse(fs.readFileSync(workflowName, "utf8"));
|
||||
const job = Object.values(workflow.jobs)[0] as { steps: Step[] };
|
||||
const jobs = Object.values(workflow.jobs) as { steps: Step[] }[];
|
||||
const values = new Map([
|
||||
["github.event.pull_request.base.sha", base],
|
||||
["github.event.pull_request.base.ref", "main"],
|
||||
|
|
@ -99,7 +99,9 @@ function runBasePreparation(
|
|||
'#!/bin/sh\nif [ "$1" = tee ]; then cat >/dev/null; fi\n',
|
||||
);
|
||||
fs.chmodSync(path.join(bin, "sudo"), 0o755);
|
||||
const step = job.steps.find((entry) => entry.name === "Prepare Testbox shell");
|
||||
const step = jobs
|
||||
.flatMap((job) => job.steps)
|
||||
.find((entry) => entry.uses === "./.github/actions/prepare-testbox-shell");
|
||||
if (!step?.uses) {
|
||||
throw new Error("Missing Testbox preparation action");
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue