diff --git a/docs/release-control/v6/internal/records/release-acceleration-v6.3.0-rc.5-2026-08-21.md b/docs/release-control/v6/internal/records/release-acceleration-v6.3.0-rc.5-2026-08-21.md index 7c2c53d3d3..39e5ee7c93 100644 --- a/docs/release-control/v6/internal/records/release-acceleration-v6.3.0-rc.5-2026-08-21.md +++ b/docs/release-control/v6/internal/records/release-acceleration-v6.3.0-rc.5-2026-08-21.md @@ -147,3 +147,14 @@ These projections do not close the objective. The release-acceleration claim remains open until the private payload reduction is measured, the production API decomposition produces a controlled PVE improvement, and a fresh exact-SHA release completes definitive customer convergence in 15 minutes or less. + +The rc.5 runner telemetry also explains why the backend used the slow path: +its one-time planner sample occurred while both credential-free release +compilers were resident and only 5,986 MiB remained available. That sample +permanently selected one API shard even though the compilers were short-lived. +The corrected admission policy waits at most 120 seconds for that useful +sibling work to release memory, then requires 14 GiB of available headroom for +two observed-size race binaries and the concurrent non-root package graph. If +the worker still cannot supply that floor, the gate reports a capacity failure +at admission instead of entering a one-shard path already known to approach +the 20-minute job timeout. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 60f192e1db..92ebaf68fb 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -636,7 +636,13 @@ upgrade, update, release, or artifact-selection behavior. execute concurrently with isolated data directories. A failed shard must fail the complete backend gate and terminate every descendant test process; sharding must never select a coverage subset or leave orphan race binaries - consuming the worker. The backend job owns a 20-minute ceiling, while each + consuming the worker. Auto-sharding must not make a permanent one-shard + decision from the transient memory peak created by sibling credential-free + release compilers. On an 8-vCPU worker it may wait up to 120 seconds for + those bounded compile processes to exit, but it must then require measured + headroom for two race binaries plus the concurrent package graph and fail + immediately with a capacity diagnosis when that admission floor is still + unavailable. The backend job owns a 20-minute ceiling, while each invocation retains the canonical 30-minute Go timeout as protection against a stuck package. The warm-path release-control performance objective is 15 minutes or less diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 4ec7daec3f..59acd1e950 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2741,6 +2741,10 @@ func TestReleaseBackendRaceGateUsesCompletePVEPartition(t *testing.T) { "go test -c -race", "python3 scripts/shard_go_tests.py", `--max-regex-bytes "$MAX_REGEX_BYTES"`, + `MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"`, + `memory_reserve_kib=$((4 * 1024 * 1024))`, + `memory_per_api_shard_kib=$((5 * 1024 * 1024))`, + "two-shard backend admission requires", "go test -race -timeout 30m", `-test.timeout 30m`, } { diff --git a/scripts/release_control/internal/release_preflight_test.py b/scripts/release_control/internal/release_preflight_test.py index 1c6d927d91..22d47254b2 100755 --- a/scripts/release_control/internal/release_preflight_test.py +++ b/scripts/release_control/internal/release_preflight_test.py @@ -228,6 +228,11 @@ class ReleasePreflightTest(unittest.TestCase): self.assertIn("go test -c -race", backend) self.assertIn("python3 scripts/shard_go_tests.py", backend) self.assertIn('--max-regex-bytes "$MAX_REGEX_BYTES"', backend) + self.assertIn( + 'MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}"', + backend, + ) + self.assertIn("two-shard backend admission requires", backend) self.assertIn("export GITHUB_ACTIONS=true", backend) self.assertIn("export CI=true", backend) diff --git a/scripts/run-release-backend-tests.sh b/scripts/run-release-backend-tests.sh index 0dfd1e033e..bc00aa477c 100755 --- a/scripts/run-release-backend-tests.sh +++ b/scripts/run-release-backend-tests.sh @@ -11,6 +11,8 @@ Options: --batch-size VALUE Top-level tests per API test-binary invocation (default: 10000) --max-regex-bytes VALUE Maximum encoded -test.run regex bytes (default: 65536) + --memory-wait-seconds VALUE + Bounded wait for two-shard memory admission (default: 120) EOF } @@ -18,6 +20,7 @@ DATA_ROOT="" API_SHARDS="${PULSE_BACKEND_TEST_SHARDS:-auto}" BATCH_SIZE=10000 MAX_REGEX_BYTES="${PULSE_BACKEND_TEST_MAX_REGEX_BYTES:-65536}" +MEMORY_WAIT_SECONDS="${PULSE_BACKEND_TEST_MEMORY_WAIT_SECONDS:-120}" while [ "$#" -gt 0 ]; do case "$1" in @@ -37,6 +40,10 @@ while [ "$#" -gt 0 ]; do MAX_REGEX_BYTES="${2:-}" shift 2 ;; + --memory-wait-seconds) + MEMORY_WAIT_SECONDS="${2:-}" + shift 2 + ;; -h|--help) usage exit 0 @@ -65,6 +72,10 @@ if [ "$MAX_REGEX_BYTES" -gt 120000 ]; then echo "Error: --max-regex-bytes cannot exceed the safe 120000-byte per-argument ceiling." >&2 exit 2 fi +if [[ ! "$MEMORY_WAIT_SECONDS" =~ ^[0-9]+$ ]]; then + echo "Error: --memory-wait-seconds must be a non-negative integer." >&2 + exit 2 +fi for command_name in go python3 getconf awk pgrep; do if ! command -v "$command_name" >/dev/null 2>&1; then @@ -87,19 +98,46 @@ rm -rf "$RUN_ROOT" mkdir -p "$PLAN_DIR" "$RUN_ROOT/data/other" VCPUS="$(getconf _NPROCESSORS_ONLN)" -AVAILABLE_KIB="$(awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo)" -if [ -z "$AVAILABLE_KIB" ]; then - AVAILABLE_KIB=0 -fi +read_available_kib() { + awk '/^MemAvailable:/ { print $2; exit }' /proc/meminfo +} + +AVAILABLE_KIB="$(read_available_kib)" +if [ -z "$AVAILABLE_KIB" ]; then AVAILABLE_KIB=0; fi if [ "$API_SHARDS" = auto ]; then cpu_shards=$((VCPUS / 4)) - memory_shards=$(((AVAILABLE_KIB - 2097152) / 4194304)) if [ "$cpu_shards" -lt 1 ]; then cpu_shards=1; fi - if [ "$memory_shards" -lt 1 ]; then memory_shards=1; fi + if [ "$cpu_shards" -gt 2 ]; then cpu_shards=2; fi + + # A release starts the public and private credential-free compilers beside + # this lane. Their short peak must not permanently collapse an otherwise + # capable 8-vCPU/24-GiB worker to one 20-minute API shard. Wait only while + # those useful jobs finish, then require measured headroom for both race + # binaries plus the concurrent non-root package graph. If that headroom + # never appears, fail at admission instead of discovering the capacity + # problem at the job timeout. + if [ "$cpu_shards" -gt 1 ]; then + memory_reserve_kib=$((4 * 1024 * 1024)) + memory_per_api_shard_kib=$((5 * 1024 * 1024)) + required_kib=$((memory_reserve_kib + cpu_shards * memory_per_api_shard_kib)) + waited_seconds=0 + while [ "$AVAILABLE_KIB" -lt "$required_kib" ] && [ "$waited_seconds" -lt "$MEMORY_WAIT_SECONDS" ]; do + remaining_seconds=$((MEMORY_WAIT_SECONDS - waited_seconds)) + wait_seconds=5 + if [ "$remaining_seconds" -lt "$wait_seconds" ]; then wait_seconds="$remaining_seconds"; fi + echo "Waiting ${wait_seconds}s for two-shard memory admission: $((AVAILABLE_KIB / 1024)) MiB available, $((required_kib / 1024)) MiB required." + sleep "$wait_seconds" + waited_seconds=$((waited_seconds + wait_seconds)) + AVAILABLE_KIB="$(read_available_kib)" + if [ -z "$AVAILABLE_KIB" ]; then AVAILABLE_KIB=0; fi + done + if [ "$AVAILABLE_KIB" -lt "$required_kib" ]; then + echo "Error: two-shard backend admission requires $((required_kib / 1024)) MiB available after a ${MEMORY_WAIT_SECONDS}s bounded wait; found $((AVAILABLE_KIB / 1024)) MiB." >&2 + exit 4 + fi + fi API_SHARDS="$cpu_shards" - if [ "$memory_shards" -lt "$API_SHARDS" ]; then API_SHARDS="$memory_shards"; fi - if [ "$API_SHARDS" -gt 2 ]; then API_SHARDS=2; fi elif [[ ! "$API_SHARDS" =~ ^[1-9][0-9]*$ ]]; then echo "Error: --api-shards must be auto or a positive integer." >&2 exit 2 @@ -109,6 +147,7 @@ echo "Release backend test plan" echo " vCPUs: $VCPUS" echo " Available MiB: $((AVAILABLE_KIB / 1024))" echo " API shards: $API_SHARDS" +echo " Memory wait: ${MEMORY_WAIT_SECONDS}s max" echo " Batch size: $BATCH_SIZE" echo " Regex bytes: $MAX_REGEX_BYTES max"