mirror of
https://github.com/openclaw/openclaw.git
synced 2026-10-03 09:39:25 +00:00
fix(e2e): size upgrade-survivor Gateway readiness for saturated runners
Release Checks 36479006821 saw both the published 2026.9.6 baseline and the 2026.9.7 candidate bind HTTP after 56-58 s and miss the fixed 90 s readiness window on saturated runners, while the same phases pass in 36-41 s on idle ones. Raise the default startup budget to 300 s and make the readiness wait honor the configured budget instead of a hardcoded 360 polls, so the OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS override actually extends it.
This commit is contained in:
Peter Steinberger
parent
cbbddb1444
commit
1fd2ad9b56
4 changed files with 62 additions and 7 deletions
|
|
@ -2029,7 +2029,7 @@ probe_gateway_endpoint() {
|
|||
start_gateway() {
|
||||
local port=18789
|
||||
local budget
|
||||
budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 90)" || return "$?"
|
||||
budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 300)" || return "$?"
|
||||
local start_epoch
|
||||
local ready_epoch
|
||||
start_epoch="$(node -e "process.stdout.write(String(Date.now()))")" || return "$?"
|
||||
|
|
@ -2040,7 +2040,7 @@ start_gateway() {
|
|||
if [ "${SCENARIO:-}" = "watchos-direct-node" ]; then
|
||||
readiness_mode="legacy-ready-log-ok"
|
||||
fi
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$GATEWAY_LOG" 360 "$port" "$readiness_mode" || return "$?"
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$GATEWAY_LOG" "$((10#$budget * 4))" "$port" "$readiness_mode" || return "$?"
|
||||
ready_epoch="$(node -e "process.stdout.write(String(Date.now()))")" || return "$?"
|
||||
start_seconds=$(((ready_epoch - start_epoch + 999) / 1000))
|
||||
if [ "$start_seconds" -gt "$budget" ]; then
|
||||
|
|
|
|||
|
|
@ -559,7 +559,7 @@ run_update_restart_probe_gateway() {
|
|||
cp "$log_file" "${log_file}.before-start" || return "$?"
|
||||
fi
|
||||
local start_epoch ready_epoch budget service_status=0
|
||||
budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 90)" || return "$?"
|
||||
budget="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 300)" || return "$?"
|
||||
start_epoch="$(node -e "process.stdout.write(String(Date.now()))")" || return "$?"
|
||||
: >"$log_file" || return "$?"
|
||||
# Install and start both use the existing manager, which alone publishes the PID.
|
||||
|
|
@ -579,7 +579,7 @@ run_update_restart_probe_gateway() {
|
|||
fi
|
||||
fi
|
||||
gateway_pid="$(cat "$OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE")" || return "$?"
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$log_file" 360 "$port" "$readiness_mode" >"$readiness_log" 2>&1 || service_status=$?
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$log_file" "$((10#$budget * 4))" "$port" "$readiness_mode" >"$readiness_log" 2>&1 || service_status=$?
|
||||
if [ "$service_status" -ne 0 ]; then
|
||||
openclaw_e2e_print_log "$readiness_log" >&2
|
||||
return "$service_status"
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ if [ "$SCENARIO" = "abandoned-update" ] && [ -z "${OPENCLAW_UPGRADE_SURVIVOR_UPD
|
|||
UPDATE_RESTART_MODE="auto-auth"
|
||||
fi
|
||||
COMMAND_TIMEOUT="${OPENCLAW_UPGRADE_SURVIVOR_COMMAND_TIMEOUT:-900s}"
|
||||
START_BUDGET_SECONDS="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 90)"
|
||||
START_BUDGET_SECONDS="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 300)"
|
||||
STATUS_BUDGET_SECONDS="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_STATUS_BUDGET_SECONDS 30)"
|
||||
PROBE_TIMEOUT_MS="$(openclaw_e2e_read_nonnegative_int_env OPENCLAW_UPGRADE_SURVIVOR_PROBE_TIMEOUT_MS 60000)"
|
||||
PROBE_ATTEMPT_TIMEOUT_MS="$(
|
||||
|
|
@ -622,7 +622,7 @@ fi
|
|||
UPDATE_RESTART_MODE="${OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE:-manual}"
|
||||
command_timeout="${OPENCLAW_UPGRADE_SURVIVOR_COMMAND_TIMEOUT:-900s}"
|
||||
PORT=18789
|
||||
START_BUDGET="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 90)"
|
||||
START_BUDGET="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS 300)"
|
||||
STATUS_BUDGET="$(openclaw_e2e_read_positive_int_env OPENCLAW_UPGRADE_SURVIVOR_STATUS_BUDGET_SECONDS 30)"
|
||||
GATEWAY_LOG="$OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT/gateway.log"
|
||||
SYSTEMCTL_SHIM_LOG="$OPENCLAW_UPGRADE_SURVIVOR_ARTIFACT_ROOT/systemctl-shim.log"
|
||||
|
|
@ -865,7 +865,7 @@ else
|
|||
start_epoch="$(node -e "process.stdout.write(String(Date.now()))")"
|
||||
openclaw gateway --port "$PORT" --bind loopback --allow-unconfigured >"$GATEWAY_LOG" 2>&1 &
|
||||
gateway_pid="$!"
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$GATEWAY_LOG" 360 "$PORT"
|
||||
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$GATEWAY_LOG" "$((10#$START_BUDGET * 4))" "$PORT"
|
||||
ready_epoch="$(node -e "process.stdout.write(String(Date.now()))")"
|
||||
start_seconds=$(((ready_epoch - start_epoch + 999) / 1000))
|
||||
if [ "$start_seconds" -gt "$START_BUDGET" ]; then
|
||||
|
|
|
|||
55
test/scripts/upgrade-survivor-start-budget.test.ts
Normal file
55
test/scripts/upgrade-survivor-start-budget.test.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
import { spawnSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
it.each([
|
||||
{ budget: undefined, expectedStatus: 0 },
|
||||
{ budget: "135", expectedStatus: 0 },
|
||||
{ budget: "90", expectedStatus: 1 },
|
||||
])("bounds survivor readiness by its startup budget ($budget)", ({ budget, expectedStatus }) => {
|
||||
const root = tempDirs.make("survivor-start-budget-");
|
||||
const runner = readFileSync("scripts/e2e/lib/upgrade-survivor/run.sh", "utf8");
|
||||
const startGateway = runner.slice(
|
||||
runner.indexOf("start_gateway() {"),
|
||||
runner.indexOf("\nensure_gateway_started() {"),
|
||||
);
|
||||
const result = spawnSync(
|
||||
"/bin/bash",
|
||||
[
|
||||
"-c",
|
||||
`set -euo pipefail
|
||||
source scripts/lib/openclaw-e2e-instance.sh
|
||||
${startGateway}
|
||||
SCENARIO=base
|
||||
ticks=0
|
||||
# Model process liveness and the clock without booting a Gateway or sleeping.
|
||||
node() { printf '0'; }
|
||||
env() { :; }
|
||||
kill() { return 0; }
|
||||
sleep() {
|
||||
ticks=$((ticks + 1))
|
||||
if [ "$ticks" -eq 480 ]; then printf '[gateway] ready\\n' >>"$GATEWAY_LOG"; fi
|
||||
}
|
||||
openclaw_e2e_probe_http() { [ "$1" = http://127.0.0.1:18789/readyz ]; }
|
||||
start_gateway
|
||||
wait "$gateway_pid"
|
||||
`,
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
PATH: process.env.PATH,
|
||||
GATEWAY_LOG: path.join(root, "gateway.log"),
|
||||
...(budget === undefined ? {} : { OPENCLAW_UPGRADE_SURVIVOR_START_BUDGET_SECONDS: budget }),
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.status, result.stdout + result.stderr).toBe(expectedStatus);
|
||||
if (expectedStatus === 1) {
|
||||
expect(result.stdout).toContain("Gateway did not become ready");
|
||||
}
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue