mirror of
https://github.com/NeuralNomadsAI/CodeNomad.git
synced 2026-08-18 21:03:25 +00:00
fix(server): prevent Linux workspace launch identity timeouts (#625)
## Summary - prevent Linux workspace startup from failing when process identity discovery exceeds its one-second command deadline - capture the launched process and its existing process-group members without spawning helper commands for every `/proc` entry - preserve the leader-exit cleanup guarantees introduced by #602 - read immutable Linux process start ticks correctly and tolerate multiline task names ## Root cause Workspace startup captures an immutable process identity before accepting the OpenCode runtime. The Linux implementation scanned every process and launched several `cat`, `cut`, `sed`, `basename`, and `dirname` helpers per entry. On the affected Mint host, that synchronous shell command exceeded its one-second timeout. Identity capture then failed closed and stopped the newly launched OpenCode process; cleanup retried the same expensive probes and produced the repeated `spawnSync sh ETIMEDOUT` errors. The identity parser also used `$20`, which POSIX shells interpret as `$2` followed by `0`, rather than the twentieth positional field. This did not cause the startup timeout but weakened immutable process matching and is corrected here. ## Safety - launch discovery still retains every already-started member of the observed process group - guarded cleanup remains identity- and launch-token-based; no unverified PID fallback is introduced - WSL, macOS, and Windows paths retain their existing platform-specific probes ## Validation - server TypeScript typecheck - focused process identity and runtime suite: 23 passed, 1 Linux-only test skipped on Windows - real WSL `/proc` probe: correct start identity, 2 group members, 4 ms - broader workspace suite: 79 passed, 3 platform skips - `git diff --check` The broader workspace run still has the unrelated existing Windows fixture failure in `git-worktrees.test.ts` (`undefined` instead of `main`). Closes #624
This commit is contained in:
parent
70c9548f93
commit
269cff64ee
4 changed files with 74 additions and 14 deletions
|
|
@ -1,5 +1,7 @@
|
|||
import assert from "node:assert/strict"
|
||||
import type { SpawnSyncReturns } from "node:child_process"
|
||||
import { spawn as spawnChild, spawnSync, type SpawnSyncReturns } from "node:child_process"
|
||||
import { once } from "node:events"
|
||||
import { readFileSync } from "node:fs"
|
||||
import { describe, it } from "node:test"
|
||||
|
||||
import {
|
||||
|
|
@ -28,6 +30,33 @@ describe("process identity probes", () => {
|
|||
assert.deepEqual(probe.ok && probe.processes.get(42), identity())
|
||||
})
|
||||
|
||||
it("queries the requested Linux launch group without per-process subprocesses", () => {
|
||||
const call = {} as Call
|
||||
const probe = probePosixProcesses(spawn("42|1|42|123456|boot-a|123456\n", call), 25, "linux", { pids: [42], groupId: 42 })
|
||||
assert.deepEqual(call.args.slice(-1), ["42"])
|
||||
assert.match(call.script, /expected_group=\$stat_group/)
|
||||
assert.doesNotMatch(call.script, /\b(?:cat|cut|sed|basename|dirname)\b/)
|
||||
assert.deepEqual(probe.ok && probe.processes.get(42), identity())
|
||||
})
|
||||
|
||||
it("captures real Linux start ticks and launch-group members within the deadline", { skip: process.platform !== "linux" }, async () => {
|
||||
const child = spawnChild("sh", ["-c", "sleep 5"], { stdio: "ignore" })
|
||||
await once(child, "spawn")
|
||||
try {
|
||||
const stat = readFileSync(`/proc/${process.pid}/stat`, "utf8")
|
||||
const expectedStart = stat.slice(stat.lastIndexOf(") ") + 2).split(" ")[19]
|
||||
const probe = probePosixProcesses(spawnSync, 1_000, "linux", { pids: [process.pid], groupId: process.pid })
|
||||
assert.equal(probe.ok && probe.processes.get(process.pid)?.startTime, expectedStart)
|
||||
assert.equal(probe.ok && probe.processes.has(child.pid!), true)
|
||||
} finally {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
const exited = once(child, "exit")
|
||||
child.kill()
|
||||
await exited
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it("uses one delimiter-safe process-table query on portable POSIX", () => {
|
||||
const call = {} as Call
|
||||
const command = "/opt/opencode 'pipe|value'\t\"quoted\" café"
|
||||
|
|
|
|||
|
|
@ -36,20 +36,36 @@ export type TokenSignalResult = { ok: boolean; signalSent: boolean; targets: Pro
|
|||
export const LAUNCH_CLEANUP_TOKEN_ENV = "CODENOMAD_LAUNCH_CLEANUP_TOKEN"
|
||||
|
||||
type SpawnCommand = typeof spawnSync
|
||||
const SHELL_DOLLAR = "$"
|
||||
|
||||
const LINUX_IDENTITY_FUNCTIONS = String.raw`
|
||||
boot=$(cat /proc/sys/kernel/random/boot_id 2>/dev/null) || exit 20
|
||||
IFS= read -r boot 2>/dev/null < /proc/sys/kernel/random/boot_id || exit 20
|
||||
read_stat() {
|
||||
line=$(cat "/proc/$1/stat" 2>/dev/null) || return 1
|
||||
stat_pid=$(printf '%s\n' "$line" | cut -d' ' -f1); rest=$(printf '%s\n' "$line" | sed 's/^.*) //'); set -- $rest
|
||||
stat_ppid=$2; stat_group=$3; stat_start=$20
|
||||
line=
|
||||
while IFS= read -r chunk || test -n "$chunk"; do line=$line$chunk; done 2>/dev/null < "/proc/$1/stat"
|
||||
test -n "$line" || return 1
|
||||
stat_pid=$1; rest=${SHELL_DOLLAR}{line##*) }; set -- $rest
|
||||
test "$#" -ge 20 || return 1
|
||||
stat_ppid=$2; stat_group=$3; shift 19; stat_start=$1
|
||||
}
|
||||
emit_linux() {
|
||||
test -n "$1" && printf '%s|' "$1"
|
||||
printf '%s|%s|%s|%s|%s|%s\n' "$stat_pid" "$stat_ppid" "$stat_group" "$stat_start" "$boot" "$stat_start"
|
||||
}
|
||||
emit_linux() { printf '%s|%s|%s|%s|%s|%s|%s\n' "$1" "$stat_pid" "$stat_ppid" "$stat_group" "$stat_start" "$boot" "$stat_start"; }
|
||||
`
|
||||
|
||||
const LINUX_SNAPSHOT_SCRIPT = String.raw`${LINUX_IDENTITY_FUNCTIONS}
|
||||
for stat in /proc/[0-9]*/stat; do
|
||||
pid=$(basename "$(dirname "$stat")"); read_stat "$pid" && emit_linux "" | cut -c2-
|
||||
directory=${SHELL_DOLLAR}{stat%/stat}; pid=${SHELL_DOLLAR}{directory##*/}; read_stat "$pid" && emit_linux ""
|
||||
done
|
||||
exit 0
|
||||
`
|
||||
|
||||
const LINUX_LAUNCH_GROUP_SNAPSHOT_SCRIPT = String.raw`${LINUX_IDENTITY_FUNCTIONS}
|
||||
leader_pid=$1; read_stat "$leader_pid" || exit 22; expected_group=$stat_group; emit_linux ""
|
||||
for stat in /proc/[0-9]*/stat; do
|
||||
directory=${SHELL_DOLLAR}{stat%/stat}; pid=${SHELL_DOLLAR}{directory##*/}; test "$pid" = "$leader_pid" && continue
|
||||
read_stat "$pid" && test "$stat_group" = "$expected_group" && emit_linux ""
|
||||
done
|
||||
exit 0
|
||||
`
|
||||
|
|
@ -60,7 +76,7 @@ shift 5; matched=0; cutoff=; signal_sent=0
|
|||
if read_stat "$leader_pid" && test "$boot" = "$leader_boot" && test "$stat_start" = "$leader_start" && test "$stat_group" = "$expected_group"; then
|
||||
matched=1
|
||||
for stat in /proc/[0-9]*/stat; do
|
||||
candidate=$(basename "$(dirname "$stat")"); read_stat "$candidate" && test "$stat_group" = "$expected_group" && emit_linux CODENOMAD_TARGET
|
||||
directory=${SHELL_DOLLAR}{stat%/stat}; candidate=${SHELL_DOLLAR}{directory##*/}; read_stat "$candidate" && test "$stat_group" = "$expected_group" && emit_linux CODENOMAD_TARGET
|
||||
done
|
||||
if kill "-$requested_signal" -- "-$expected_group" 2>/dev/null; then
|
||||
signal_sent=1
|
||||
|
|
@ -111,7 +127,7 @@ pass=0
|
|||
while test "$pass" -lt "$passes"; do
|
||||
pass=$((pass + 1))
|
||||
for environ in /proc/[0-9]*/environ; do
|
||||
pid=$(basename "$(dirname "$environ")")
|
||||
directory=${SHELL_DOLLAR}{environ%/environ}; pid=${SHELL_DOLLAR}{directory##*/}
|
||||
if matches_token "$pid" && read_stat "$pid"; then
|
||||
test -n "$requested_signal" && prefix=CODENOMAD_TARGET || prefix=CODENOMAD_PROCESS
|
||||
emit_linux "$prefix"
|
||||
|
|
@ -416,10 +432,16 @@ export function descendantsOf(processes: Map<number, ProcessIdentity>, rootPid:
|
|||
|
||||
export function probePosixProcesses(spawnCommand: SpawnCommand, timeoutMs: number,
|
||||
platform: NodeJS.Platform = process.platform, filter?: PosixProcessFilter): ProcessSnapshot {
|
||||
if (platform === "linux") return querySnapshot(
|
||||
() => runLinuxScript(spawnCommand, LINUX_SNAPSHOT_SCRIPT, [], timeoutMs, "codenomad-posix-identity"),
|
||||
(output) => parseDelimitedSnapshot(output, true),
|
||||
)
|
||||
if (platform === "linux") {
|
||||
const pids = filter?.pids?.filter((pid) => Number.isInteger(pid) && pid > 0).map(String) ?? []
|
||||
const launchGroupProbe = pids.length === 1 && filter?.groupId === Number(pids[0])
|
||||
return querySnapshot(
|
||||
() => runLinuxScript(spawnCommand, launchGroupProbe ? LINUX_LAUNCH_GROUP_SNAPSHOT_SCRIPT : LINUX_SNAPSHOT_SCRIPT,
|
||||
launchGroupProbe ? pids : [], timeoutMs, "codenomad-posix-identity"),
|
||||
(output) => parseDelimitedSnapshot(output, true),
|
||||
{ allowEmpty: Boolean(filter) },
|
||||
)
|
||||
}
|
||||
// POSIX has no portable pidfd/start ticks; collect one coherent table instead of probing every PID.
|
||||
return querySnapshot(
|
||||
() => spawnCommand("ps", ["-axo", "pid=,ppid=,pgid=,lstart=,comm="], {
|
||||
|
|
|
|||
|
|
@ -83,6 +83,15 @@ async function harness(options: WorkspaceRuntimeOptions & { binary?: string; out
|
|||
return { runtime, child, timers, calls, launch, abort }
|
||||
}
|
||||
describe("workspace runtime lifecycle contracts", () => {
|
||||
it("captures the Linux launch group with one bounded shell command", async () => {
|
||||
let launchCall: Call | undefined
|
||||
await harness({ spawnSync: ((command: string, args: readonly string[]) => {
|
||||
launchCall ??= { command, args: [...args] }
|
||||
return result(posix([[4242, 1, 4242, "100"]]))
|
||||
}) as unknown as Command })
|
||||
assert.deepEqual(launchCall?.args.slice(-1), ["4242"])
|
||||
})
|
||||
|
||||
it("cancels before spawn and while waiting for a port without losing retryable cleanup", async () => {
|
||||
let spawned = false
|
||||
const runtime = new WorkspaceRuntime(new EventBus(), pino({ level: "silent" }), {
|
||||
|
|
|
|||
|
|
@ -276,7 +276,7 @@ export class WorkspaceRuntime {
|
|||
this.spawnCommand,
|
||||
this.stopCommandTimeoutMs,
|
||||
this.platform,
|
||||
this.platform === "linux" ? undefined : { pids: [child.pid], groupId: child.pid },
|
||||
{ pids: [child.pid], groupId: child.pid },
|
||||
)
|
||||
: { ok: false as const, error: "spawned child did not expose a PID" }
|
||||
const launchLeader = launchSnapshot.ok && child.pid ? launchSnapshot.processes.get(child.pid) : undefined
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue