fix(daemon): report the selected launchd job state (#162838)

* fix(daemon): report the selected launchd job state

Decode runtime facts through the existing root-field parser so nested coalition states cannot overwrite the selected job state. Reuse those facts for native observation and cleanup PID selection, remove the flat parser, and preserve decoded string trimming.

Regression reproduced before repair. Validation: 396 sibling cases, 15 final affected cases, selected checks and independent P2 review pass. A read-only native macOS probe matches independently extracted root state and PID. Production delta: -6 lines; no schema or service mutation change.

* test(daemon): use native launchd framing in update fixtures

Complete the transport fixture migration in update, Doctor, retained-policy and managed-handoff tests. Keep all production code and safety assertions unchanged.

Reproduced both CI failure groups before repair; all 13 original failures, additional fixture siblings, affected checks and full-candidate P2 review pass. Preserve the broad macOS systemd-mode failures and bounded baseline attribution without claiming all those local cases pass.
This commit is contained in:
Peter Steinberger 2026-10-01 11:52:46 -07:00 • committed by GitHub
parent bf3f9d7256
commit dc73f4bb40
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
16 changed files with 144 additions and 87 deletions

View file

@ -232,7 +232,7 @@ it
? {
code: 0,
termination: "exit",
stdout: `state = running\npid = ${gatewayPid}`,
stdout: `${args[1]} = {\n\tstate = running\n\tpid = ${gatewayPid}\n}`,
stderr: "",
}
: { code: 113, termination: "exit", stdout: "", stderr: "Could not find service" };

View file

@ -838,7 +838,7 @@ describe("preserved update activation with real version guards", () => {
const state = nativeRunning ? "running" : "stopped";
return {
code: 0,
stdout: args[0] === "print" ? `state = ${state}\n` : "",
stdout: args[0] === "print" ? `${args[1]} = {\n\tstate = ${state}\n}` : "",
stderr: "",
termination: "exit",
};

View file

@ -112,11 +112,11 @@ describe("Doctor disabled LaunchAgent diagnosis", () => {
await fs.mkdir(path.join(home, "Library", "LaunchAgents"), { recursive: true });
await fs.writeFile(path.join(home, "Library", "LaunchAgents", `${label}.plist`), "fixture");
}
vi.mocked(execLaunchctl).mockImplementation(async ([action]) => {
vi.mocked(execLaunchctl).mockImplementation(async ([action, target]) => {
if (action === "print") {
return {
code: loaded ? 0 : 113,
stdout: loaded ? "state = running" : "",
stdout: loaded ? `${target} = {\n\tstate = running\n}` : "",
stderr: loaded ? "" : "Could not find service",
termination: "exit",
};

View file

@ -143,6 +143,11 @@ function executeLaunchctlMock(file: string, args: string[]) {
};
}
if (call[0] === "print") {
const printed = (fields: string) => ({
stdout: [`${call[1]} = {`, ...fields.split("\n").map((line) => `\t${line}`), "}"].join("\n"),
stderr: "",
code: 0,
});
if (state.printNotLoadedRemaining > 0) {
state.printNotLoadedRemaining -= 1;
return { stdout: "", stderr: "Could not find service", code: 113 };
@ -156,21 +161,21 @@ function executeLaunchctlMock(file: string, args: string[]) {
return { stdout: "", stderr: "Could not find service", code: 113 };
}
if (serviceState === "stopped") {
return { stdout: ["state = waiting", "pid = 0"].join("\n"), stderr: "", code: 0 };
return printed("state = waiting\npid = 0");
}
if (serviceState === "running") {
return { stdout: ["state = running", "pid = 4242"].join("\n"), stderr: "", code: 0 };
return printed("state = running\npid = 4242");
}
if (!state.serviceLoaded) {
return { stdout: "", stderr: "Could not find service", code: 113 };
}
if (state.printOutput) {
return { stdout: state.printOutput, stderr: "", code: 0 };
return printed(state.printOutput);
}
if (!state.serviceRunning) {
return { stdout: ["state = waiting", "pid = 0"].join("\n"), stderr: "", code: 0 };
return printed("state = waiting\npid = 0");
}
return { stdout: ["state = running", "pid = 4242"].join("\n"), stderr: "", code: 0 };
return printed("state = running\npid = 4242");
}
if (call[0] === "disable" && state.disableError) {
return { stdout: "", stderr: state.disableError, code: state.disableCode };

View file

@ -94,7 +94,7 @@ it.each([
let loaded = true;
const activations: string[] = [];
const bootstrapDefinitions: string[] = [];
native.command.mockImplementation(async ([binary, nativeAction]) => {
native.command.mockImplementation(async ([binary, nativeAction, target]) => {
const action = expectDefined(nativeAction, "Expected a native launchctl action");
expect(binary).toBe("launchctl");
if (action === "bootout" || action === "unload") {
@ -112,7 +112,7 @@ it.each([
}
return {
code: action === "print" && !loaded ? 1 : 0,
stdout: action === "print" && loaded ? "state = waiting\n" : "",
stdout: action === "print" && loaded ? `${target} = {\n\tstate = waiting\n}` : "",
stderr: action === "print" && !loaded ? "Could not find service" : "",
signal: null,
killed: false,
@ -231,14 +231,14 @@ it.each([
it("retains another writer's artifact instead of claiming successful LaunchAgent recovery", async () => {
const { args, plist, environment } = await fixture();
let current = true;
native.command.mockResolvedValue({
native.command.mockImplementation(async (argv) => ({
code: 0,
stdout: "state = waiting\n",
stdout: `${argv[2]} = {\n\tstate = waiting\n}`,
stderr: "",
signal: null,
killed: false,
termination: "exit",
});
}));
const rename = fs.rename;
vi.spyOn(fs, "rename").mockImplementation(async (from, to) => {
await rename(from, to);

View file

@ -16,7 +16,7 @@ import {
bootstrapLaunchAgentOrThrow,
isLaunchctlAlreadyLoaded,
isUnsupportedGuiDomain,
parseLaunchctlPrint,
parseLaunchctlJob,
probeLaunchAgentState,
readLaunchAgentRuntime,
resolveLaunchAgentGatewayContext,
@ -58,7 +58,7 @@ function readLaunchAgentPidForCleanupSync(serviceTarget: string): number {
if (result.code !== 0) {
throw new Error(`launchctl print failed: ${formatLaunchctlResultDetail(result)}`);
}
const pid = parseLaunchctlPrint(result.stdout || result.stderr || "").pid;
const pid = parseLaunchctlJob(result.stdout || result.stderr || "", serviceTarget).runtime.pid;
if (pid === undefined) {
throw new Error("launchctl print did not report a running pid");
}

View file

@ -43,7 +43,9 @@ it.each(
return success;
}
if (command === "print") {
return loaded ? success : { ...success, code: 1, stderr: "Could not find service" };
return loaded
? { ...success, stdout: `${args[1]} = {\n\tstate = waiting\n}` }
: { ...success, code: 1, stderr: "Could not find service" };
}
if (command === "bootstrap") {
if (firstBootstrap && scenario === "bootstrap-failure") {
@ -129,7 +131,7 @@ it.each(
if (command === "print") {
inspections += 1;
return inspections === 1 && phase !== "disabled-bootstrap"
? success
? { ...success, stdout: `${args[1]} = {\n\tstate = waiting\n}` }
: { ...success, code: 1, stderr: "Could not find service" };
}
if (command === "print-disabled") {

View file

@ -0,0 +1,70 @@
import { expect, it, vi } from "vitest";
import { execLaunchctl } from "./launchd-exec.js";
import { probeLaunchAgentState } from "./launchd-runtime.js";
vi.mock("./launchd-exec.js", async (importOriginal) => ({
...(await importOriginal<typeof import("./launchd-exec.js")>()),
execLaunchctl: vi.fn(),
}));
const target = "gui/501/ai.openclaw.fixture";
it("reports the selected job state instead of its native coalition states", async () => {
vi.mocked(execLaunchctl).mockResolvedValue({
code: 0,
termination: "exit",
stderr: "",
stdout: [
`${target} = {`,
"\tstate = running \t",
"\tpid = 4242",
"\tlast exit status = 1",
"\tlast exit reason = exited \t",
"\tresource coalition = {",
"\t\tstate = active",
"\t}",
"\tjetsam coalition = {",
"\t\tstate = active",
"\t}",
"}",
].join("\n"),
});
await expect(probeLaunchAgentState(target)).resolves.toEqual({
state: "running",
runtime: { state: "running", pid: 4242, lastExitStatus: 1, lastExitReason: "exited" },
});
});
it("rejects pid and exit status values with junk suffixes", async () => {
vi.mocked(execLaunchctl).mockResolvedValue({
code: 0,
termination: "exit",
stderr: "",
stdout: [
`${target} = {`,
"\tstate = waiting",
"\tpid = 123abc",
"\tlast exit status = 7ms",
"\tlast exit reason = exited",
"}",
].join("\n"),
});
await expect(probeLaunchAgentState(target)).resolves.toEqual({
state: "stopped",
runtime: { state: "waiting", lastExitReason: "exited" },
});
});
it.each([
"gui/501/ai.openclaw.other = {\n\tstate = running\n\tpid = 4242\n}",
`${target} = {\n\tstate = running\n\tstate = waiting\n}`,
])("keeps unrecognized native job output unknown", async (stdout) => {
vi.mocked(execLaunchctl).mockResolvedValue({ code: 0, termination: "exit", stderr: "", stdout });
await expect(probeLaunchAgentState(target)).resolves.toMatchObject({
state: "unknown",
detail: expect.any(String),
});
});

View file

@ -8,6 +8,7 @@ import {
} from "@openclaw/normalization-core/number-coercion";
import { isStringRecord } from "@openclaw/normalization-core/record-coerce";
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import { formatErrorMessage } from "../infra/errors.js";
import { parseTcpPort, parseTcpPortFromArgs } from "../infra/tcp-port.js";
import { hasCommandProcessCleanupError } from "../process/exec-result.js";
import { sleep } from "../utils.js";
@ -36,7 +37,6 @@ import {
formatSystemLaunchDaemonOwnershipSummary,
inspectSystemLaunchDaemonOwnership,
} from "./launchd-system.js";
import { parseKeyValueOutput } from "./runtime-parse.js";
import { mergeGatewayServiceEnv } from "./service-env-merge.js";
import {
ServiceInspectionError,
@ -77,8 +77,26 @@ export function parseLaunchctlJob(output: string, serviceTarget: string) {
}
const block = (name: string) =>
output.match(new RegExp(`^\\t${name} = \\{\\n([\\s\\S]*?)^\\t\\}`, "m"))?.[1];
const runtime: LaunchctlPrintInfo = {};
const state = fields.get("state")?.trim();
if (state) {
runtime.state = state;
}
const pid = parseStrictPositiveInteger(fields.get("pid"));
if (pid !== undefined) {
runtime.pid = pid;
}
const status = parseStrictInteger(fields.get("last exit status"));
if (status !== undefined) {
runtime.lastExitStatus = status;
}
const exitReason = fields.get("last exit reason")?.trim();
if (exitReason) {
runtime.lastExitReason = exitReason;
}
return {
fields,
runtime,
arguments: block("arguments")
?.split("\n")
.filter((line) => line.startsWith("\t\t"))
@ -150,9 +168,7 @@ export async function readLoadedLaunchAgentState(
...(Object.keys(environment).length ? { environment } : {}),
sourcePath,
};
const parsed = parseLaunchctlPrint(
[...job.fields].map(([name, value]) => `${name} = ${value}`).join("\n"),
);
const parsed = job.runtime;
const running = parsed.state === "running" || (parsed.pid !== undefined && parsed.pid > 1);
return {
installed: true,
@ -421,28 +437,6 @@ type LaunchctlPrintInfo = {
lastExitReason?: string;
};
export function parseLaunchctlPrint(output: string): LaunchctlPrintInfo {
const entries = parseKeyValueOutput(output, "=");
const info: LaunchctlPrintInfo = {};
const state = entries.state;
if (state) {
info.state = state;
}
const pid = parseStrictPositiveInteger(entries.pid);
if (pid !== undefined) {
info.pid = pid;
}
const status = parseStrictInteger(entries["last exit status"]);
if (status !== undefined) {
info.lastExitStatus = status;
}
const exitReason = entries["last exit reason"];
if (exitReason) {
info.lastExitReason = exitReason;
}
return info;
}
export function parseLaunchAgentEnabled(output: string, label: string): boolean {
const labelPrefix = `"${label}"`;
for (const line of output.split("\n")) {
@ -607,12 +601,13 @@ export async function probeLaunchAgentState(
inspectionReason: launchctlInspectionReason(probe, serviceTarget),
};
}
const runtime = parseLaunchctlPrint(probe.stdout || probe.stderr || "");
if (
normalizeLowercaseStringOrEmpty(runtime.state) === "running" ||
(typeof runtime.pid === "number" && runtime.pid > 1)
) {
return { state: "running", runtime };
try {
const { runtime } = parseLaunchctlJob(probe.stdout || probe.stderr || "", serviceTarget);
const running =
normalizeLowercaseStringOrEmpty(runtime.state) === "running" ||
(typeof runtime.pid === "number" && runtime.pid > 1);
return { state: running ? "running" : "stopped", runtime };
} catch (error) {
return { state: "unknown", detail: formatErrorMessage(error) };
}
return { state: "stopped", runtime };
}

View file

@ -214,7 +214,12 @@ beforeEach(async () => {
prints++;
onPrint?.();
return isPidAlive(pid)
? { code: 0, termination: "exit", stdout: `state = running\npid = ${pid}\n`, stderr: "" }
? {
code: 0,
termination: "exit",
stdout: `${args[1]} = {\n\tstate = running\n\tpid = ${pid}\n}`,
stderr: "",
}
: { code: 113, termination: "exit", stdout: "", stderr: "Could not find service" };
}
if (args[0] === "bootout" || args[0] === "disable") {

View file

@ -45,7 +45,6 @@ import {
isLaunchAgentLoaded,
parkCurrentLaunchAgentForMaintenance,
parseLaunchAgentEnabled,
parseLaunchctlPrint,
parseLaunchctlListOpenClawUpdateJobs,
readLaunchAgentProgramArguments,
readLaunchAgentRuntime,
@ -192,34 +191,6 @@ describe("launchd runtime parsing", () => {
"launchctl print-disabled failed: Operation not permitted",
);
});
it("parses state, pid, and exit status", () => {
const output = [
"state = running",
"pid = 4242",
"last exit status = 1",
"last exit reason = exited",
].join("\n");
expect(parseLaunchctlPrint(output)).toEqual({
state: "running",
pid: 4242,
lastExitStatus: 1,
lastExitReason: "exited",
});
});
it("rejects pid and exit status values with junk suffixes", () => {
const output = [
"state = waiting",
"pid = 123abc",
"last exit status = 7ms",
"last exit reason = exited",
].join("\n");
expect(parseLaunchctlPrint(output)).toEqual({
state: "waiting",
lastExitReason: "exited",
});
});
});
describe("launchd runtime state", () => {

View file

@ -13,7 +13,6 @@ export {
isLaunchAgentLoaded,
launchAgentPlistExists,
parseLaunchAgentEnabled,
parseLaunchctlPrint,
readLaunchAgentProgramArguments,
readLaunchAgentRuntime,
} from "./launchd-runtime.js";

View file

@ -137,7 +137,12 @@ describe("native service command inspection", () => {
args[1]?.startsWith("system/")
? { code: 113, termination: "exit", stdout: "", stderr: "Could not find service" }
: condition === "registered"
? { code: 0, termination: "exit", stdout: "state = waiting", stderr: "" }
? {
code: 0,
termination: "exit",
stdout: `${args[1]} = {\n\tstate = waiting\n}`,
stderr: "",
}
: {
code: 1,
termination: "error",

View file

@ -226,7 +226,12 @@ it.each(["publication", "activation"])(
loaded = false;
}
if (args[0] !== "print" || loaded) {
return { code: 0, stdout: "state = running\npid = 42", stderr: "", termination: "exit" };
return {
code: 0,
stdout: `${args[1]} = {\n\tstate = running\n\tpid = 42\n}`,
stderr: "",
termination: "exit",
};
}
return { code: 113, stdout: "", stderr: "Could not find service", termination: "exit" };
});

View file

@ -428,7 +428,7 @@ describe("service definition backup receipts", () => {
return job === "unknown"
? { ...ok, code: 13, stderr: "Access denied" }
: cached
? { ...ok, stdout: "state = waiting\n" }
? { ...ok, stdout: `${args[1]} = {\n\tstate = waiting\n}` }
: missing;
}
if (args[0] === "bootout") {

View file

@ -408,11 +408,11 @@ if (${JSON.stringify(kind)} === "systemd") {
}
const fault = ${JSON.stringify(options?.launchdFault)};
if (state.restored && fault === "missing-restored-pid") {
process.stdout.write("state = running\\n");
process.stdout.write(args[1] + " = {\\n\\tstate = running\\n}\\n");
} else {
const restoredPid = fault === "dead-restored-pid" ? 2147483647 : ${process.pid};
const currentPid = fault === "wrong-parent" ? ${process.pid} : ${parentPid};
process.stdout.write("state = running\\npid = " + (state.restored ? restoredPid : currentPid) + "\\n");
process.stdout.write(args[1] + " = {\\n\\tstate = running\\n\\tpid = " + (state.restored ? restoredPid : currentPid) + "\\n}\\n");
}
}
}