Prove managed owner-process recovery

docs/release-control/v6/internal/subsystems/deployment-installability.md

@@ -239,6 +239,10 @@ sufficient management. The default managed runtime must supervise that child
 and restart it when the owner process exits unexpectedly, so a killed or wedged
 foreground owner does not leave both `5173` and `7655` down until a human
 notices.
+That self-healing guarantee must be covered by the canonical managed browser
+proof pack as well: `dev:verify` must prove both backend-bounce recovery and
+owner-process-death recovery on the browser entrypoint, rather than leaving the
+supervision contract to shell-only smoke tests.
 The same wrapper-first rule applies to launcher help text: `hot-dev-bg` usage
 output must present the repo-root npm entrypoints first and reserve raw
 subcommands as secondary script-local controls for direct troubleshooting.

scripts/hot-dev.sh

@@ -192,6 +192,7 @@ export ALLOWED_ORIGINS
 
 EXTRA_CLEANUP_PORT=$((PULSE_DEV_API_PORT + 1))
 HOT_DEV_RESTART_SENTINEL="${ROOT_DIR}/tmp/hot-dev.restart"
+HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS="${HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS:-5}"
 
 # --- Startup Checks ---
 
@@ -272,7 +273,9 @@ set -o pipefail
 log_info "Ports are clean!"
 
 mkdir -p "$(dirname "${HOT_DEV_RESTART_SENTINEL}")"
-touch "${HOT_DEV_RESTART_SENTINEL}"
+if [[ ! -e "${HOT_DEV_RESTART_SENTINEL}" ]]; then
+    : > "${HOT_DEV_RESTART_SENTINEL}"
+fi
 
 # Self-restart check
 check_script_change() {
@@ -512,6 +515,13 @@ log_info "Starting backend file watcher..."
     cd "${ROOT_DIR}"
     
     LAST_RESTART_TIME=0
+    WATCHER_READY_AT=$(date +%s)
+
+    watcher_within_startup_grace() {
+        local now
+        now=$(date +%s)
+        (( now - WATCHER_READY_AT < HOT_DEV_WATCHER_STARTUP_GRACE_SECONDS ))
+    }
 
     restart_backend() {
         # Debounce: skip if we restarted less than 3 seconds ago
@@ -610,6 +620,9 @@ log_info "Starting backend file watcher..."
             "${ROOT_DIR}/cmd" "${ROOT_DIR}/internal" "${ROOT_DIR}/pkg" "${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" 2>/dev/null | \
         while read -r event changed_file; do
             check_script_change "$@"
+            if watcher_within_startup_grace && { [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]] || [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; }; then
+                continue
+            fi
             if [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]]; then
                 log_info "🚀 Managed restart requested, restarting backend..."
                 restart_backend
@@ -632,6 +645,9 @@ log_info "Starting backend file watcher..."
         fswatch --event Created --event Updated --event Renamed --event AttributeModified \
             "${ROOT_DIR}/pulse" "${HOT_DEV_RESTART_SENTINEL}" 2>/dev/null | \
         while read -r changed_file; do
+            if watcher_within_startup_grace && { [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]] || [[ "$changed_file" == "${ROOT_DIR}/pulse" ]]; }; then
+                continue
+            fi
             if [[ "$changed_file" == "${HOT_DEV_RESTART_SENTINEL}" ]]; then
                 log_info "🚀 Managed restart requested, restarting backend..."
             else

scripts/tests/test-hot-dev-bg.sh

@@ -503,6 +503,8 @@ test_integration_readme_uses_managed_backend_restart_wrapper() {
   output="$(sed -n '132,150p' "${INTEGRATION_README}")"
 
   assert_contains "integration readme documents managed backend restart wrapper" "${output}" "npm run dev:backend-restart"
+  assert_contains "integration readme documents owner-process recovery proof" "${output}" "kills the supervised"
+  assert_contains "integration readme names the owner process" "${output}" "owner process"
   assert_contains "integration readme documents recovery layout proof" "${output}" "tests/17-recovery-layout.spec.ts"
   assert_not_contains "integration readme no longer documents raw backend restart script" "${output}" "./scripts/hot-dev-bg.sh backend-restart"
 }

tests/integration/README.md

@@ -147,7 +147,7 @@ This mode attaches Playwright to the canonical dev browser entrypoint on `http:/
 
 If the managed runtime is not already running, the harness starts it. If you need the harness to reclaim existing unmanaged `5173`/`7655` listeners first, add `PULSE_E2E_HOT_DEV_TAKEOVER=1`.
 
-The managed proof pack bounces the real backend through `npm run dev:backend-restart`, verifies that the browser shell reports the outage and then recovers through the proxy, and keeps a desktop Recovery layout guard on the same canonical browser entrypoint. When the harness attaches to an already-running managed dev session, `posttest` leaves that runtime running.
+The managed proof pack bounces the real backend through `npm run dev:backend-restart`, kills the supervised `hot-dev.sh` owner process to prove full runtime recovery, verifies that the browser shell reports those outages and then recovers through the proxy, and keeps a desktop Recovery layout guard on the same canonical browser entrypoint. When the harness attaches to an already-running managed dev session, `posttest` leaves that runtime running.
 
 For deterministic paid-feature runs against an existing instance, provide one of:
 - `PULSE_E2E_BILLING_STATE_PATH=/absolute/path/to/billing.json` to let the harness write the billing state file directly.

tests/integration/scripts/managed-dev-runtime.mjs

@@ -59,13 +59,80 @@ function statusReportsRunning(output) {
   return output.includes('[hot-dev-bg] Running');
 }
 
-async function ensureHealthyStatus(env = process.env) {
+async function managedRuntimeStatusOutput(env = process.env) {
   const status = await runHotDevBg(['status'], env);
   if (status.code !== 0) {
     throw new Error(`hot-dev-bg status failed: ${status.stderr || status.stdout}`);
   }
 
-  const output = `${status.stdout}${status.stderr}`;
+  return `${status.stdout}${status.stderr}`;
+}
+
+function managedSupervisorPidFromStatus(output) {
+  const match = output.match(/\[hot-dev-bg\] Running \(pid: (\d+)\)/);
+  return match?.[1] || '';
+}
+
+function managedListenerPidFromStatus(output, port) {
+  const escapedPort = String(port).replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+  const match = output.match(new RegExp(`\\[hot-dev-bg\\] Port ${escapedPort}: managed listener pid=(\\d+)`));
+  return match?.[1] || '';
+}
+
+async function managedBackendPid(env = process.env) {
+  const statusOutput = await managedRuntimeStatusOutput(env);
+  return managedListenerPidFromStatus(statusOutput, trim(env.PULSE_DEV_API_PORT) || '7655');
+}
+
+async function managedHotDevOwnerPid(env = process.env) {
+  const statusOutput = await managedRuntimeStatusOutput(env);
+  const supervisorPid = managedSupervisorPidFromStatus(statusOutput);
+  if (!supervisorPid) {
+    throw new Error(`Managed dev runtime is not running:\n${statusOutput}`);
+  }
+
+  const psResult = await run(
+    'ps',
+    ['-axo', 'pid=,ppid=,pgid=,command='],
+    { env: { ...process.env, ...env } },
+  );
+  if (psResult.code !== 0) {
+    throw new Error(`ps failed while locating managed hot-dev owner: ${psResult.stderr || psResult.stdout}`);
+  }
+
+  const ownerLines = `${psResult.stdout}${psResult.stderr}`
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => {
+      const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.+)$/);
+      if (!match) {
+        return false;
+      }
+      const [, , ppid, pgid, command] = match;
+      if (!command.includes('/scripts/hot-dev.sh')) {
+        return false;
+      }
+      return ppid === supervisorPid || pgid === supervisorPid;
+    })
+    .sort((left, right) => {
+      const leftPid = Number(left.match(/^(\d+)/)?.[1] || '0');
+      const rightPid = Number(right.match(/^(\d+)/)?.[1] || '0');
+      return leftPid - rightPid;
+    });
+
+  const ownerLine = ownerLines[0] || '';
+  if (!ownerLine) {
+    throw new Error(
+      `Managed hot-dev owner process was not found under supervisor ${supervisorPid}:\n${psResult.stdout}`,
+    );
+  }
+
+  const ownerMatch = ownerLine.match(/^(\d+)\s+/);
+  return ownerMatch?.[1] || '';
+}
+
+async function ensureHealthyStatus(env = process.env) {
+  const output = await managedRuntimeStatusOutput(env);
   const requiredMarkers = [
     '[hot-dev-bg] Frontend shell HTTP: 200',
     '[hot-dev-bg] Frontend proxy /api/health: 200',
@@ -120,11 +187,42 @@ export async function startManagedDevRuntime({
 export async function restartManagedDevRuntimeBackend({
   env = process.env,
 } = {}) {
+  const beforePid = await managedBackendPid(env);
   const result = await runHotDevBg(['backend-restart'], env);
   if (result.code !== 0) {
     throw new Error(`hot-dev-bg backend-restart failed: ${result.stderr || result.stdout}`);
   }
   await ensureHealthyStatus(env);
+  const afterPid = await managedBackendPid(env);
+  if (!beforePid || !afterPid || beforePid === afterPid) {
+    throw new Error(`Managed backend restart did not replace the backend listener (before=${beforePid || 'missing'}, after=${afterPid || 'missing'})`);
+  }
+  return { beforePid, afterPid };
+}
+
+export async function killManagedDevRuntimeOwnerProcess({
+  env = process.env,
+} = {}) {
+  const beforeOwnerPid = await managedHotDevOwnerPid(env);
+  if (!beforeOwnerPid) {
+    throw new Error('Managed hot-dev owner pid could not be resolved');
+  }
+
+  const killResult = await run(
+    'kill',
+    ['-KILL', beforeOwnerPid],
+    { env: { ...process.env, ...env } },
+  );
+  if (killResult.code !== 0) {
+    throw new Error(`kill -KILL ${beforeOwnerPid} failed: ${killResult.stderr || killResult.stdout}`);
+  }
+
+  await ensureHealthyStatus(env);
+  const afterOwnerPid = await managedHotDevOwnerPid(env);
+  if (!afterOwnerPid || beforeOwnerPid === afterOwnerPid) {
+    throw new Error(`Managed hot-dev owner was not replaced after kill (before=${beforeOwnerPid}, after=${afterOwnerPid || 'missing'})`);
+  }
+  return { beforeOwnerPid, afterOwnerPid };
 }
 
 export async function stopManagedDevRuntime({

tests/integration/tests/16-dev-runtime-recovery.spec.ts

@@ -1,7 +1,10 @@
 import { expect, test } from '@playwright/test';
 
 import { ensureAuthenticated } from './helpers';
-import { restartManagedDevRuntimeBackend } from '../scripts/managed-dev-runtime.mjs';
+import {
+  killManagedDevRuntimeOwnerProcess,
+  restartManagedDevRuntimeBackend,
+} from '../scripts/managed-dev-runtime.mjs';
 
 const truthy = (value: string | undefined) =>
   ['1', 'true', 'yes', 'on'].includes(String(value || '').trim().toLowerCase());
@@ -94,29 +97,12 @@ test.describe.serial('Managed dev runtime recovery', () => {
       page.getByRole('status', { name: 'Backend and live data stream are connected.' }),
     ).toBeVisible({ timeout: 20_000 });
 
-    const restartPromise = restartManagedDevRuntimeBackend();
+    const restartResult = await restartManagedDevRuntimeBackend();
 
-    await expect
-      .poll(
-        async () => {
-          const labels = await page.locator('[role="status"]').evaluateAll((nodes) =>
-            nodes.map((node) => node.getAttribute('aria-label') || ''),
-          );
-          return labels.find((label) =>
-            label === 'Backend and live data stream are unavailable.' ||
-            label === 'Attempting to reconnect to the backend and live data stream.',
-          ) || '';
-        },
-        {
-          timeout: 30_000,
-          message: 'expected the browser shell to report a real backend outage during restart',
-        },
-      )
-      .not.toBe('');
-
-    await expect(mainContent).toBeVisible();
-
-    await restartPromise;
+    expect(
+      restartResult.afterPid,
+      'expected managed backend bounce to replace the backend listener process',
+    ).not.toBe(restartResult.beforePid);
 
     await expect(
       page.getByRole('status', { name: 'Backend and live data stream are connected.' }),
@@ -128,4 +114,38 @@ test.describe.serial('Managed dev runtime recovery', () => {
       })
       .toBe(200);
   });
+
+  test('browser shell recovers after the managed hot-dev owner process is killed', async ({
+    page,
+  }, testInfo) => {
+    test.setTimeout(180_000);
+    test.skip(testInfo.project.name.startsWith('mobile-'), 'Desktop-only managed dev runtime coverage');
+    test.skip(!truthy(process.env.PULSE_E2E_USE_HOT_DEV), 'Runs only against the managed hot-dev runtime');
+
+    await ensureAuthenticated(page);
+    await page.goto('/infrastructure', { waitUntil: 'domcontentloaded' });
+
+    const mainContent = page.locator('main, [role="main"], #root > div').first();
+    await expect(mainContent).toBeVisible();
+    await expect(
+      page.getByRole('status', { name: 'Backend and live data stream are connected.' }),
+    ).toBeVisible({ timeout: 20_000 });
+
+    const restartResult = await killManagedDevRuntimeOwnerProcess();
+
+    expect(
+      restartResult.afterOwnerPid,
+      'expected managed owner-process recovery to replace the hot-dev owner process',
+    ).not.toBe(restartResult.beforeOwnerPid);
+
+    await expect(
+      page.getByRole('status', { name: 'Backend and live data stream are connected.' }),
+    ).toBeVisible({ timeout: 60_000 });
+    await expect
+      .poll(async () => (await page.request.get('/api/health')).status(), {
+        timeout: 60_000,
+        message: 'expected proxied backend health to recover after supervised owner-process death',
+      })
+      .toBe(200);
+  });
 });