Merge branch 'nested-virt': ship libkrunfw with CONFIG_KVM=y

Adds out-of-the-box nested KVM support (Docker-in-Docker, qemu inside
the guest) and reroutes msb's writable state off ~/.microsandbox/ into
$XDG_STATE_HOME/agent-vm/msb-home/. See 6b2fdea for the full commit
message.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

.github/workflows/release-npm.yml

@@ -184,13 +184,95 @@ jobs:
           retention-days: 7
           if-no-files-found: error
 
-  # Assemble per-platform npm subpackage: pull in the two binaries
-  # from the parallel build jobs, fetch libkrunfw inline (the curl
-  # is ~10s so not worth a separate job), upload the populated
-  # subpackage tree as the per-platform artifact the publish job
-  # downloads.
+  # Build libkrunfw from source with the KVM-hypervisor config patch
+  # from libkrunfw-overrides/. Stock libkrunfw ships only the
+  # paravirt-guest helpers (CONFIG_KVM_GUEST=y) — agent-vm needs the
+  # in-kernel hypervisor (CONFIG_KVM=y + Intel/AMD backends) so the
+  # guest can host nested KVM (Docker-in-Docker, qemu, etc.). See
+  # libkrunfw-overrides/README.md for details.
+  build-libkrunfw:
+    needs: resolve-version
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux-x64
+            runner: ubuntu-latest
+            arch: x86_64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v5
+        # Need the patch file + LIBKRUNFW_VERSION constant. Submodules
+        # via the vendor path; recursive not required for libkrunfw
+        # itself (cloned separately below).
+        with:
+          submodules: true
+
+      - name: Resolve libkrunfw version from vendor/microsandbox
+        id: lkv
+        shell: bash
+        run: |
+          set -euo pipefail
+          libkrunfw_version=$(awk -F\" \
+            '/^pub const LIBKRUNFW_VERSION *: *&str *= *"/{print $2; exit}' \
+            vendor/microsandbox/crates/utils/lib/lib.rs)
+          test -n "$libkrunfw_version" || { echo "::error::could not extract libkrunfw version"; exit 1; }
+          echo "libkrunfw_version=$libkrunfw_version" >> "$GITHUB_OUTPUT"
+
+      - name: Cache libkrunfw build output
+        id: lk-cache
+        uses: actions/cache@v4
+        with:
+          # Key on version + arch + sha256 of the patch + workflow
+          # file so a patch edit or pin bump invalidates the cache.
+          key: libkrunfw-${{ steps.lkv.outputs.libkrunfw_version }}-${{ matrix.arch }}-${{ hashFiles('libkrunfw-overrides/config-libkrunfw_x86_64.patch', '.github/workflows/release-npm.yml') }}
+          path: libkrunfw-out/
+
+      - name: Install kernel build deps
+        if: steps.lk-cache.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential bc flex bison libelf-dev libssl-dev \
+            kmod cpio bzip2 xz-utils python3 python3-pyelftools \
+            curl ca-certificates patch
+
+      - name: Clone + patch + build libkrunfw
+        if: steps.lk-cache.outputs.cache-hit != 'true'
+        shell: bash
+        env:
+          LK_V: ${{ steps.lkv.outputs.libkrunfw_version }}
+          ARCH: ${{ matrix.arch }}
+        run: |
+          set -euo pipefail
+          git clone --branch "v${LK_V}" --depth 1 \
+            https://github.com/containers/libkrunfw.git libkrunfw-src
+          cd libkrunfw-src
+          patch -p1 < ../libkrunfw-overrides/config-libkrunfw_${ARCH}.patch
+          # olddefconfig (run by make rule on first invocation) resolves
+          # support symbols (KVM_X86, KVM_SMM, KVM_HYPERV, KVM_VFIO, …).
+          make -j"$(nproc)"
+          # Sanity: hypervisor symbols must be in the rebuilt vmlinux,
+          # otherwise the .so would silently ship the wrong kernel.
+          nm linux-*/vmlinux | grep -qE " kvm_dev_ioctl$" \
+            || { echo "::error::kvm_dev_ioctl missing — patch did not take"; exit 1; }
+          mkdir -p ../libkrunfw-out
+          cp libkrunfw.so."${LK_V}" ../libkrunfw-out/
+
+      - name: Upload libkrunfw artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: libkrunfw-${{ matrix.platform }}
+          path: libkrunfw-out/libkrunfw.so.*
+          retention-days: 7
+          if-no-files-found: error
+
+  # Assemble per-platform npm subpackage: pull in the three artefacts
+  # (agent-vm + msb + libkrunfw with KVM=y) from the parallel build
+  # jobs, create soname symlinks, upload the populated subpackage tree
+  # as the per-platform artifact the publish job downloads.
   package:
-    needs: [resolve-version, build-agent-vm, build-msb]
+    needs: [resolve-version, build-agent-vm, build-msb, build-libkrunfw]
     strategy:
       fail-fast: false
       matrix:
@@ -241,46 +323,27 @@ jobs:
           echo "libkrunfw_version=$libkrunfw_version" >> "$GITHUB_OUTPUT"
           echo "libkrunfw_abi=$libkrunfw_abi"         >> "$GITHUB_OUTPUT"
 
-      - name: Fetch libkrunfw from upstream microsandbox release
+      - name: Download patched libkrunfw artifact
+        uses: actions/download-artifact@v8
+        with:
+          name: libkrunfw-${{ matrix.platform }}
+          path: npm-dist/agent-vm-${{ matrix.platform }}/lib/
+
+      - name: Create libkrunfw soname symlinks
         shell: bash
         env:
-          MSB_V:    ${{ steps.lkv.outputs.msb_version }}
-          LK_V:     ${{ steps.lkv.outputs.libkrunfw_version }}
-          LK_ABI:   ${{ steps.lkv.outputs.libkrunfw_abi }}
-          LK_ARCH:  ${{ matrix.libkrunfw_arch }}
+          LK_V:   ${{ steps.lkv.outputs.libkrunfw_version }}
+          LK_ABI: ${{ steps.lkv.outputs.libkrunfw_abi }}
         run: |
           set -euo pipefail
-          if [[ "${{ matrix.platform }}" == darwin-* ]]; then
-            os_tag=darwin
-          else
-            os_tag=linux
-          fi
-          url="https://github.com/superradcompany/microsandbox/releases/download/v${MSB_V}/microsandbox-${os_tag}-${LK_ARCH}.tar.gz"
-          echo "Downloading $url"
           dst=npm-dist/agent-vm-${{ matrix.platform }}/lib
-          mkdir -p "$dst"
-          tmp=$(mktemp -d)
-          curl -fsSL -o "$tmp/bundle.tar.gz" "$url"
-          tar -xzf "$tmp/bundle.tar.gz" -C "$tmp"
-          shopt -s globstar nullglob
-          found=0
-          for f in "$tmp"/**/libkrunfw*; do
-            cp -a "$f" "$dst/"
-            found=$((found+1))
-          done
-          if [[ $found -eq 0 ]]; then
-            echo "::error::no libkrunfw* in $url"; ls -R "$tmp"; exit 1
-          fi
-          rm -rf "$tmp"
-          if [[ "$os_tag" == darwin ]]; then
+          if [[ "${{ matrix.platform }}" == darwin-* ]]; then
             full="libkrunfw.${LK_ABI}.dylib"
-            if [[ ! -e "$dst/$full" ]]; then
-              ln -s "libkrunfw.${LK_V}.dylib" "$dst/$full"
-            fi
+            test -f "$dst/$full" || { echo "::error::$dst/$full missing"; exit 1; }
           else
             full="libkrunfw.so.${LK_V}"
             soname="libkrunfw.so.${LK_ABI}"
-            test -f "$dst/$full" || { echo "::error::$dst/$full missing after extraction"; exit 1; }
+            test -f "$dst/$full" || { echo "::error::$dst/$full missing"; exit 1; }
             ln -sf "$full"   "$dst/$soname"
             ln -sf "$soname" "$dst/libkrunfw.so"
           fi

crates/agent-vm/src/main.rs

@@ -66,24 +66,24 @@ fn main() -> Result<()> {
     // contexts where the bundled msb may not be available
     // (e.g. inside the guest VM), so skip the check there too.
     //
-    // CRITICAL: `point_at_msb()` mutates the process environment via
-    // `unsafe { std::env::set_var("MSB_PATH", ...) }`. setenv() is
-    // not thread-safe under POSIX. We MUST run it before the tokio
-    // multi-thread runtime spawns workers (which happens inside
-    // `Runtime::new()`). Hence the manual sync `fn main` + manual
-    // runtime construction instead of `#[tokio::main]`.
+    // CRITICAL: `point_at_msb()` / `point_at_msb_home()` mutate the
+    // process environment via `unsafe { std::env::set_var(...) }`.
+    // setenv() is not thread-safe under POSIX. We MUST run them
+    // before the tokio multi-thread runtime spawns workers (which
+    // happens inside `Runtime::new()`). Hence the manual sync `fn
+    // main` + manual runtime construction instead of `#[tokio::main]`.
     let needs_msb_setup = !matches!(cli.cmd, Cmd::InterceptHook(_) | Cmd::Clipboard(_));
     if needs_msb_setup {
         msb_install::point_at_msb()?;
+        // Reroute msb's writable state off `~/.microsandbox/` and into
+        // agent-vm's own state dir. msb still finds `libkrunfw.so.*`
+        // via MSB_PATH → sibling `../lib/` (the bundle layout), so no
+        // copy/sync into MSB_HOME is needed — only the writable state
+        // (db, sandboxes, cache, tls/CA, logs) lives here.
+        msb_install::point_at_msb_home()?;
     }
     let runtime = tokio::runtime::Runtime::new().context("starting tokio runtime")?;
     runtime.block_on(async move {
-        if needs_msb_setup {
-            // Phase 9: auto-install the upstream microsandbox runtime
-            // libs (libkrunfw) into ~/.microsandbox if missing.
-            // Idempotent. Async because the bundle is fetched over HTTP.
-            msb_install::ensure_runtime_installed().await?;
-        }
         match cli.cmd {
             Cmd::Setup(args) => setup::run(args).await,
             Cmd::Pull(args) => pull::run(args).await,

crates/agent-vm/src/msb_install.rs

@@ -100,26 +100,35 @@ pub fn resolved_msb_path() -> Result<Option<PathBuf>> {
     Ok(None)
 }
 
-/// Ensure microsandbox's runtime libs (libkrunfw etc.) exist under
-/// `~/.microsandbox/{bin,lib}`. Downloads the upstream bundle if
-/// missing. Idempotent.
+/// Point msb at the agent-vm-controlled state dir instead of
+/// `~/.microsandbox/`.
 ///
-/// agent-vm prefers its own patched `msb` via [`point_at_msb`] (set
-/// after this call), but we still need libkrunfw — `cargo build`
-/// doesn't produce it; it comes from the upstream release bundle.
+/// Why we don't use the default upstream layout (`~/.microsandbox/`):
 ///
-/// TODO(npm): bundle libkrunfw inside the per-platform npm subpackage
-/// alongside the binaries so first-launch is fully offline-capable.
-pub async fn ensure_runtime_installed() -> Result<()> {
-    if microsandbox::setup::is_installed() {
-        return Ok(());
-    }
-    eprintln!("==> microsandbox runtime libs missing; downloading bundle (~17 MB, one-time)");
-    microsandbox::setup::install()
-        .await
-        .context("downloading microsandbox runtime bundle")?;
-    eprintln!("==> microsandbox runtime ready");
-    Ok(())
+/// agent-vm ships its own patched `msb` and a libkrunfw rebuilt with
+/// `CONFIG_KVM=y` (the upstream one has `# CONFIG_KVM is not set`,
+/// killing nested KVM). If a user has a separate microsandbox install
+/// — or just ran an older agent-vm version — the existing
+/// `~/.microsandbox/lib/libkrunfw.so.X.Y.Z` would shadow ours and
+/// `/dev/kvm` would silently not appear in the guest. Avoid the
+/// conflict entirely by giving agent-vm its own `MSB_HOME` under
+/// `state_root()`. Read-only bits (msb, libkrunfw) come from the
+/// bundle next to `agent-vm` via msb's own resolver
+/// (`MSB_PATH` → sibling `lib/`); writable bits (db, cache,
+/// sandboxes, logs, secrets, tls/CA) live here.
+///
+/// Idempotent. Returns the path that was pinned.
+pub fn point_at_msb_home() -> Result<PathBuf> {
+    let dir = crate::host_paths::state_root()
+        .ok_or_else(|| anyhow::anyhow!("could not resolve agent-vm state root ($HOME unset?)"))?
+        .join("msb-home");
+    std::fs::create_dir_all(&dir)
+        .with_context(|| format!("creating MSB_HOME at {}", dir.display()))?;
+    // SAFETY: like [`point_at_msb`], called before the tokio runtime
+    // spins up. setenv() is not thread-safe; this ordering invariant
+    // is the only thing that makes the call sound.
+    unsafe { std::env::set_var("MSB_HOME", &dir) };
+    Ok(dir)
 }
 
 /// Resolve and pin the patched `msb` binary for this process.

libkrunfw-overrides/README.md

@@ -0,0 +1,32 @@
+# libkrunfw overrides
+
+Patches applied on top of `containers/libkrunfw` at `LIBKRUNFW_VERSION`
+(read from `vendor/microsandbox/crates/utils/lib/lib.rs`) before
+building the `.so` that ships in the agent-vm npm package.
+
+The CI `package` job in `.github/workflows/release-npm.yml`:
+
+1. `git clone --branch v$LIBKRUNFW_VERSION --depth 1 https://github.com/containers/libkrunfw`
+2. `patch -p1 < libkrunfw-overrides/config-libkrunfw_$ARCH.patch`
+3. `make`
+4. Copies the resulting `libkrunfw.so.$LIBKRUNFW_VERSION` into
+   `npm-dist/agent-vm-$PLATFORM/lib/`.
+
+## Why these patches
+
+Stock libkrunfw ships only the paravirt-guest helpers
+(`CONFIG_KVM_GUEST=y`) — the guest kernel can talk to a host KVM but
+can't host its own. `agent-vm` needs the in-kernel KVM hypervisor for
+Docker-in-Docker / nested KVM use cases, so we flip:
+
+| symbol | reason |
+|---|---|
+| `CONFIG_KVM=y` | enable the in-kernel KVM hypervisor |
+| `CONFIG_KVM_INTEL=y` | Intel VMX backend |
+| `CONFIG_KVM_AMD=y` | AMD SVM backend |
+
+`olddefconfig` pulls in the rest (`KVM_X86`, `KVM_SMM`, `KVM_HYPERV`,
+`KVM_VFIO`, …) automatically.
+
+`devtmpfs` and `DEVTMPFS_MOUNT` are already enabled upstream so
+`/dev/kvm` materializes at boot without further config changes.

libkrunfw-overrides/config-libkrunfw_x86_64.patch

@@ -0,0 +1,17 @@
+--- a/config-libkrunfw_x86_64
++++ b/config-libkrunfw_x86_64
+@@ -485,7 +485,7 @@
+ 
+ CONFIG_HAVE_KVM=y
+ CONFIG_VIRTUALIZATION=y
+-# CONFIG_KVM is not set
++CONFIG_KVM=y
+ CONFIG_AS_AVX512=y
+ CONFIG_AS_SHA1_NI=y
+ CONFIG_AS_SHA256_NI=y
+@@ -2389,3 +2389,5 @@
+ #
+ # end of Rust hacking
+ # end of Kernel hacking
++CONFIG_KVM_INTEL=y
++CONFIG_KVM_AMD=y