feat(provisioner): support ClusterIP services and scoped skills PVC mounts (#4016)

* feat(provisioner): add ClusterIP mode for sandbox services

* feat(provisioner): support optional skills pvc subpath

backend/tests/test_provisioner_pvc_volumes.py

@@ -196,6 +196,22 @@ class TestBuildVolumeMounts:
         userdata_mount = mounts[-1]
         assert userdata_mount.sub_path is None
 
+    def test_skills_pvc_does_not_set_subpath_by_default(self, provisioner_module):
+        """PVC-backed skills keep legacy root mount unless explicitly configured."""
+        provisioner_module.SKILLS_PVC_NAME = "my-skills-pvc"
+        provisioner_module.SKILLS_PVC_SUBPATH_TEMPLATE = ""
+        mounts = provisioner_module._build_volume_mounts("thread-42", user_id="user-7")
+        skills_mount = mounts[0]
+        assert skills_mount.sub_path is None
+
+    def test_skills_pvc_can_use_user_scoped_subpath_template(self, provisioner_module):
+        """Operators can opt into per-user/thread skills subPath for shared PVCs."""
+        provisioner_module.SKILLS_PVC_NAME = "my-skills-pvc"
+        provisioner_module.SKILLS_PVC_SUBPATH_TEMPLATE = "deer-flow/users/{user_id}/threads/{thread_id}/skills"
+        mounts = provisioner_module._build_volume_mounts("thread-42", user_id="user-7")
+        skills_mount = mounts[0]
+        assert skills_mount.sub_path == "deer-flow/users/user-7/threads/thread-42/skills"
+
     def test_pvc_sets_user_scoped_subpath(self, provisioner_module):
         """PVC mode should include user_id in the user-data subPath."""
         provisioner_module.USERDATA_PVC_NAME = "my-pvc"
@@ -290,3 +306,12 @@ class TestBuildPodVolumes:
         assert mount_paths["skills-public"] == "/mnt/skills/public"
         assert mount_paths["skills-custom"] == "/mnt/skills/custom"
         assert mount_paths["skills-legacy"] == "/mnt/skills/legacy"
+
+    def test_pod_pvc_mode_can_use_user_scoped_skills_subpath(self, provisioner_module):
+        """Pod should use a configured user-scoped subPath for PVC skills."""
+        provisioner_module.SKILLS_PVC_NAME = "skills-pvc"
+        provisioner_module.SKILLS_PVC_SUBPATH_TEMPLATE = "deer-flow/users/{user_id}/threads/{thread_id}/skills"
+        provisioner_module.USERDATA_PVC_NAME = "userdata-pvc"
+        pod = provisioner_module._build_pod("sandbox-1", "thread-1", user_id="user-7")
+        skills_mount = pod.spec.containers[0].volume_mounts[0]
+        assert skills_mount.sub_path == "deer-flow/users/user-7/threads/thread-1/skills"

backend/tests/test_provisioner_request_threading.py

@@ -12,6 +12,7 @@ from types import SimpleNamespace
 import httpx
 import pytest
 from blockbuster import BlockBuster
+from kubernetes.client.rest import ApiException
 
 
 class _RecordingCoreV1:
@@ -20,11 +21,13 @@ class _RecordingCoreV1:
         *,
         event_loop_thread_id: int,
         ready_after_service_reads: dict[str, int] | None = None,
+        service_read_failures: dict[str, list[int]] | None = None,
     ) -> None:
         self.event_loop_thread_id = event_loop_thread_id
         self.thread_ids: list[int] = []
         self.service_sandboxes: set[str] = {"sandbox-existing"}
         self.ready_after_service_reads = ready_after_service_reads or {}
+        self.service_read_failures = service_read_failures or {}
         self.service_read_counts: dict[str, int] = {}
         self.created_pods: list[str] = []
         self.created_pod_specs: dict[str, object] = {}
@@ -46,10 +49,13 @@ class _RecordingCoreV1:
         self._record_k8s_call()
         sandbox_id = _sandbox_id_from_service_name(_name)
         self.service_read_counts[sandbox_id] = self.service_read_counts.get(sandbox_id, 0) + 1
+        failures = self.service_read_failures.get(sandbox_id) or []
+        if failures:
+            raise ApiException(status=failures.pop(0))
         ready_after_reads = self.ready_after_service_reads.get(sandbox_id, 1)
         if sandbox_id not in self.service_sandboxes or self.service_read_counts[sandbox_id] < ready_after_reads:
-            return _service_without_node_port(sandbox_id)
-        return _service(sandbox_id)
+            raise ApiException(status=404)
+        return _node_port_service(sandbox_id)
 
     def read_namespaced_pod(self, _name: str, _namespace: str):
         self._record_k8s_call()
@@ -76,20 +82,13 @@ class _RecordingCoreV1:
     def list_namespaced_service(self, _namespace: str, *, label_selector: str):
         self._record_k8s_call()
         assert label_selector == "app=deer-flow-sandbox"
-        return SimpleNamespace(items=[_service("sandbox-listed")])
+        return SimpleNamespace(items=[_node_port_service("sandbox-listed")])
 
 
-def _service(sandbox_id: str):
+def _node_port_service(sandbox_id: str):
     return SimpleNamespace(
         metadata=SimpleNamespace(labels={"sandbox-id": sandbox_id}),
-        spec=SimpleNamespace(ports=[SimpleNamespace(name="http", node_port=32123)]),
-    )
-
-
-def _service_without_node_port(sandbox_id: str):
-    return SimpleNamespace(
-        metadata=SimpleNamespace(labels={"sandbox-id": sandbox_id}),
-        spec=SimpleNamespace(ports=[]),
+        spec=SimpleNamespace(ports=[SimpleNamespace(name="http", port=8080, node_port=32123)]),
     )
 
 
@@ -202,3 +201,52 @@ def test_create_sandbox_route_builds_expected_skills_mount_layout(
     mount_names = [mount.name for mount in pod.spec.containers[0].volume_mounts]
     assert volume_names == expected_mount_names
     assert mount_names == expected_mount_names
+
+
+def test_create_sandbox_retries_transient_service_read_errors(monkeypatch: pytest.MonkeyPatch, provisioner_module) -> None:
+    fake_core_v1 = _RecordingCoreV1(
+        event_loop_thread_id=-1,
+        ready_after_service_reads={"sandbox-transient": 3},
+        service_read_failures={"sandbox-transient": [503, 429]},
+    )
+    monkeypatch.setattr(provisioner_module, "core_v1", fake_core_v1)
+    monkeypatch.setattr(provisioner_module.time, "sleep", lambda _seconds: None)
+
+    response = provisioner_module.create_sandbox(
+        provisioner_module.CreateSandboxRequest(
+            sandbox_id="sandbox-transient",
+            thread_id="thread-1",
+            user_id="user-1",
+        )
+    )
+
+    assert response.status == "Running"
+    assert response.sandbox_url == provisioner_module._sandbox_url("sandbox-transient", node_port=32123)
+    assert fake_core_v1.service_read_counts["sandbox-transient"] == 3
+
+
+def test_sandbox_service_defaults_to_node_port_with_node_host_url(provisioner_module) -> None:
+    provisioner_module.K8S_NAMESPACE = "mdv-sit"
+    provisioner_module.SANDBOX_CONTAINER_PORT = 8080
+    provisioner_module.SANDBOX_SERVICE_TYPE = "NodePort"
+    provisioner_module.NODE_HOST = "node.example"
+
+    service = provisioner_module._build_service("abc123")
+
+    assert service.spec.type == "NodePort"
+    assert service.spec.ports[0].port == 8080
+    assert service.spec.ports[0].target_port == 8080
+    assert provisioner_module._sandbox_url("abc123", node_port=32123) == "http://node.example:32123"
+
+
+def test_sandbox_service_supports_cluster_ip_with_dns_url(provisioner_module) -> None:
+    provisioner_module.K8S_NAMESPACE = "mdv-sit"
+    provisioner_module.SANDBOX_CONTAINER_PORT = 8080
+    provisioner_module.SANDBOX_SERVICE_TYPE = "ClusterIP"
+
+    service = provisioner_module._build_service("abc123")
+
+    assert service.spec.type == "ClusterIP"
+    assert service.spec.ports[0].port == 8080
+    assert service.spec.ports[0].target_port == 8080
+    assert provisioner_module._sandbox_url("abc123") == ("http://sandbox-abc123-svc.mdv-sit.svc.cluster.local:8080")

docker/provisioner/README.md

@@ -13,8 +13,8 @@ The **Sandbox Provisioner** is a FastAPI service that dynamically manages sandbo
                                                         │
                           ┌─────────────┐         ┌────▼─────┐
                           │   Backend   │ ──────▸ │  Sandbox │
-                          │ (via Docker │ NodePort│  Pod(s)  │
-                          │   network)  │         └──────────┘
+                          │ (NodePort   │ or DNS  │  Pod(s)  │
+                          │  /ClusterIP)│         └──────────┘
                           └─────────────┘
 ```
 
@@ -30,16 +30,16 @@ The **Sandbox Provisioner** is a FastAPI service that dynamically manages sandbo
    - Resource limits (CPU, memory, ephemeral storage)
    - Readiness/liveness probes
 
-3. **Service Creation**: A NodePort Service is created to expose the Pod, with Kubernetes auto-allocating a port from the NodePort range (typically 30000-32767).
+3. **Service Creation**: A Service is created to expose the Pod. By default this is a NodePort Service for Docker Compose compatibility. Set `SANDBOX_SERVICE_TYPE=ClusterIP` when the backend runs inside the Kubernetes cluster.
 
-4. **Access URL**: The provisioner returns `http://host.docker.internal:{NodePort}` to the backend, which the backend containers can reach directly.
+4. **Access URL**: In NodePort mode, the provisioner returns `http://{NODE_HOST}:{NodePort}`. In ClusterIP mode, it returns a Kubernetes service DNS URL like `http://sandbox-{sandbox_id}-svc.{namespace}.svc.cluster.local:8080`.
 
 5. **Cleanup**: When the session ends, `DELETE /api/sandboxes/{sandbox_id}` removes both the Pod and Service.
 
 The sandbox business endpoints are implemented as synchronous FastAPI handlers
 because the Kubernetes Python client used here is synchronous. Starlette runs
 sync handlers in its worker pool, keeping create/read/list/delete K8s API calls
-and the NodePort polling sleep off the ASGI event-loop thread. Keep `/health`
+and service access polling off the ASGI event-loop thread. Keep `/health`
 lightweight; do not move the sandbox CRUD handlers back to `async def` unless
 the K8s client path is also made async or explicitly offloaded.
 
@@ -148,9 +148,11 @@ The provisioner is configured via environment variables (set in [docker-compose-
 | `SKILLS_HOST_PATH` | - | **Host machine** path to skills directory (must be absolute) |
 | `THREADS_HOST_PATH` | - | **Host machine** path to threads data directory (must be absolute) |
 | `SKILLS_PVC_NAME` | empty (use hostPath) | PVC name for skills volume; when set, sandbox Pods use PVC instead of hostPath |
+| `SKILLS_PVC_SUBPATH_TEMPLATE` | empty | Optional `subPath` template for `SKILLS_PVC_NAME`. Supports `{user_id}` and `{thread_id}`. When empty, the skills PVC root is mounted unchanged |
 | `USERDATA_PVC_NAME` | empty (use hostPath) | PVC name for user-data volume; when set, uses PVC with `subPath: deer-flow/users/{user_id}/threads/{thread_id}/user-data` |
 | `KUBECONFIG_PATH` | `/root/.kube/config` | Path to kubeconfig **inside** the provisioner container |
-| `NODE_HOST` | `host.docker.internal` | Hostname that backend containers use to reach host NodePorts |
+| `SANDBOX_SERVICE_TYPE` | `NodePort` | Service type for sandbox access. Use `ClusterIP` when backend and provisioner run inside the same Kubernetes cluster |
+| `NODE_HOST` | `host.docker.internal` | Hostname that backend containers use to reach host NodePorts; ignored when `SANDBOX_SERVICE_TYPE=ClusterIP` |
 | `K8S_API_SERVER` | (from kubeconfig) | Override K8s API server URL (e.g., `https://host.docker.internal:26443`) |
 
 ### Custom sandbox image
@@ -175,6 +177,8 @@ PYTHONPATH=. python scripts/migrate_user_isolation.py --user-id <target-user-id>
 
 This moves legacy `threads/{thread_id}/user-data` data under `users/<target-user-id>/threads/{thread_id}/user-data`, which matches the new provisioner PVC subPath when the gateway base directory is mounted at `deer-flow/` on the PVC. Use `default` as the target user only when the legacy data should remain in the default no-auth user namespace. Run the migration while no gateway or sandbox Pods are writing to those paths.
 
+When skills are materialized per thread on the same PVC, set `SKILLS_PVC_NAME` to that PVC and configure `SKILLS_PVC_SUBPATH_TEMPLATE=deer-flow/users/{user_id}/threads/{thread_id}/skills`. Leaving the template empty preserves the legacy behavior of mounting the skills PVC root at `/mnt/skills`.
+
 ### Important: K8S_API_SERVER Override
 
 If your kubeconfig uses `localhost`, `127.0.0.1`, or `0.0.0.0` as the API server address (common with OrbStack, minikube, kind), the provisioner **cannot** reach it from inside the Docker container. 
@@ -333,13 +337,13 @@ docker exec deer-flow-gateway curl -s $SANDBOX_URL/v1/sandbox
 
 ### Issue: Cannot access sandbox URL from backend
 
-**Cause**: NodePort not reachable or `NODE_HOST` misconfigured.
+**Cause**: The backend cannot resolve or reach the sandbox ClusterIP Service DNS. This usually means the backend is not running inside the same Kubernetes cluster/network or cluster DNS/network policy is blocking access.
 
 **Solution**:
 - Verify the Service exists: `kubectl get svc -n deer-flow`
-- Test from host: `curl http://localhost:NODE_PORT/v1/sandbox`
-- Ensure `extra_hosts` is set in docker-compose (Linux)
-- Check `NODE_HOST` env var matches how backend reaches host
+- In NodePort mode, test from the backend container: `curl http://$NODE_HOST:NODE_PORT/v1/sandbox`
+- In ClusterIP mode, test from the backend Pod: `curl http://sandbox-XXX-svc.deer-flow.svc.cluster.local:8080/v1/sandbox`
+- Check `NODE_HOST` for NodePort deployments, or cluster DNS / NetworkPolicy / service mesh rules for ClusterIP deployments
 
 ## Security Considerations
 
@@ -347,7 +351,7 @@ docker exec deer-flow-gateway curl -s $SANDBOX_URL/v1/sandbox
 
 2. **Resource Limits**: Each sandbox Pod has CPU, memory, and storage limits to prevent resource exhaustion.
 
-3. **Network Isolation**: Sandbox Pods run in the `deer-flow` namespace but share the host's network namespace via NodePort. Consider NetworkPolicies for stricter isolation.
+3. **Network Isolation**: Sandbox Pods run in the configured namespace and are exposed through NodePort or ClusterIP Services. Prefer ClusterIP with NetworkPolicies for in-cluster deployments.
 
 4. **kubeconfig Access**: The provisioner has full access to your Kubernetes cluster via the mounted kubeconfig. Run it only in trusted environments.
 

docker/provisioner/app.py

@@ -1,12 +1,12 @@
 """DeerFlow Sandbox Provisioner Service.
 
 Dynamically creates and manages per-sandbox Pods in Kubernetes.
-Each ``sandbox_id`` gets its own Pod + NodePort Service.  The backend
-accesses sandboxes directly via ``{NODE_HOST}:{NodePort}``.
+Each ``sandbox_id`` gets its own Pod + Service.  The backend accesses sandboxes
+through NodePort or Kubernetes service DNS, depending on configuration.
 
 The provisioner connects to the host machine's Kubernetes cluster via a
-mounted kubeconfig (``~/.kube/config``).  Sandbox Pods run on the host
-K8s and are accessed by the backend via ``{NODE_HOST}:{NodePort}``.
+mounted kubeconfig (``~/.kube/config``) or in-cluster config.  Sandbox Pods
+run in K8s and are accessed by the backend via the configured Service mode.
 
 Endpoints:
     POST   /api/sandboxes              — Create a sandbox Pod + Service
@@ -23,8 +23,8 @@ Architecture (docker-compose-dev):
                                                            │ creates
                           ┌─────────────┐           ┌──────▼───────┐
                           │   backend   │ ────────▸ │   sandbox    │
-                          │             │  direct   │   Pod(s)     │
-                          └─────────────┘ NodePort  └──────────────┘
+                          │             │ direct/DNS│   Pod(s)     │
+                          └─────────────┘           └──────────────┘
 """
 
 from __future__ import annotations
@@ -63,17 +63,17 @@ THREADS_HOST_PATH = os.environ.get("THREADS_HOST_PATH", "/.deer-flow/threads")
 DEER_FLOW_HOST_BASE_DIR = os.environ.get("DEER_FLOW_HOST_BASE_DIR", "/.deer-flow")
 SKILLS_PVC_NAME = os.environ.get("SKILLS_PVC_NAME", "")
 USERDATA_PVC_NAME = os.environ.get("USERDATA_PVC_NAME", "")
+SKILLS_PVC_SUBPATH_TEMPLATE = os.environ.get("SKILLS_PVC_SUBPATH_TEMPLATE", "")
 SANDBOX_CONTAINER_PORT_RAW = os.environ.get("SANDBOX_CONTAINER_PORT", "8080")
+SANDBOX_SERVICE_TYPE = os.environ.get("SANDBOX_SERVICE_TYPE", "NodePort")
 try:
     SANDBOX_CONTAINER_PORT = int(SANDBOX_CONTAINER_PORT_RAW)
 except ValueError as exc:
-    raise RuntimeError(
-        f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT_RAW!r}; expected an integer TCP port"
-    ) from exc
+    raise RuntimeError(f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT_RAW!r}; expected an integer TCP port") from exc
 if not (1 <= SANDBOX_CONTAINER_PORT <= 65535):
-    raise RuntimeError(
-        f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT}; expected a value in [1, 65535]"
-    )
+    raise RuntimeError(f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT}; expected a value in [1, 65535]")
+if SANDBOX_SERVICE_TYPE not in {"NodePort", "ClusterIP"}:
+    raise RuntimeError(f"Invalid SANDBOX_SERVICE_TYPE={SANDBOX_SERVICE_TYPE!r}; expected 'NodePort' or 'ClusterIP'")
 SAFE_THREAD_ID_PATTERN = r"^[A-Za-z0-9_\-]+$"
 SAFE_USER_ID_PATTERN = r"^[A-Za-z0-9_\-]+$"
 DEFAULT_USER_ID = "default"
@@ -82,9 +82,9 @@ DEFAULT_USER_ID = "default"
 # Typically the host's ~/.kube/config is mounted here.
 KUBECONFIG_PATH = os.environ.get("KUBECONFIG_PATH", "/root/.kube/config")
 
-# The hostname / IP that the *backend container* uses to reach NodePort
-# services on the host Kubernetes node.  On Docker Desktop for macOS this
-# is ``host.docker.internal``; on Linux it may be the host's LAN IP.
+# The hostname / IP that the backend uses to reach NodePort services. On Docker
+# Desktop for macOS this is ``host.docker.internal``; on Linux it may be the
+# host's LAN IP. Ignored when SANDBOX_SERVICE_TYPE=ClusterIP.
 NODE_HOST = os.environ.get("NODE_HOST", "host.docker.internal")
 
 
@@ -122,27 +122,18 @@ def _init_k8s_client() -> k8s_client.CoreV1Api:
     """
     if os.path.exists(KUBECONFIG_PATH):
         if os.path.isdir(KUBECONFIG_PATH):
-            raise RuntimeError(
-                f"KUBECONFIG_PATH points to a directory, expected a file: {KUBECONFIG_PATH}"
-            )
+            raise RuntimeError(f"KUBECONFIG_PATH points to a directory, expected a file: {KUBECONFIG_PATH}")
         try:
             k8s_config.load_kube_config(config_file=KUBECONFIG_PATH)
             logger.info(f"Loaded kubeconfig from {KUBECONFIG_PATH}")
         except Exception as exc:
-            raise RuntimeError(
-                f"Failed to load kubeconfig from {KUBECONFIG_PATH}: {exc}"
-            ) from exc
+            raise RuntimeError(f"Failed to load kubeconfig from {KUBECONFIG_PATH}: {exc}") from exc
     else:
-        logger.warning(
-            f"Kubeconfig not found at {KUBECONFIG_PATH}; trying in-cluster config"
-        )
+        logger.warning(f"Kubeconfig not found at {KUBECONFIG_PATH}; trying in-cluster config")
         try:
             k8s_config.load_incluster_config()
         except Exception as exc:
-            raise RuntimeError(
-                "Failed to initialize Kubernetes client. "
-                f"No kubeconfig at {KUBECONFIG_PATH}, and in-cluster config is unavailable: {exc}"
-            ) from exc
+            raise RuntimeError(f"Failed to initialize Kubernetes client. No kubeconfig at {KUBECONFIG_PATH}, and in-cluster config is unavailable: {exc}") from exc
 
     # When connecting from inside Docker to the host's K8s API, the
     # kubeconfig may reference ``localhost`` or ``127.0.0.1``.  We
@@ -168,19 +159,11 @@ def _wait_for_kubeconfig(timeout: int = 30) -> None:
                 logger.info(f"Found kubeconfig file at {KUBECONFIG_PATH}")
                 return
             if os.path.isdir(KUBECONFIG_PATH):
-                raise RuntimeError(
-                    "Kubeconfig path is a directory. "
-                    f"Please mount a kubeconfig file at {KUBECONFIG_PATH}."
-                )
-            raise RuntimeError(
-                f"Kubeconfig path exists but is not a regular file: {KUBECONFIG_PATH}"
-            )
+                raise RuntimeError(f"Kubeconfig path is a directory. Please mount a kubeconfig file at {KUBECONFIG_PATH}.")
+            raise RuntimeError(f"Kubeconfig path exists but is not a regular file: {KUBECONFIG_PATH}")
         logger.info(f"Waiting for kubeconfig at {KUBECONFIG_PATH} …")
         time.sleep(2)
-    logger.warning(
-        f"Kubeconfig not found at {KUBECONFIG_PATH} after {timeout}s; "
-        "will attempt in-cluster Kubernetes config"
-    )
+    logger.warning(f"Kubeconfig not found at {KUBECONFIG_PATH} after {timeout}s; will attempt in-cluster Kubernetes config")
 
 
 def _ensure_namespace() -> None:
@@ -233,7 +216,7 @@ class CreateSandboxRequest(BaseModel):
 
 class SandboxResponse(BaseModel):
     sandbox_id: str
-    sandbox_url: str  # Direct access URL, e.g. http://host.docker.internal:{NodePort}
+    sandbox_url: str
     status: str
 
 
@@ -248,10 +231,15 @@ def _svc_name(sandbox_id: str) -> str:
     return f"sandbox-{sandbox_id}-svc"
 
 
-def _sandbox_url(node_port: int) -> str:
-    """Build the sandbox URL using the configured NODE_HOST."""
+def _sandbox_url(sandbox_id: str, node_port: int | None = None) -> str:
+    """Build the sandbox access URL for the configured Service mode."""
+    if SANDBOX_SERVICE_TYPE == "ClusterIP":
+        return f"http://{_svc_name(sandbox_id)}.{K8S_NAMESPACE}.svc.cluster.local:{SANDBOX_CONTAINER_PORT}"
+    if node_port is None:
+        raise RuntimeError("node_port is required when SANDBOX_SERVICE_TYPE=NodePort")
     return f"http://{NODE_HOST}:{node_port}"
 
+
 def _build_volumes(
     thread_id: str,
     user_id: str = DEFAULT_USER_ID,
@@ -272,10 +260,7 @@ def _build_volumes(
     if SKILLS_PVC_NAME:
         # PVC mode: three-way subPath not yet supported; fall back to
         # single-volume mount for backward compatibility.
-        logger.warning(
-            "SKILLS_PVC_NAME is set — three-way skills layout is not "
-            "supported in PVC mode yet; falling back to single /mnt/skills mount"
-        )
+        logger.warning("SKILLS_PVC_NAME is set — three-way skills layout is not supported in PVC mode yet; falling back to single /mnt/skills mount")
         volumes.append(
             k8s_client.V1Volume(
                 name="skills",
@@ -299,7 +284,11 @@ def _build_volumes(
         )
 
         user_custom_path = join_host_path(
-            DEER_FLOW_HOST_BASE_DIR, "users", user_id, "skills", "custom",
+            DEER_FLOW_HOST_BASE_DIR,
+            "users",
+            user_id,
+            "skills",
+            "custom",
         )
         volumes.append(
             k8s_client.V1Volume(
@@ -355,18 +344,23 @@ def _build_volume_mounts(
 
     Skills are mounted to ``/mnt/skills/{public,custom,legacy}/`` so that
     category-aware ``Skill.get_container_path()`` paths resolve correctly.
-    PVC mode falls back to a single ``/mnt/skills`` mount.
+    PVC mode falls back to a single ``/mnt/skills`` mount and can optionally
+    scope that mount with ``SKILLS_PVC_SUBPATH_TEMPLATE``.
     """
     mounts: list[k8s_client.V1VolumeMount] = []
 
     if SKILLS_PVC_NAME:
-        mounts.append(
-            k8s_client.V1VolumeMount(
+        skills_mount = k8s_client.V1VolumeMount(
             name="skills",
             mount_path="/mnt/skills",
             read_only=True,
         )
+        if SKILLS_PVC_SUBPATH_TEMPLATE:
+            skills_mount.sub_path = SKILLS_PVC_SUBPATH_TEMPLATE.format(
+                user_id=user_id,
+                thread_id=thread_id,
             )
+        mounts.append(skills_mount)
     else:
         mounts.extend(
             [
@@ -397,9 +391,7 @@ def _build_volume_mounts(
         read_only=False,
     )
     if USERDATA_PVC_NAME:
-        userdata_mount.sub_path = (
-            f"deer-flow/users/{user_id}/threads/{thread_id}/user-data"
-        )
+        userdata_mount.sub_path = f"deer-flow/users/{user_id}/threads/{thread_id}/user-data"
     mounts.append(userdata_mount)
 
     return mounts
@@ -491,7 +483,7 @@ def _build_pod(
 
 
 def _build_service(sandbox_id: str) -> k8s_client.V1Service:
-    """Construct a NodePort Service manifest (port auto-allocated by K8s)."""
+    """Construct a Service manifest for the configured access mode."""
     return k8s_client.V1Service(
         metadata=k8s_client.V1ObjectMeta(
             name=_svc_name(sandbox_id),
@@ -504,14 +496,13 @@ def _build_service(sandbox_id: str) -> k8s_client.V1Service:
             },
         ),
         spec=k8s_client.V1ServiceSpec(
-            type="NodePort",
+            type=SANDBOX_SERVICE_TYPE,
             ports=[
                 k8s_client.V1ServicePort(
                     name="http",
                     port=SANDBOX_CONTAINER_PORT,
                     target_port=SANDBOX_CONTAINER_PORT,
                     protocol="TCP",
-                    # nodePort omitted → K8s auto-allocates from the range
                 )
             ],
             selector={
@@ -521,16 +512,35 @@ def _build_service(sandbox_id: str) -> k8s_client.V1Service:
     )
 
 
-def _get_node_port(sandbox_id: str) -> int | None:
-    """Read the K8s-allocated NodePort from the Service."""
+def _url_from_service(svc, sandbox_id: str) -> str | None:
+    """Build the backend-facing sandbox URL from an already-fetched Service."""
+    if SANDBOX_SERVICE_TYPE == "ClusterIP":
+        return _sandbox_url(sandbox_id)
+
+    for port in svc.spec.ports or []:
+        if port.name == "http" and port.node_port:
+            return _sandbox_url(sandbox_id, node_port=port.node_port)
+    return None
+
+
+def _sandbox_access_url(sandbox_id: str, *, tolerate_read_errors: bool = False) -> str | None:
+    """Read the sandbox Service and return its backend-facing URL when ready."""
     try:
         svc = core_v1.read_namespaced_service(_svc_name(sandbox_id), K8S_NAMESPACE)
-        for port in svc.spec.ports or []:
-            if port.name == "http":
-                return port.node_port
-    except ApiException:
-        pass
+    except ApiException as exc:
+        if exc.status == 404:
             return None
+        if tolerate_read_errors and exc.status not in {401, 403}:
+            logger.warning(
+                "Transient error reading Service %s: status=%s reason=%s",
+                _svc_name(sandbox_id),
+                exc.status,
+                exc.reason,
+            )
+            return None
+        raise
+
+    return _url_from_service(svc, sandbox_id)
 
 
 def _get_pod_phase(sandbox_id: str) -> str:
@@ -553,7 +563,7 @@ async def health():
 
 @app.post("/api/sandboxes", response_model=SandboxResponse)
 def create_sandbox(req: CreateSandboxRequest):
-    """Create a sandbox Pod + NodePort Service for *sandbox_id*.
+    """Create a sandbox Pod + Service for *sandbox_id*.
 
     If the sandbox already exists, returns the existing information
     (idempotent).
@@ -572,11 +582,11 @@ def create_sandbox(req: CreateSandboxRequest):
     )
 
     # ── Fast path: sandbox already exists ────────────────────────────
-    existing_port = _get_node_port(sandbox_id)
-    if existing_port:
+    existing_url = _sandbox_access_url(sandbox_id, tolerate_read_errors=True)
+    if existing_url:
         return SandboxResponse(
             sandbox_id=sandbox_id,
-            sandbox_url=_sandbox_url(existing_port),
+            sandbox_url=existing_url,
             status=_get_pod_phase(sandbox_id),
         )
 
@@ -594,9 +604,7 @@ def create_sandbox(req: CreateSandboxRequest):
         logger.info(f"Created Pod {_pod_name(sandbox_id)}")
     except ApiException as exc:
         if exc.status != 409:  # 409 = AlreadyExists
-            raise HTTPException(
-                status_code=500, detail=f"Pod creation failed: {exc.reason}"
-            )
+            raise HTTPException(status_code=500, detail=f"Pod creation failed: {exc.reason}")
 
     # ── Create Service ───────────────────────────────────────────────
     try:
@@ -609,26 +617,22 @@ def create_sandbox(req: CreateSandboxRequest):
                 core_v1.delete_namespaced_pod(_pod_name(sandbox_id), K8S_NAMESPACE)
             except ApiException:
                 pass
-            raise HTTPException(
-                status_code=500, detail=f"Service creation failed: {exc.reason}"
-            )
+            raise HTTPException(status_code=500, detail=f"Service creation failed: {exc.reason}")
 
-    # ── Read the auto-allocated NodePort ─────────────────────────────
-    node_port: int | None = None
+    # ── Wait until the Service has a usable access URL ───────────────
+    sandbox_url: str | None = None
     for _ in range(20):
-        node_port = _get_node_port(sandbox_id)
-        if node_port:
+        sandbox_url = _sandbox_access_url(sandbox_id, tolerate_read_errors=True)
+        if sandbox_url:
             break
         time.sleep(0.5)
 
-    if not node_port:
-        raise HTTPException(
-            status_code=500, detail="NodePort was not allocated in time"
-        )
+    if not sandbox_url:
+        raise HTTPException(status_code=500, detail="Service access URL was not available in time")
 
     return SandboxResponse(
         sandbox_id=sandbox_id,
-        sandbox_url=_sandbox_url(node_port),
+        sandbox_url=sandbox_url,
         status=_get_pod_phase(sandbox_id),
     )
 
@@ -655,9 +659,7 @@ def destroy_sandbox(sandbox_id: str):
             errors.append(f"pod: {exc.reason}")
 
     if errors:
-        raise HTTPException(
-            status_code=500, detail=f"Partial cleanup: {', '.join(errors)}"
-        )
+        raise HTTPException(status_code=500, detail=f"Partial cleanup: {', '.join(errors)}")
 
     return {"ok": True, "sandbox_id": sandbox_id}
 
@@ -665,13 +667,13 @@ def destroy_sandbox(sandbox_id: str):
 @app.get("/api/sandboxes/{sandbox_id}", response_model=SandboxResponse)
 def get_sandbox(sandbox_id: str):
     """Return current status and URL for a sandbox."""
-    node_port = _get_node_port(sandbox_id)
-    if not node_port:
+    sandbox_url = _sandbox_access_url(sandbox_id)
+    if not sandbox_url:
         raise HTTPException(status_code=404, detail=f"Sandbox '{sandbox_id}' not found")
 
     return SandboxResponse(
         sandbox_id=sandbox_id,
-        sandbox_url=_sandbox_url(node_port),
+        sandbox_url=sandbox_url,
         status=_get_pod_phase(sandbox_id),
     )
 
@@ -685,25 +687,20 @@ def list_sandboxes():
             label_selector="app=deer-flow-sandbox",
         )
     except ApiException as exc:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to list services: {exc.reason}"
-        )
+        raise HTTPException(status_code=500, detail=f"Failed to list services: {exc.reason}")
 
     sandboxes: list[SandboxResponse] = []
     for svc in services.items:
         sid = (svc.metadata.labels or {}).get("sandbox-id")
         if not sid:
             continue
-        node_port = None
-        for port in svc.spec.ports or []:
-            if port.name == "http":
-                node_port = port.node_port
-                break
-        if node_port:
+        sandbox_url = _url_from_service(svc, sid)
+        if not sandbox_url:
+            continue
         sandboxes.append(
             SandboxResponse(
                 sandbox_id=sid,
-                    sandbox_url=_sandbox_url(node_port),
+                sandbox_url=sandbox_url,
                 status=_get_pod_phase(sid),
             )
         )