"""DeerFlow Sandbox Provisioner Service. Dynamically creates and manages per-sandbox Pods in Kubernetes. Each ``sandbox_id`` gets its own Pod + Service. The backend accesses sandboxes through NodePort or Kubernetes service DNS, depending on configuration. The provisioner connects to the host machine's Kubernetes cluster via a mounted kubeconfig (``~/.kube/config``) or in-cluster config. Sandbox Pods run in K8s and are accessed by the backend via the configured Service mode. Endpoints: POST /api/sandboxes — Create a sandbox Pod + Service DELETE /api/sandboxes/{sandbox_id} — Destroy a sandbox Pod + Service GET /api/sandboxes/{sandbox_id} — Get sandbox status & URL GET /api/sandboxes — List all sandboxes GET /health — Provisioner health check Architecture (docker-compose-dev): ┌────────────┐ HTTP ┌─────────────┐ K8s API ┌──────────────┐ │ remote │ ─────▸ │ provisioner │ ────────▸ │ host K8s │ │ _backend │ │ :8002 │ │ API server │ └────────────┘ └─────────────┘ └──────┬───────┘ │ creates ┌─────────────┐ ┌──────▼───────┐ │ backend │ ────────▸ │ sandbox │ │ │ direct/DNS│ Pod(s) │ └─────────────┘ └──────────────┘ """ from __future__ import annotations import logging import os import re import time from contextlib import asynccontextmanager import urllib3 from fastapi import FastAPI, HTTPException from kubernetes import client as k8s_client from kubernetes import config as k8s_config from kubernetes.client.rest import ApiException from pydantic import BaseModel, Field # Suppress only the InsecureRequestWarning from urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) logger = logging.getLogger(__name__) logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", ) # ── Configuration (all tuneable via environment variables) ─────────────── K8S_NAMESPACE = os.environ.get("K8S_NAMESPACE", "deer-flow") SANDBOX_IMAGE = os.environ.get( "SANDBOX_IMAGE", "enterprise-public-cn-beijing.cr.volces.com/vefaas-public/all-in-one-sandbox:latest", ) SKILLS_HOST_PATH = os.environ.get("SKILLS_HOST_PATH", "/skills") THREADS_HOST_PATH = os.environ.get("THREADS_HOST_PATH", "/.deer-flow/threads") DEER_FLOW_HOST_BASE_DIR = os.environ.get("DEER_FLOW_HOST_BASE_DIR", "/.deer-flow") SKILLS_PVC_NAME = os.environ.get("SKILLS_PVC_NAME", "") USERDATA_PVC_NAME = os.environ.get("USERDATA_PVC_NAME", "") SKILLS_PVC_SUBPATH_TEMPLATE = os.environ.get("SKILLS_PVC_SUBPATH_TEMPLATE", "") SANDBOX_CONTAINER_PORT_RAW = os.environ.get("SANDBOX_CONTAINER_PORT", "8080") SANDBOX_SERVICE_TYPE = os.environ.get("SANDBOX_SERVICE_TYPE", "NodePort") try: SANDBOX_CONTAINER_PORT = int(SANDBOX_CONTAINER_PORT_RAW) except ValueError as exc: raise RuntimeError(f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT_RAW!r}; expected an integer TCP port") from exc if not (1 <= SANDBOX_CONTAINER_PORT <= 65535): raise RuntimeError(f"Invalid SANDBOX_CONTAINER_PORT={SANDBOX_CONTAINER_PORT}; expected a value in [1, 65535]") if SANDBOX_SERVICE_TYPE not in {"NodePort", "ClusterIP"}: raise RuntimeError(f"Invalid SANDBOX_SERVICE_TYPE={SANDBOX_SERVICE_TYPE!r}; expected 'NodePort' or 'ClusterIP'") SAFE_THREAD_ID_PATTERN = r"^[A-Za-z0-9_\-]+$" SAFE_USER_ID_PATTERN = r"^[A-Za-z0-9_\-]+$" DEFAULT_USER_ID = "default" # Path to the kubeconfig *inside* the provisioner container. # Typically the host's ~/.kube/config is mounted here. KUBECONFIG_PATH = os.environ.get("KUBECONFIG_PATH", "/root/.kube/config") # The hostname / IP that the backend uses to reach NodePort services. On Docker # Desktop for macOS this is ``host.docker.internal``; on Linux it may be the # host's LAN IP. Ignored when SANDBOX_SERVICE_TYPE=ClusterIP. NODE_HOST = os.environ.get("NODE_HOST", "host.docker.internal") def join_host_path(base: str, *parts: str) -> str: """Join host filesystem path segments while preserving native style.""" if not parts: return base if re.match(r"^[A-Za-z]:[\\/]", base) or base.startswith("\\\\") or "\\" in base: from pathlib import PureWindowsPath result = PureWindowsPath(base) for part in parts: result /= part return str(result) from pathlib import Path result = Path(base) for part in parts: result /= part return str(result) # ── K8s client setup ──────────────────────────────────────────────────── core_v1: k8s_client.CoreV1Api | None = None def _init_k8s_client() -> k8s_client.CoreV1Api: """Load kubeconfig from the mounted host config and return a CoreV1Api. Tries the mounted kubeconfig first, then falls back to in-cluster config (useful if the provisioner itself runs inside K8s). """ if os.path.exists(KUBECONFIG_PATH): if os.path.isdir(KUBECONFIG_PATH): raise RuntimeError(f"KUBECONFIG_PATH points to a directory, expected a file: {KUBECONFIG_PATH}") try: k8s_config.load_kube_config(config_file=KUBECONFIG_PATH) logger.info(f"Loaded kubeconfig from {KUBECONFIG_PATH}") except Exception as exc: raise RuntimeError(f"Failed to load kubeconfig from {KUBECONFIG_PATH}: {exc}") from exc else: logger.warning(f"Kubeconfig not found at {KUBECONFIG_PATH}; trying in-cluster config") try: k8s_config.load_incluster_config() except Exception as exc: raise RuntimeError(f"Failed to initialize Kubernetes client. No kubeconfig at {KUBECONFIG_PATH}, and in-cluster config is unavailable: {exc}") from exc # When connecting from inside Docker to the host's K8s API, the # kubeconfig may reference ``localhost`` or ``127.0.0.1``. We # optionally rewrite the server address so it reaches the host. k8s_api_server = os.environ.get("K8S_API_SERVER") if k8s_api_server: configuration = k8s_client.Configuration.get_default_copy() configuration.host = k8s_api_server # Self-signed certs are common for local clusters configuration.verify_ssl = False api_client = k8s_client.ApiClient(configuration) return k8s_client.CoreV1Api(api_client) return k8s_client.CoreV1Api() def _wait_for_kubeconfig(timeout: int = 30) -> None: """Wait for kubeconfig file if configured, then continue with fallback support.""" deadline = time.time() + timeout while time.time() < deadline: if os.path.exists(KUBECONFIG_PATH): if os.path.isfile(KUBECONFIG_PATH): logger.info(f"Found kubeconfig file at {KUBECONFIG_PATH}") return if os.path.isdir(KUBECONFIG_PATH): raise RuntimeError(f"Kubeconfig path is a directory. Please mount a kubeconfig file at {KUBECONFIG_PATH}.") raise RuntimeError(f"Kubeconfig path exists but is not a regular file: {KUBECONFIG_PATH}") logger.info(f"Waiting for kubeconfig at {KUBECONFIG_PATH} …") time.sleep(2) logger.warning(f"Kubeconfig not found at {KUBECONFIG_PATH} after {timeout}s; will attempt in-cluster Kubernetes config") def _ensure_namespace() -> None: """Create the K8s namespace if it does not yet exist.""" try: core_v1.read_namespace(K8S_NAMESPACE) logger.info(f"Namespace '{K8S_NAMESPACE}' already exists") except ApiException as exc: if exc.status == 404: ns = k8s_client.V1Namespace( metadata=k8s_client.V1ObjectMeta( name=K8S_NAMESPACE, labels={ "app.kubernetes.io/name": "deer-flow", "app.kubernetes.io/component": "sandbox", }, ) ) core_v1.create_namespace(ns) logger.info(f"Created namespace '{K8S_NAMESPACE}'") else: raise # ── FastAPI lifespan ───────────────────────────────────────────────────── @asynccontextmanager async def lifespan(_app: FastAPI): global core_v1 _wait_for_kubeconfig() core_v1 = _init_k8s_client() _ensure_namespace() logger.info("Provisioner is ready (using host Kubernetes)") yield app = FastAPI(title="DeerFlow Sandbox Provisioner", lifespan=lifespan) # ── Request / Response models ─────────────────────────────────────────── class CreateSandboxRequest(BaseModel): sandbox_id: str thread_id: str = Field(pattern=SAFE_THREAD_ID_PATTERN) user_id: str = Field(default=DEFAULT_USER_ID, pattern=SAFE_USER_ID_PATTERN) include_legacy_skills: bool = False class SandboxResponse(BaseModel): sandbox_id: str sandbox_url: str status: str # ── K8s resource helpers ───────────────────────────────────────────────── def _pod_name(sandbox_id: str) -> str: return f"sandbox-{sandbox_id}" def _svc_name(sandbox_id: str) -> str: return f"sandbox-{sandbox_id}-svc" def _sandbox_url(sandbox_id: str, node_port: int | None = None) -> str: """Build the sandbox access URL for the configured Service mode.""" if SANDBOX_SERVICE_TYPE == "ClusterIP": return f"http://{_svc_name(sandbox_id)}.{K8S_NAMESPACE}.svc.cluster.local:{SANDBOX_CONTAINER_PORT}" if node_port is None: raise RuntimeError("node_port is required when SANDBOX_SERVICE_TYPE=NodePort") return f"http://{NODE_HOST}:{node_port}" def _build_volumes( thread_id: str, user_id: str = DEFAULT_USER_ID, *, include_legacy_skills: bool = False, ) -> list[k8s_client.V1Volume]: """Build volume list: PVC when configured, otherwise hostPath. Skills are split into public, per-user custom, and legacy (global-custom) volumes so that ``/mnt/skills/{public,custom,legacy}/`` paths resolve correctly inside the sandbox — matching the hostPath layout produced by ``LocalSandboxProvider`` and ``AioSandboxProvider``. """ volumes: list[k8s_client.V1Volume] = [] # ── Skills volumes ──────────────────────────────────────────────── if SKILLS_PVC_NAME: # PVC mode: three-way subPath not yet supported; fall back to # single-volume mount for backward compatibility. logger.warning("SKILLS_PVC_NAME is set — three-way skills layout is not supported in PVC mode yet; falling back to single /mnt/skills mount") volumes.append( k8s_client.V1Volume( name="skills", persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( claim_name=SKILLS_PVC_NAME, read_only=True, ), ) ) else: # hostPath mode: three-way layout public_path = join_host_path(SKILLS_HOST_PATH, "public") volumes.append( k8s_client.V1Volume( name="skills-public", host_path=k8s_client.V1HostPathVolumeSource( path=public_path, type="Directory", ), ) ) user_custom_path = join_host_path( DEER_FLOW_HOST_BASE_DIR, "users", user_id, "skills", "custom", ) volumes.append( k8s_client.V1Volume( name="skills-custom", host_path=k8s_client.V1HostPathVolumeSource( path=user_custom_path, type="DirectoryOrCreate", ), ) ) if include_legacy_skills: legacy_path = join_host_path(SKILLS_HOST_PATH, "custom") volumes.append( k8s_client.V1Volume( name="skills-legacy", host_path=k8s_client.V1HostPathVolumeSource( path=legacy_path, type="Directory", ), ) ) # ── User-data volume ────────────────────────────────────────────── if USERDATA_PVC_NAME: userdata_vol = k8s_client.V1Volume( name="user-data", persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( claim_name=USERDATA_PVC_NAME, ), ) else: userdata_vol = k8s_client.V1Volume( name="user-data", host_path=k8s_client.V1HostPathVolumeSource( path=join_host_path(THREADS_HOST_PATH, thread_id, "user-data"), type="DirectoryOrCreate", ), ) volumes.append(userdata_vol) return volumes def _build_volume_mounts( thread_id: str, user_id: str = DEFAULT_USER_ID, *, include_legacy_skills: bool = False, ) -> list[k8s_client.V1VolumeMount]: """Build volume mount list, mirroring three-way skills layout. Skills are mounted to ``/mnt/skills/{public,custom,legacy}/`` so that category-aware ``Skill.get_container_path()`` paths resolve correctly. PVC mode falls back to a single ``/mnt/skills`` mount and can optionally scope that mount with ``SKILLS_PVC_SUBPATH_TEMPLATE``. """ mounts: list[k8s_client.V1VolumeMount] = [] if SKILLS_PVC_NAME: skills_mount = k8s_client.V1VolumeMount( name="skills", mount_path="/mnt/skills", read_only=True, ) if SKILLS_PVC_SUBPATH_TEMPLATE: skills_mount.sub_path = SKILLS_PVC_SUBPATH_TEMPLATE.format( user_id=user_id, thread_id=thread_id, ) mounts.append(skills_mount) else: mounts.extend( [ k8s_client.V1VolumeMount( name="skills-public", mount_path="/mnt/skills/public", read_only=True, ), k8s_client.V1VolumeMount( name="skills-custom", mount_path="/mnt/skills/custom", read_only=True, ), ] ) if include_legacy_skills: mounts.append( k8s_client.V1VolumeMount( name="skills-legacy", mount_path="/mnt/skills/legacy", read_only=True, ) ) userdata_mount = k8s_client.V1VolumeMount( name="user-data", mount_path="/mnt/user-data", read_only=False, ) if USERDATA_PVC_NAME: userdata_mount.sub_path = f"deer-flow/users/{user_id}/threads/{thread_id}/user-data" mounts.append(userdata_mount) return mounts def _build_pod( sandbox_id: str, thread_id: str, user_id: str = DEFAULT_USER_ID, *, include_legacy_skills: bool = False, ) -> k8s_client.V1Pod: """Construct a Pod manifest for a single sandbox.""" return k8s_client.V1Pod( metadata=k8s_client.V1ObjectMeta( name=_pod_name(sandbox_id), namespace=K8S_NAMESPACE, labels={ "app": "deer-flow-sandbox", "sandbox-id": sandbox_id, "app.kubernetes.io/name": "deer-flow", "app.kubernetes.io/component": "sandbox", }, ), spec=k8s_client.V1PodSpec( containers=[ k8s_client.V1Container( name="sandbox", image=SANDBOX_IMAGE, image_pull_policy="IfNotPresent", ports=[ k8s_client.V1ContainerPort( name="http", container_port=SANDBOX_CONTAINER_PORT, protocol="TCP", ) ], readiness_probe=k8s_client.V1Probe( http_get=k8s_client.V1HTTPGetAction( path="/v1/sandbox", port=SANDBOX_CONTAINER_PORT, ), initial_delay_seconds=5, period_seconds=5, timeout_seconds=3, failure_threshold=3, ), liveness_probe=k8s_client.V1Probe( http_get=k8s_client.V1HTTPGetAction( path="/v1/sandbox", port=SANDBOX_CONTAINER_PORT, ), initial_delay_seconds=10, period_seconds=10, timeout_seconds=3, failure_threshold=3, ), resources=k8s_client.V1ResourceRequirements( requests={ "cpu": "100m", "memory": "256Mi", "ephemeral-storage": "500Mi", }, limits={ "cpu": "1000m", "memory": "1Gi", "ephemeral-storage": "500Mi", }, ), volume_mounts=_build_volume_mounts( thread_id, user_id=user_id, include_legacy_skills=include_legacy_skills, ), security_context=k8s_client.V1SecurityContext( privileged=False, allow_privilege_escalation=True, ), ) ], volumes=_build_volumes( thread_id, user_id=user_id, include_legacy_skills=include_legacy_skills, ), restart_policy="Always", ), ) def _build_service(sandbox_id: str) -> k8s_client.V1Service: """Construct a Service manifest for the configured access mode.""" return k8s_client.V1Service( metadata=k8s_client.V1ObjectMeta( name=_svc_name(sandbox_id), namespace=K8S_NAMESPACE, labels={ "app": "deer-flow-sandbox", "sandbox-id": sandbox_id, "app.kubernetes.io/name": "deer-flow", "app.kubernetes.io/component": "sandbox", }, ), spec=k8s_client.V1ServiceSpec( type=SANDBOX_SERVICE_TYPE, ports=[ k8s_client.V1ServicePort( name="http", port=SANDBOX_CONTAINER_PORT, target_port=SANDBOX_CONTAINER_PORT, protocol="TCP", ) ], selector={ "sandbox-id": sandbox_id, }, ), ) def _url_from_service(svc, sandbox_id: str) -> str | None: """Build the backend-facing sandbox URL from an already-fetched Service.""" if SANDBOX_SERVICE_TYPE == "ClusterIP": return _sandbox_url(sandbox_id) for port in svc.spec.ports or []: if port.name == "http" and port.node_port: return _sandbox_url(sandbox_id, node_port=port.node_port) return None def _sandbox_access_url(sandbox_id: str, *, tolerate_read_errors: bool = False) -> str | None: """Read the sandbox Service and return its backend-facing URL when ready.""" try: svc = core_v1.read_namespaced_service(_svc_name(sandbox_id), K8S_NAMESPACE) except ApiException as exc: if exc.status == 404: return None if tolerate_read_errors and exc.status not in {401, 403}: logger.warning( "Transient error reading Service %s: status=%s reason=%s", _svc_name(sandbox_id), exc.status, exc.reason, ) return None raise return _url_from_service(svc, sandbox_id) def _get_pod_phase(sandbox_id: str) -> str: """Return the Pod phase (Pending / Running / Succeeded / Failed / Unknown).""" try: pod = core_v1.read_namespaced_pod(_pod_name(sandbox_id), K8S_NAMESPACE) return pod.status.phase or "Unknown" except ApiException: return "NotFound" # ── API endpoints ──────────────────────────────────────────────────────── @app.get("/health") async def health(): """Provisioner health check.""" return {"status": "ok"} @app.post("/api/sandboxes", response_model=SandboxResponse) def create_sandbox(req: CreateSandboxRequest): """Create a sandbox Pod + Service for *sandbox_id*. If the sandbox already exists, returns the existing information (idempotent). """ sandbox_id = req.sandbox_id thread_id = req.thread_id user_id = req.user_id include_legacy_skills = req.include_legacy_skills logger.info( "Received request to create sandbox '%s' for thread '%s' user '%s' include_legacy_skills=%s", sandbox_id, thread_id, user_id, include_legacy_skills, ) # ── Fast path: sandbox already exists ──────────────────────────── existing_url = _sandbox_access_url(sandbox_id, tolerate_read_errors=True) if existing_url: return SandboxResponse( sandbox_id=sandbox_id, sandbox_url=existing_url, status=_get_pod_phase(sandbox_id), ) # ── Create Pod ─────────────────────────────────────────────────── try: core_v1.create_namespaced_pod( K8S_NAMESPACE, _build_pod( sandbox_id, thread_id, user_id=user_id, include_legacy_skills=include_legacy_skills, ), ) logger.info(f"Created Pod {_pod_name(sandbox_id)}") except ApiException as exc: if exc.status != 409: # 409 = AlreadyExists raise HTTPException(status_code=500, detail=f"Pod creation failed: {exc.reason}") # ── Create Service ─────────────────────────────────────────────── try: core_v1.create_namespaced_service(K8S_NAMESPACE, _build_service(sandbox_id)) logger.info(f"Created Service {_svc_name(sandbox_id)}") except ApiException as exc: if exc.status != 409: # Roll back the Pod on failure try: core_v1.delete_namespaced_pod(_pod_name(sandbox_id), K8S_NAMESPACE) except ApiException: pass raise HTTPException(status_code=500, detail=f"Service creation failed: {exc.reason}") # ── Wait until the Service has a usable access URL ─────────────── sandbox_url: str | None = None for _ in range(20): sandbox_url = _sandbox_access_url(sandbox_id, tolerate_read_errors=True) if sandbox_url: break time.sleep(0.5) if not sandbox_url: raise HTTPException(status_code=500, detail="Service access URL was not available in time") return SandboxResponse( sandbox_id=sandbox_id, sandbox_url=sandbox_url, status=_get_pod_phase(sandbox_id), ) @app.delete("/api/sandboxes/{sandbox_id}") def destroy_sandbox(sandbox_id: str): """Destroy a sandbox Pod + Service.""" errors: list[str] = [] # Delete Service try: core_v1.delete_namespaced_service(_svc_name(sandbox_id), K8S_NAMESPACE) logger.info(f"Deleted Service {_svc_name(sandbox_id)}") except ApiException as exc: if exc.status != 404: errors.append(f"service: {exc.reason}") # Delete Pod try: core_v1.delete_namespaced_pod(_pod_name(sandbox_id), K8S_NAMESPACE) logger.info(f"Deleted Pod {_pod_name(sandbox_id)}") except ApiException as exc: if exc.status != 404: errors.append(f"pod: {exc.reason}") if errors: raise HTTPException(status_code=500, detail=f"Partial cleanup: {', '.join(errors)}") return {"ok": True, "sandbox_id": sandbox_id} @app.get("/api/sandboxes/{sandbox_id}", response_model=SandboxResponse) def get_sandbox(sandbox_id: str): """Return current status and URL for a sandbox.""" sandbox_url = _sandbox_access_url(sandbox_id) if not sandbox_url: raise HTTPException(status_code=404, detail=f"Sandbox '{sandbox_id}' not found") return SandboxResponse( sandbox_id=sandbox_id, sandbox_url=sandbox_url, status=_get_pod_phase(sandbox_id), ) @app.get("/api/sandboxes") def list_sandboxes(): """List every sandbox currently managed in the namespace.""" try: services = core_v1.list_namespaced_service( K8S_NAMESPACE, label_selector="app=deer-flow-sandbox", ) except ApiException as exc: raise HTTPException(status_code=500, detail=f"Failed to list services: {exc.reason}") sandboxes: list[SandboxResponse] = [] for svc in services.items: sid = (svc.metadata.labels or {}).get("sandbox-id") if not sid: continue sandbox_url = _url_from_service(svc, sid) if not sandbox_url: continue sandboxes.append( SandboxResponse( sandbox_id=sid, sandbox_url=sandbox_url, status=_get_pod_phase(sid), ) ) return {"sandboxes": sandboxes, "count": len(sandboxes)}