Studio: keep pandas out of the backend startup import graph (#8962)

* Studio: keep pandas out of the backend startup import graph

* Hold the resolved pandas module in a global so a repeat call is not an import statement

* Resolve the unstructured seed plugin on first use, not at startup

Importing data_designer_unstructured_seed.chunking runs the package __init__
first, and that re-exports .config and .impl, which import the data designer
engine, which imports pandas and pyarrow. Dropping the module-scope import
inside chunking.py therefore left the whole cost in import main's graph
wherever the plugin is installed: measured 534ms cumulative for
routes.data_recipe.seed, with pandas and pyarrow both loaded.

The route resolves the plugin through _chunking() on first use instead. None
still means "not installed", so the existing 500 and the raw-text fallback
are unchanged, and the failed probe is remembered rather than retried per
request.

The runtime guards in test_startup_defers_torch.py pass vacuously wherever
the optional plugin is not installed, which is most CI jobs, so add a
source-level guard that reads seed.py and fails on a module-scope import of
the plugin either way. test_data_recipe_seed.py covers the availability
cases: unavailable without the plugin on both preview entry points, probed
once, raw text fallback, and the installed path.

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Tighten the comments on the deferred imports

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Tighten comments in the deferred pandas import paths

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: oobabooga <112222186+oobabooga@users.noreply.github.com>
Co-authored-by: danielhanchen <unslothshared@gmail.com>

studio/backend/plugins/data-designer-unstructured-seed/src/data_designer_unstructured_seed/chunking.py

@@ -8,8 +8,6 @@ import re
 from pathlib import Path
 from typing import Any
 
-import pandas as pd
-
 from utils.paths import ensure_dir, unstructured_seed_cache_root
 
 DEFAULT_CHUNK_SIZE = 1200
@@ -18,6 +16,22 @@ MAX_CHUNK_SIZE = 20000
 _MIN_BREAK_RATIO = 0.6
 _CACHE_DIR = unstructured_seed_cache_root()
 
+_PANDAS = None
+
+
+def _pandas():
+    # Imported on use, not at module scope, so importing this module costs nothing.
+    global _PANDAS
+    if _PANDAS is None:
+        try:
+            import pandas as pd
+        except ImportError as exc:  # pragma: no cover
+            raise RuntimeError(
+                f"pandas is required for unstructured seed processing: {exc}"
+            ) from exc
+        _PANDAS = pd
+    return _PANDAS
+
 
 def resolve_chunking(chunk_size: Any, chunk_overlap: Any) -> tuple[int, int]:
     size = _to_int(chunk_size, DEFAULT_CHUNK_SIZE)
@@ -39,12 +53,7 @@ def build_unstructured_preview_rows(
     if rows:
         return rows[:count]
 
-    try:
-        import pandas as pd
-    except ImportError as exc:  # pragma: no cover
-        raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
-
-    dataframe = pd.read_parquet(parquet_path).head(count)
+    dataframe = _pandas().read_parquet(parquet_path).head(count)
     return [
         {"chunk_text": str(value.get("chunk_text", "")).strip()}
         for value in dataframe.to_dict(orient = "records")
@@ -130,13 +139,8 @@ def materialize_unstructured_seed_dataset(
 
     rows = [{"chunk_text": chunk} for chunk in chunks]
     ensure_dir(_CACHE_DIR)
-    try:
-        import pandas as pd
-    except ImportError as exc:  # pragma: no cover
-        raise RuntimeError(f"pandas is required for unstructured seed processing: {exc}") from exc
-
     tmp_path = _CACHE_DIR / f"{key}.tmp.parquet"
-    pd.DataFrame(rows).to_parquet(tmp_path, index = False)
+    _pandas().DataFrame(rows).to_parquet(tmp_path, index = False)
     tmp_path.replace(parquet_path)
     return parquet_path, rows
 
@@ -152,7 +156,7 @@ def materialize_multi_file_unstructured_seed(
     cache_key = _compute_multi_file_cache_key(file_entries, chunk_size, chunk_overlap)
     cached = _CACHE_DIR / f"{cache_key}.parquet"
     if cached.exists():
-        df = pd.read_parquet(cached)
+        df = _pandas().read_parquet(cached)
         rows = df.to_dict(orient = "records")
         return cached, rows
 
@@ -170,7 +174,7 @@ def materialize_multi_file_unstructured_seed(
     if not all_rows:
         raise ValueError("No text found in any uploaded files.")
 
-    df = pd.DataFrame(all_rows)
+    df = _pandas().DataFrame(all_rows)
     ensure_dir(_CACHE_DIR)
     tmp = _CACHE_DIR / f"{cache_key}.tmp.parquet"
     df.to_parquet(tmp, index = False)

studio/backend/routes/data_recipe/seed.py

@@ -18,18 +18,6 @@ from uuid import uuid4
 
 from fastapi import APIRouter, HTTPException, UploadFile, File as FastAPIFile, Form
 
-try:
-    from data_designer_unstructured_seed.chunking import (
-        build_multi_file_preview_rows,
-        build_unstructured_preview_rows,
-        normalize_unstructured_text,
-        resolve_chunking,
-    )
-except ImportError:
-    build_multi_file_preview_rows = None
-    build_unstructured_preview_rows = None
-    normalize_unstructured_text = None
-    resolve_chunking = None
 from core.data_recipe.jsonable import to_preview_jsonable
 from loggers import get_logger
 from utils.paths import ensure_dir, seed_uploads_root, unstructured_uploads_root
@@ -53,6 +41,24 @@ from models.data_recipe import (
 logger = get_logger(__name__)
 router = APIRouter()
 
+# Resolved on first use, not at module scope: the plugin package pulls the data
+# designer engine, pandas and pyarrow, delaying uvicorn binding the port.
+# False means "probed once, not installed", so callers still just see None.
+_CHUNKING: Any = None
+
+
+def _chunking() -> Any:
+    global _CHUNKING
+    if _CHUNKING is None:
+        try:
+            from data_designer_unstructured_seed import chunking
+        except ImportError:
+            _CHUNKING = False
+        else:
+            _CHUNKING = chunking
+    return _CHUNKING or None
+
+
 DATA_EXTS = (".parquet", ".jsonl", ".json", ".csv")
 DEFAULT_SPLIT = "train"
 LOCAL_UPLOAD_EXTS = {".csv", ".json", ".jsonl"}
@@ -251,14 +257,15 @@ def _read_preview_rows_from_local_file(path: Path, preview_size: int) -> list[di
 def _read_preview_rows_from_unstructured_file(
     *, path: Path, preview_size: int, chunk_size: int | None, chunk_overlap: int | None
 ) -> list[dict[str, Any]]:
-    if resolve_chunking is None or build_unstructured_preview_rows is None:
+    chunking = _chunking()
+    if chunking is None:
         raise HTTPException(
             500,
             "Unstructured seed support not available (missing data_designer_unstructured_seed)",
         )
-    size, overlap = resolve_chunking(chunk_size, chunk_overlap)
+    size, overlap = chunking.resolve_chunking(chunk_size, chunk_overlap)
     try:
-        rows = build_unstructured_preview_rows(
+        rows = chunking.build_unstructured_preview_rows(
             source_path = path,
             preview_size = preview_size,
             chunk_size = size,
@@ -284,7 +291,8 @@ def _read_preview_rows_from_multi_files(
     chunk_size: int | None,
     chunk_overlap: int | None,
 ) -> list[dict[str, str]]:
-    if build_multi_file_preview_rows is None:
+    chunking = _chunking()
+    if chunking is None:
         raise HTTPException(
             500,
             "Unstructured seed support not available (missing data_designer_unstructured_seed)",
@@ -299,7 +307,7 @@ def _read_preview_rows_from_multi_files(
             raise HTTPException(404, f"Extracted text not found for file: {fname} (id: {fid})")
         file_entries.append((extracted, fname))
 
-    return build_multi_file_preview_rows(
+    return chunking.build_multi_file_preview_rows(
         file_entries = file_entries,
         preview_size = preview_size,
         chunk_size = chunk_size,
@@ -414,9 +422,10 @@ def _extract_text_from_file(file_path: Path, ext: str) -> str:
     else:
         raise ValueError(f"Unsupported file type: {ext}")
 
-    if normalize_unstructured_text is None:
+    chunking = _chunking()
+    if chunking is None:
         return raw
-    return normalize_unstructured_text(raw)
+    return chunking.normalize_unstructured_text(raw)
 
 
 def _get_block_total_size(block_dir: Path) -> int:

studio/backend/tests/test_data_recipe_seed.py

@@ -221,3 +221,95 @@ def test_total_upload_quota_is_scoped_per_block(monkeypatch, tmp_path):
     # Another block starts with its own untouched budget.
     other = _run_upload(seed_route, "c.txt", b"123", block_id = "other")
     assert other.status == "ok"
+
+
+class _BlockPlugin:
+    """Meta path finder making the optional seed plugin look uninstalled."""
+
+    def __init__(self, name: str = "data_designer_unstructured_seed"):
+        self.name = name
+        self.attempts = 0
+
+    def find_spec(
+        self,
+        fullname,
+        path = None,
+        target = None,
+    ):
+        if fullname == self.name or fullname.startswith(self.name + "."):
+            self.attempts += 1
+            raise ModuleNotFoundError(f"No module named {fullname!r}", name = fullname)
+        return None
+
+
+def _without_plugin(monkeypatch, seed_route):
+    import sys
+
+    blocker = _BlockPlugin()
+    monkeypatch.setattr(sys, "meta_path", [blocker, *sys.meta_path])
+    for name in [m for m in sys.modules if m.split(".")[0] == blocker.name]:
+        monkeypatch.delitem(sys.modules, name)
+    seed_route._CHUNKING = None
+    return blocker
+
+
+def test_unstructured_preview_reports_unavailable_without_the_plugin(monkeypatch, tmp_path):
+    """Deferring the plugin import must not change what a missing plugin looks like."""
+    seed_route = _load_seed_route(monkeypatch, tmp_path)
+    _without_plugin(monkeypatch, seed_route)
+
+    assert seed_route._chunking() is None
+
+    with pytest.raises(seed_route.HTTPException) as exc:
+        seed_route._read_preview_rows_from_unstructured_file(
+            path = tmp_path / "a.txt", preview_size = 5, chunk_size = None, chunk_overlap = None
+        )
+    assert exc.value.status_code == 500
+    assert "Unstructured seed support not available" in exc.value.detail
+
+    with pytest.raises(seed_route.HTTPException) as exc:
+        seed_route._read_preview_rows_from_multi_files(
+            block_id = "block",
+            file_ids = ["a"],
+            file_names = ["a.txt"],
+            preview_size = 5,
+            chunk_size = None,
+            chunk_overlap = None,
+        )
+    assert exc.value.status_code == 500
+    assert "Unstructured seed support not available" in exc.value.detail
+
+
+def test_missing_plugin_is_probed_once(monkeypatch, tmp_path):
+    """A failed probe is remembered, so previews do not retry the import every time."""
+    seed_route = _load_seed_route(monkeypatch, tmp_path)
+    blocker = _without_plugin(monkeypatch, seed_route)
+
+    assert seed_route._chunking() is None
+    assert seed_route._chunking() is None
+    assert blocker.attempts == 1
+
+
+def test_text_extraction_falls_back_to_raw_without_the_plugin(monkeypatch, tmp_path):
+    """normalize_unstructured_text lives in the plugin; without it raw text stands."""
+    seed_route = _load_seed_route(monkeypatch, tmp_path)
+    _without_plugin(monkeypatch, seed_route)
+    source = tmp_path / "notes.txt"
+    source.write_text("a\n\n\n\nb", encoding = "utf-8")
+
+    # The plugin is what collapses the run of blank lines.
+    assert seed_route._extract_text_from_file(source, ".txt") == "a\n\n\n\nb"
+
+
+def test_plugin_resolution_survives_a_reload_and_normalizes(monkeypatch, tmp_path):
+    """With the plugin installed the same call sites still go through it."""
+    pytest.importorskip("data_designer_unstructured_seed")
+    seed_route = _load_seed_route(monkeypatch, tmp_path)
+    seed_route._CHUNKING = None
+
+    chunking = seed_route._chunking()
+    assert chunking is not None
+    assert chunking.resolve_chunking(0, 0)[0] == 1
+    source = tmp_path / "notes.txt"
+    source.write_text("a\n\n\n\nb", encoding = "utf-8")
+    assert seed_route._extract_text_from_file(source, ".txt") == "a\n\nb"

studio/backend/tests/test_startup_defers_torch.py

@@ -1,8 +1,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
 
-"""Invariant: ``import main`` must not import torch, and the warm that replaces
-it must be safe.
+"""Invariant: ``import main`` must not import torch or pandas, and the warm that
+replaces torch must be safe.
 
 uvicorn binds the socket only after ``import main`` and the lifespan both finish, so
 anything they import is time the login screen does not exist. torch and what it drags
@@ -13,9 +13,24 @@ in was about 5s of that on a GPU host. Four eager edges caused it:
   core/inference/orchestrator.py  from utils.hf_xet_fallback import DownloadStallError
   utils/datasets/raw_text.py    from datasets import Dataset  (annotation only)
 
-All four are lazy now and detection moved onto utils/torch_warmup.py. A fresh interpreter
-is used for the import invariant, since importing in-process would measure an already-warm
-sys.modules. CPU-only, no network, no GPU, no weights.
+pandas arrived by a fifth, through the data-recipe seed route:
+
+  routes/data_recipe/seed.py    from data_designer_unstructured_seed.chunking import ...
+  ...chunking.py                import pandas as pd  at module scope
+  ...__init__.py                re-exports .config and .impl, which import the data
+                                designer engine, which imports pandas and pyarrow
+
+Importing the submodule runs the package first, so dropping only the chunking-level
+import left the cost in place. The route resolves the plugin on first use now. The
+Startup profile workflow measured this edge at 2.247s of a 7.284s ``import main`` on
+windows-latest, 901ms self on macos-15.
+
+All of them are lazy. A fresh interpreter is used for the import invariant, since
+importing in-process would measure an already-warm sys.modules. CPU-only, no network,
+no GPU, no weights.
+
+The runtime guards only bite where the optional plugin is installed, so a source-level
+guard covers the environments that do not have it.
 """
 
 from __future__ import annotations
@@ -31,7 +46,7 @@ import pytest
 
 _BACKEND_DIR = Path(__file__).resolve().parent.parent  # studio/backend
 
-_HEAVY = ("torch", "transformers", "unsloth_zoo", "scipy", "sklearn", "sympy")
+_HEAVY = ("torch", "transformers", "unsloth_zoo", "scipy", "sklearn", "sympy", "pandas", "pyarrow")
 
 _IMPORT_MAIN_SNIPPET = r"""
 import sys
@@ -81,10 +96,11 @@ def test_import_main_does_not_import_torch():
         "routes.models",
         "utils.hf_xet_fallback",
         "core.rag.embeddings",
+        "routes.data_recipe.seed",
     ],
 )
 def test_module_import_does_not_pull_torch(module_path: str):
-    """Each module that used to force torch must import clean on its own."""
+    """Each module that used to force torch or pandas must import clean on its own."""
     snippet = (
         "import sys, importlib\n"
         f"importlib.import_module({module_path!r})\n"
@@ -99,6 +115,43 @@ def test_module_import_does_not_pull_torch(module_path: str):
     assert "CLEAN" in proc.stdout
 
 
+def _module_scope_imports(tree: ast.Module) -> list[ast.stmt]:
+    """Imports that run on `import <module>`: module body and nesting like try/if,
+    but nothing inside a function or class."""
+    found: list[ast.stmt] = []
+    stack: list[ast.stmt] = list(tree.body)
+    while stack:
+        node = stack.pop()
+        if isinstance(node, (ast.Import, ast.ImportFrom)):
+            found.append(node)
+        elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+            continue
+        else:
+            stack.extend(c for c in ast.iter_child_nodes(node) if isinstance(c, ast.stmt))
+    return found
+
+
+@pytest.mark.parametrize(
+    ("rel_path", "banned"),
+    [("routes/data_recipe/seed.py", "data_designer_unstructured_seed")],
+)
+def test_module_scope_does_not_import_the_seed_plugin(rel_path: str, banned: str):
+    """The runtime guards above pass vacuously wherever the plugin is not installed,
+    which is most CI jobs. This one reads the source, so it holds either way."""
+    tree = ast.parse((_BACKEND_DIR / rel_path).read_text(encoding = "utf-8"))
+    offenders = [
+        node.lineno
+        for node in _module_scope_imports(tree)
+        if (getattr(node, "module", None) or "").startswith(banned)
+        or any(alias.name.startswith(banned) for alias in node.names)
+    ]
+    assert not offenders, (
+        f"{rel_path} imports {banned} at module scope (line(s) {offenders}). The package "
+        "re-exports the data designer engine, so this puts pandas and pyarrow back into "
+        "main's startup graph. Resolve it on first use instead."
+    )
+
+
 def test_detection_sets_still_resolve_under_their_old_names():
     """The PEP 562 shim must keep `from ... import _VLM_MODEL_TYPES` working."""
     if importlib.util.find_spec("transformers") is None: