Merge pull request #2085 from Cedric921/fix/chat-manifest-temperature

fix(local): send a curated chat model's publisher temperature

docs/architecture/local-models/catalog.md

@@ -662,7 +662,7 @@ and the screen in `download-chat-models.test.tsx`, `install-view.test.tsx` and t
 
 - Deleting the image or audio model in use removes its file while sd-server or audiocpp_server may still have it open. Untested on Windows, which refuses to delete an open file, so there the delete may fail until that server is stopped first.
 - Only the three audio defaults are validated; `validated` is empty on every other build.
-- `sampling`, `template.system_role` and llama.cpp's `run.args` are committed but nothing reads them, so chat does not use the publisher's sampling yet. sd.cpp's `image` defaults and `run.args` reach sd-server as launch flags. `template.tools` and `template.reasoning` reach a row's support, which the screen does not show.
+- Of `sampling`, chat sends only `temperature` ([`runtime.md`](runtime.md)); `top_p`, `top_k` and `min_p` wait on the `Generator` protocol carrying them. `template.system_role` and llama.cpp's `run.args` are committed but unread: chat asks the loaded template for its system role, and the router ignores per-model load arguments. sd.cpp's `image` defaults and `run.args` reach sd-server as launch flags. `template.tools` and `template.reasoning` reach a row's support, which the screen does not show.
 - The API does not cache search and nothing debounces typing: once the query has two characters, every keystroke sends a request, unless the renderer's 300 s cache holds that exact query.
 - A curated file that can no longer be fetched at its pinned commit, because the repo was deleted, gated or made private, gets the generic install error, and so does a checksum mismatch; nothing says which.
 - Nothing on the screen says whether sd-server is up: an image row reads In use as soon as it is chosen, while Electron starts sd-server on it only when a Studio job needs it. The hard-coded list's route reported that, and went with it.

docs/architecture/local-models/runtime.md

@@ -319,6 +319,11 @@ unconstrained, because losing a whole Studio format to a template quirk is worse
 than an answer the parser can still repair. Before sending, `for_template()`
 folds the system prompt into the first non-system turn for a template with no
 system role ([`selection.md`](selection.md)).
+A request that sets no temperature carries the one its curated entry commits
+for the mode it answers in: the `thinking` set when the template reasons and
+thinking is not turned off, else `non_thinking`, and nothing when that mode has
+no set, rather than the other mode's ([`sampling.py`](../../../surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/sampling.py)).
+A caller's own temperature, such as the title's zero, wins.
 
 There is no `pull()`. Fetching weights by name made sense when the runtime owned
 the download; here SurfSense fetches the GGUF itself, because an in-process fetch

surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/manifest_fields.py

@@ -13,12 +13,16 @@ class Template(BaseModel):
 
     tools: bool | None = None
     reasoning: bool | None = None
+    # Unread: chat asks the loaded model's own template (`capabilities.py`),
+    # which answers for the file actually running. Kept for a screen that shows it.
     system_role: bool | None = None
 
 
 class SamplingSet(BaseModel):
     model_config = STRICT
 
+    # Sent when the caller sets none (`llamacpp/sampling.py`). The other three wait
+    # on the `Generator` protocol carrying them, which every provider implements.
     temperature: float | None = None
     top_p: float | None = None
     top_k: int | None = None

surfsense_local/backend/modules/llm/catalog/local/engines/llamacpp/sampling.py

@@ -0,0 +1,25 @@
+"""The publisher's sampling for a curated chat model, as its entry commits it."""
+
+from collections.abc import Sequence
+
+from modules.llm.catalog.local.manifest import CuratedModel
+
+
+def publisher_temperature(
+    models: Sequence[CuratedModel], runtime_name: str, reasoning: bool | None
+) -> float | None:
+    """The temperature for the mode the model answers in, or None to keep the
+    runtime's default: a missing set is not filled from the other mode's."""
+    entry = next(
+        (
+            model
+            for model in models
+            if any(b.runtime_name == runtime_name for b in model.as_builds())
+        ),
+        None,
+    )
+    if entry is None or entry.sampling is None:
+        return None
+    thinks = reasoning is not False and entry.template.reasoning is True
+    chosen = entry.sampling.thinking if thinks else entry.sampling.non_thinking
+    return chosen.temperature if chosen is not None else None

surfsense_local/backend/modules/llm/catalog/local/manifest/schema.py

@@ -77,6 +77,8 @@ class ManifestBuild(BaseModel):
 
     quantization: str = Field(min_length=1)
     files: list[ManifestFile] = Field(min_length=1)
+    # sd-server's launch flags. Empty on every llama.cpp build and unread there:
+    # the router ignores per-model load arguments (runtime.md).
     run: RunArgs = Field(default_factory=RunArgs)
     validated: Validated = Field(default_factory=Validated)
 

surfsense_local/backend/modules/llm/catalog/local/service.py

@@ -16,6 +16,7 @@ from modules.llm.catalog.local.engines.llamacpp.engine import LlamaCppEngine
 from modules.llm.catalog.local.engines.llamacpp.models_folder.scan import (
     ProjectorNotice,
 )
+from modules.llm.catalog.local.engines.llamacpp.sampling import publisher_temperature
 from modules.llm.catalog.local.engines.sdcpp.engine import SdCppEngine
 from modules.llm.catalog.local.install import download
 from modules.llm.catalog.local.install.disk_room import refuse_without_room
@@ -191,6 +192,10 @@ class LocalCatalogService:
             repo, lambda build, tag: self._tickets.mint(build, pipeline_tag=tag)
         )
 
+    def publisher_temperature(self, model: str, reasoning: bool | None) -> float | None:
+        """What a curated chat model's publisher set for this mode, else None."""
+        return publisher_temperature(self._manifest.models, model, reasoning)
+
     # installing -------------------------------------------------------------
 
     def resolve_install(self, catalog_id: str) -> InstallPlan | None:

surfsense_local/backend/modules/llm/providers/__init__.py

@@ -4,10 +4,19 @@ from modules.llm.providers.llamacpp import PROVIDER, LlamaCppProvider
 from modules.llm.providers.protocols import Generator
 from shared.config import get_llm_settings
 
+
+def _publisher_temperature(model: str, reasoning: bool | None) -> float | None:
+    # Imported here: the catalog imports this package for the runtime's name.
+    from modules.llm.catalog.local.dependencies import get_local_catalog
+
+    return get_local_catalog().publisher_temperature(model, reasoning)
+
+
 REGISTRY: dict[str, Callable[[], Generator]] = {
     PROVIDER: lambda: LlamaCppProvider(
         get_llm_settings().llamacpp_base_url,
         get_llm_settings().llamacpp_models_dir,
+        publisher_temperature=_publisher_temperature,
     ),
 }
 

surfsense_local/backend/modules/llm/providers/llamacpp/provider.py

@@ -23,7 +23,7 @@ concern; this adapter answers questions and reports what is on disk.
 """
 
 import logging
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Callable
 from pathlib import Path
 
 import httpx
@@ -58,9 +58,12 @@ class LlamaCppProvider:
         models_dir: Path | None = None,
         *,
         transport: httpx.BaseTransport | None = None,
+        publisher_temperature: Callable[[str, bool | None], float | None] | None = None,
     ) -> None:
         self._base_url = base_url.rstrip("/")
         self._models_dir = models_dir
+        # A curated model's reviewed setting, for a caller that chose none.
+        self._publisher_temperature = publisher_temperature
         self._router = RouterClient(self._base_url, transport=transport)
         # `/props` describes a resident model and nothing about it changes
         # between turns, so it is read once per load rather than once per
@@ -224,6 +227,8 @@ class LlamaCppProvider:
         # Downgrade at the seam: `modules/chat` assembles one conversation and
         # never learns that templates differ.
         shaped = for_template(messages, await self.capabilities(model))
+        if temperature is None and self._publisher_temperature is not None:
+            temperature = self._publisher_temperature(model, reasoning)
         try:
             async for delta in self._chat.chat_deltas(
                 model,

surfsense_local/backend/tests/integration/chat/test_chat.py

@@ -469,6 +469,23 @@ async def test_a_reply_with_no_text_leaves_no_trace(
     assert threads[0]["title"] == "New chat"
 
 
+async def test_a_curated_model_answers_at_its_publishers_temperature(
+    client: AsyncClient,
+    engine: Engine,
+    real_model: object,
+    llamacpp_server: list[dict],
+) -> None:
+    """Qwen3's entry commits 0.6 for thinking; the title keeps its own zero."""
+    workspace_id, _ids = _seed(engine)
+    thread_id = await _open_thread(client, workspace_id)
+
+    await _send(client, thread_id, "how did revenue move?")
+
+    title, answer = sorted(llamacpp_server, key=lambda r: r.get("max_tokens") != 12)
+    assert title["temperature"] == 0
+    assert answer["temperature"] == 0.6
+
+
 async def test_a_thread_with_no_model_selected_is_a_409(client: AsyncClient) -> None:
     """Refused before retrieval, so the frontend can route the user to setup."""
     workspace = (await client.post("/workspaces", json={"name": "w"})).json()

surfsense_local/backend/tests/unit/llm/catalog/local/test_llamacpp_sampling.py

@@ -0,0 +1,41 @@
+"""A curated chat model answers at its publisher's temperature for the mode it is in."""
+
+import pytest
+
+from modules.llm.catalog.local.engines.llamacpp.sampling import publisher_temperature
+from modules.llm.catalog.local.manifest import load_local_manifest
+
+pytestmark = pytest.mark.unit
+
+
+def runtime_name(model_id: str) -> str:
+    """What llama-server calls the entry's first build once it is installed."""
+    model = next(m for m in load_local_manifest().models if m.id == model_id)
+    return model.as_builds()[0].runtime_name
+
+
+@pytest.mark.parametrize(
+    ("model_id", "reasoning", "expected"),
+    [
+        # Qwen3 thinks by default, and its entry commits the thinking set.
+        ("qwen3-4b", None, 0.6),
+        # With thinking turned off it answers in a mode its entry has no set for.
+        ("qwen3-4b", False, None),
+        # Gemma 3 never thinks, so its only set is the one it answers in.
+        ("gemma-3-4b", None, 1.0),
+    ],
+)
+def test_the_set_follows_the_mode_the_model_answers_in(
+    model_id: str, reasoning: bool | None, expected: float | None
+) -> None:
+    """Sending the other mode's settings would be a guess the publisher never made."""
+    models = load_local_manifest().models
+
+    assert publisher_temperature(models, runtime_name(model_id), reasoning) == expected
+
+
+def test_a_model_no_entry_names_keeps_the_runtimes_default() -> None:
+    """A searched or hand-copied file has no reviewed settings to send."""
+    assert (
+        publisher_temperature(load_local_manifest().models, "my-own-q4", None) is None
+    )

surfsense_local/backend/tests/unit/llm/providers/llamacpp/test_publisher_sampling.py

@@ -0,0 +1,41 @@
+"""The runtime sends a curated model's temperature unless the caller chose one."""
+
+import pytest
+
+from modules.llm.providers.llamacpp import LlamaCppProvider
+from modules.llm.providers.types import Message
+from tests.unit.llm.providers.llamacpp.fake_router import FakeRouter
+
+pytestmark = pytest.mark.unit
+
+
+def provider(fake: FakeRouter) -> LlamaCppProvider:
+    """A runtime whose curated entry commits a temperature of 0.6."""
+    return LlamaCppProvider(
+        "http://127.0.0.1:1234",
+        transport=fake.transport(),
+        publisher_temperature=lambda model, reasoning: 0.6,
+    )
+
+
+async def test_a_request_carries_the_publishers_temperature() -> None:
+    """Chat sets none of its own, so the entry's reviewed setting applies."""
+    fake = FakeRouter(["qwen3"])
+
+    _ = [c async for c in provider(fake).chat("qwen3", [Message("user", "hi")])]
+
+    assert fake.chat_bodies[0]["temperature"] == 0.6
+
+
+async def test_a_callers_own_temperature_wins() -> None:
+    """A title is asked for at zero to be the same every time."""
+    fake = FakeRouter(["qwen3"])
+
+    _ = [
+        c
+        async for c in provider(fake).chat(
+            "qwen3", [Message("user", "hi")], temperature=0
+        )
+    ]
+
+    assert fake.chat_bodies[0]["temperature"] == 0