open-notebook/tests/test_models_api.py
Luis Novo b83f1d61e6
fix(models): stop auto-assign from re-filling cleared optional defaults (#1186)
* fix(models): stop auto-assign from re-filling cleared optional defaults

Auto-assign treated every empty default slot as "missing" and filled it,
so an optional slot a user deliberately cleared (to fall back to the chat
model) got silently re-populated on the next run, undoing the intent.

- Auto-assign now fills only the required slots (chat, embedding); the
  optional slots (transformation, tools, large context, TTS, STT) are
  left untouched.
- get_default_model("large_context") now falls back to the chat model
  when unset, matching transformation/tools (TTS/STT still return None).
- Settings UI shows an inline hint on each empty optional slot: the text
  slots show "using chat model (<name>)"; TTS/STT show a not-configured
  hint. Required slots remain non-clearable. New i18n keys across all 14
  locales.

Closes #1098

* test: guard await_args against None for mypy
2026-07-19 18:06:17 -03:00

620 lines
23 KiB
Python

from unittest.mock import AsyncMock, patch
import pytest
from fastapi.testclient import TestClient
@pytest.fixture
def client():
"""Create test client after environment variables have been cleared by conftest."""
from api.main import app
return TestClient(app)
class TestModelCreation:
"""Test suite for Model Creation endpoint."""
@pytest.mark.asyncio
@patch("open_notebook.database.repository.repo_query")
@patch("api.routers.models.Model.save")
async def test_create_duplicate_model_same_case(
self, mock_save, mock_repo_query, client
):
"""Test that creating a duplicate model with same case returns 400."""
# Mock repo_query to return a duplicate model
mock_repo_query.return_value = [
{
"id": "model:123",
"name": "gpt-4",
"provider": "openai",
"type": "language",
}
]
# Attempt to create duplicate
response = client.post(
"/api/models",
json={"name": "gpt-4", "provider": "openai", "type": "language"},
)
assert response.status_code == 400
assert (
response.json()["detail"]
== "Model 'gpt-4' already exists for provider 'openai' with type 'language'"
)
@pytest.mark.asyncio
@patch("open_notebook.database.repository.repo_query")
@patch("api.routers.models.Model.save")
async def test_create_duplicate_model_different_case(
self, mock_save, mock_repo_query, client
):
"""Test that creating a duplicate model with different case returns 400."""
# Mock repo_query to return a duplicate model (case-insensitive match)
mock_repo_query.return_value = [
{
"id": "model:123",
"name": "gpt-4",
"provider": "openai",
"type": "language",
}
]
# Attempt to create duplicate with different case
response = client.post(
"/api/models",
json={"name": "GPT-4", "provider": "OpenAI", "type": "language"},
)
assert response.status_code == 400
assert (
response.json()["detail"]
== "Model 'GPT-4' already exists for provider 'OpenAI' with type 'language'"
)
@pytest.mark.asyncio
@patch("open_notebook.database.repository.repo_query")
async def test_create_same_model_name_different_provider(
self, mock_repo_query, client
):
"""Test that creating a model with same name but different provider is allowed."""
from open_notebook.ai.models import Model
# Mock repo_query to return empty (no duplicate found for different provider)
mock_repo_query.return_value = []
# Patch the save method on the Model class
with patch.object(Model, "save", new_callable=AsyncMock):
# Attempt to create same model name with different provider (anthropic)
response = client.post(
"/api/models",
json={"name": "gpt-4", "provider": "anthropic", "type": "language"},
)
# Should succeed because provider is different
assert response.status_code == 200
@pytest.mark.asyncio
@patch("open_notebook.database.repository.repo_query")
async def test_create_same_model_name_different_type(self, mock_repo_query, client):
"""Test that creating a model with same name but different type is allowed."""
from open_notebook.ai.models import Model
# Mock repo_query to return empty (no duplicate found for different type)
mock_repo_query.return_value = []
# Patch the save method on the Model class
with patch.object(Model, "save", new_callable=AsyncMock):
# Attempt to create same model name with different type (embedding instead of language)
response = client.post(
"/api/models",
json={"name": "gpt-4", "provider": "openai", "type": "embedding"},
)
# Should succeed because type is different
assert response.status_code == 200
class TestModelsProviderAvailability:
"""Test suite for Models Provider Availability endpoint."""
@patch(
"api.routers.models._check_provider_has_credential",
new_callable=AsyncMock,
return_value=False,
)
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_blank_anthropic_compatible_env_vars_are_unavailable(
self, mock_esperanto, mock_env, mock_has_credential, client
):
def env_side_effect(key):
if key in {
"ANTHROPIC_COMPATIBLE_BASE_URL",
"ANTHROPIC_COMPATIBLE_API_KEY",
}:
return " "
return None
mock_env.side_effect = env_side_effect
mock_esperanto.return_value = {"language": ["anthropic"]}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
assert "anthropic_compatible" not in data["available"]
assert "anthropic_compatible" in data["unavailable"]
assert "anthropic_compatible" not in data["supported_types"]
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_generic_env_var_enables_all_modes(self, mock_esperanto, mock_env, client):
"""Test that OPENAI_COMPATIBLE_BASE_URL enables all 4 modes."""
# Mock environment: only generic var is set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL":
return "http://localhost:1234/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# openai-compatible should be available
assert "openai_compatible" in data["available"]
# Should support all 4 types
assert "openai_compatible" in data["supported_types"]
supported = data["supported_types"]["openai_compatible"]
assert "language" in supported
assert "embedding" in supported
assert "speech_to_text" in supported
assert "text_to_speech" in supported
assert len(supported) == 4
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_mode_specific_env_vars_llm_embedding(
self, mock_esperanto, mock_env, client
):
"""Test mode-specific env vars (LLM + EMBEDDING) enable only those 2 modes."""
# Mock environment: only LLM and EMBEDDING specific vars are set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
return "http://localhost:1234/v1"
if key == "OPENAI_COMPATIBLE_BASE_URL_EMBEDDING":
return "http://localhost:8080/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# openai-compatible should be available
assert "openai_compatible" in data["available"]
# Should support only language and embedding
assert "openai_compatible" in data["supported_types"]
supported = data["supported_types"]["openai_compatible"]
assert "language" in supported
assert "embedding" in supported
assert "speech_to_text" not in supported
assert "text_to_speech" not in supported
assert len(supported) == 2
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_no_env_vars_set(self, mock_esperanto, mock_env, client):
"""Test that openai-compatible is not available when no env vars are set."""
# Mock environment: no openai-compatible vars are set
def env_side_effect(key):
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# openai-compatible should NOT be available
assert "openai_compatible" not in data["available"]
assert "openai_compatible" in data["unavailable"]
# Should not have supported_types entry
assert "openai_compatible" not in data["supported_types"]
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_mixed_config_generic_and_mode_specific(
self, mock_esperanto, mock_env, client
):
"""Test mixed config: generic + mode-specific (generic should enable all)."""
# Mock environment: both generic and mode-specific vars are set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL":
return "http://localhost:1234/v1"
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
return "http://localhost:5678/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# openai-compatible should be available
assert "openai_compatible" in data["available"]
# Generic var enables all, so all 4 should be supported
assert "openai_compatible" in data["supported_types"]
supported = data["supported_types"]["openai_compatible"]
assert "language" in supported
assert "embedding" in supported
assert "speech_to_text" in supported
assert "text_to_speech" in supported
assert len(supported) == 4
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_individual_mode_llm_only(self, mock_esperanto, mock_env, client):
"""Test individual mode-specific var (LLM only)."""
# Mock environment: only LLM specific var is set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL_LLM":
return "http://localhost:1234/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# Should support only language
supported = data["supported_types"]["openai_compatible"]
assert supported == ["language"]
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_individual_mode_embedding_only(self, mock_esperanto, mock_env, client):
"""Test individual mode-specific var (EMBEDDING only)."""
# Mock environment: only EMBEDDING specific var is set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL_EMBEDDING":
return "http://localhost:8080/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# Should support only embedding
supported = data["supported_types"]["openai_compatible"]
assert supported == ["embedding"]
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_individual_mode_stt_only(self, mock_esperanto, mock_env, client):
"""Test individual mode-specific var (STT only)."""
# Mock environment: only STT specific var is set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL_STT":
return "http://localhost:9000/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# Should support only speech_to_text
supported = data["supported_types"]["openai_compatible"]
assert supported == ["speech_to_text"]
@patch("api.routers.models.os.environ.get")
@patch("api.routers.models.AIFactory.get_available_providers")
def test_individual_mode_tts_only(self, mock_esperanto, mock_env, client):
"""Test individual mode-specific var (TTS only)."""
# Mock environment: only TTS specific var is set
def env_side_effect(key):
if key == "OPENAI_COMPATIBLE_BASE_URL_TTS":
return "http://localhost:9000/v1"
return None
mock_env.side_effect = env_side_effect
# Mock Esperanto response
mock_esperanto.return_value = {
"language": ["openai-compatible"],
"embedding": ["openai-compatible"],
"speech_to_text": ["openai-compatible"],
"text_to_speech": ["openai-compatible"],
}
response = client.get("/api/models/providers")
assert response.status_code == 200
data = response.json()
# Should support only text_to_speech
supported = data["supported_types"]["openai_compatible"]
assert supported == ["text_to_speech"]
class TestUpdateDefaultModels:
"""PUT /models/defaults must distinguish 'field absent' (keep) from
'field explicitly null' (clear).
The handler used `is not None` guards, so a null sent to clear a default
was silently ignored — the old value survived while the client saw
success (#1091, same anti-pattern fixed for credentials in #1046). Now
keyed on model_fields_set, with the required defaults (chat, embedding)
rejecting explicit nulls.
"""
def _mock_defaults(self):
from unittest.mock import MagicMock
defaults = MagicMock()
defaults.default_chat_model = "model:chat"
defaults.default_transformation_model = "model:transform"
defaults.large_context_model = None
defaults.default_text_to_speech_model = "model:tts"
defaults.default_speech_to_text_model = None
defaults.default_embedding_model = "model:embed"
defaults.default_tools_model = "model:tools"
defaults.update = AsyncMock()
return defaults
def _put(self, client, defaults, body):
with patch(
"api.routers.models.DefaultModels.get_instance",
new=AsyncMock(return_value=defaults),
):
return client.put("/api/models/defaults", json=body)
def test_explicit_null_clears_optional_default(self, client):
defaults = self._mock_defaults()
response = self._put(client, defaults, {"default_transformation_model": None})
assert response.status_code == 200
assert defaults.default_transformation_model is None
defaults.update.assert_awaited_once()
assert response.json()["default_transformation_model"] is None
def test_absent_field_keeps_current_value(self, client):
defaults = self._mock_defaults()
response = self._put(client, defaults, {"default_tools_model": "model:new-tools"})
assert response.status_code == 200
assert defaults.default_tools_model == "model:new-tools"
# Not in the payload -> untouched (JSON null semantics must not leak in)
assert defaults.default_transformation_model == "model:transform"
assert response.json()["default_transformation_model"] == "model:transform"
def test_explicit_null_on_required_default_is_rejected(self, client):
for field in ("default_chat_model", "default_embedding_model"):
defaults = self._mock_defaults()
response = self._put(client, defaults, {field: None})
assert response.status_code == 400, field
assert field in response.json()["detail"]
defaults.update.assert_not_awaited()
def test_required_default_can_still_be_reassigned(self, client):
defaults = self._mock_defaults()
response = self._put(client, defaults, {"default_chat_model": "model:new-chat"})
assert response.status_code == 200
assert defaults.default_chat_model == "model:new-chat"
defaults.update.assert_awaited_once()
class TestAutoAssignDefaults:
"""POST /models/auto-assign must only fill the two REQUIRED slots
(chat, embedding). Optional slots that a user deliberately cleared must
stay empty so auto-assign doesn't silently undo that intent (#1098).
"""
def _mock_defaults(self):
from unittest.mock import MagicMock
defaults = MagicMock()
# Everything empty, including the optional slots a user may have cleared.
defaults.default_chat_model = None
defaults.default_embedding_model = None
defaults.default_transformation_model = None
defaults.default_tools_model = None
defaults.large_context_model = None
defaults.default_text_to_speech_model = None
defaults.default_speech_to_text_model = None
defaults.update = AsyncMock()
return defaults
def _models(self):
return [
{"id": "model:lang", "provider": "openai", "name": "gpt-4o", "type": "language"},
{"id": "model:embed", "provider": "openai", "name": "text-embedding-3", "type": "embedding"},
{"id": "model:tts", "provider": "openai", "name": "tts-1", "type": "text_to_speech"},
{"id": "model:stt", "provider": "openai", "name": "whisper-1", "type": "speech_to_text"},
]
def _post(self, client, defaults):
with patch(
"api.routers.models.DefaultModels.get_instance",
new=AsyncMock(return_value=defaults),
), patch(
"open_notebook.database.repository.repo_query",
new=AsyncMock(return_value=self._models()),
):
return client.post("/api/models/auto-assign")
def test_fills_only_required_slots(self, client):
defaults = self._mock_defaults()
response = self._post(client, defaults)
assert response.status_code == 200
body = response.json()
# Required slots got filled.
assert body["assigned"]["default_chat_model"] == "model:lang"
assert body["assigned"]["default_embedding_model"] == "model:embed"
assert defaults.default_chat_model == "model:lang"
assert defaults.default_embedding_model == "model:embed"
# Optional slots must NOT be assigned even though models exist.
for slot in (
"default_transformation_model",
"default_tools_model",
"large_context_model",
"default_text_to_speech_model",
"default_speech_to_text_model",
):
assert slot not in body["assigned"]
assert defaults.default_transformation_model is None
assert defaults.large_context_model is None
assert defaults.default_text_to_speech_model is None
defaults.update.assert_awaited_once()
def test_skips_already_filled_required_slot(self, client):
defaults = self._mock_defaults()
defaults.default_chat_model = "model:existing-chat"
response = self._post(client, defaults)
assert response.status_code == 200
body = response.json()
assert "default_chat_model" in body["skipped"]
assert body["assigned"]["default_embedding_model"] == "model:embed"
class TestGetDefaultModelFallback:
"""get_default_model must fall back to the chat model for the three text
optional slots (transformation, tools, large_context) when unset (#1098).
"""
def _defaults(self):
from unittest.mock import MagicMock
defaults = MagicMock()
defaults.default_chat_model = "model:chat"
defaults.default_transformation_model = None
defaults.default_tools_model = None
defaults.large_context_model = None
defaults.default_text_to_speech_model = None
defaults.default_speech_to_text_model = None
defaults.default_embedding_model = "model:embed"
return defaults
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model_type", ["transformation", "tools", "large_context"]
)
async def test_text_optional_slots_fall_back_to_chat(self, model_type):
from open_notebook.ai.models import model_manager
defaults = self._defaults()
with patch.object(
model_manager, "get_defaults", new=AsyncMock(return_value=defaults)
), patch.object(
model_manager, "get_model", new=AsyncMock(return_value="chat-model-obj")
) as mock_get_model:
result = await model_manager.get_default_model(model_type)
assert result == "chat-model-obj"
mock_get_model.assert_awaited_once()
assert mock_get_model.await_args is not None
assert mock_get_model.await_args.args[0] == "model:chat"
@pytest.mark.asyncio
async def test_audio_slots_do_not_fall_back(self):
from open_notebook.ai.models import model_manager
defaults = self._defaults()
with patch.object(
model_manager, "get_defaults", new=AsyncMock(return_value=defaults)
), patch.object(
model_manager, "get_model", new=AsyncMock(return_value="obj")
) as mock_get_model:
tts = await model_manager.get_default_model("text_to_speech")
stt = await model_manager.get_default_model("speech_to_text")
assert tts is None
assert stt is None
mock_get_model.assert_not_awaited()