diff --git a/CHANGELOG.md b/CHANGELOG.md index 865a9117..f8b36c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed - Dead Streamlit-era service layer (~2,000 lines): `api/client.py` (a synchronous HTTP client that called the app's own API) and 13 `api/*_service.py` wrappers that consumed the app's own HTTP API — none were imported by any router, command or test. Also removed the toy `process_text`/`analyze_data` demo commands (`commands/example_commands.py`) from the background worker +- Pre-1.6 embedding job compatibility shims (the `embed_single_item`, `embed_chunk` and `vectorize_source` command handlers) — they existed only so jobs queued by a pre-1.6 version could drain after an upgrade, and any worker restarted on 1.6+ has no such jobs. **Upgrade note:** if you are upgrading from a version older than 1.6 with embedding jobs still queued, drain the queue on a 1.x release before upgrading past this change. Also removed dead tooling config from `pyproject.toml`: the `[tool.mypy]` block (the real config is `mypy.ini`) and Streamlit-era ruff per-file-ignores for files that no longer exist ## [1.11.0] - 2026-07-11 diff --git a/commands/embedding_commands.py b/commands/embedding_commands.py index 0a2a0b57..97c30ca9 100644 --- a/commands/embedding_commands.py +++ b/commands/embedding_commands.py @@ -118,58 +118,6 @@ class EmbedSourceOutput(CommandOutput): error_message: Optional[str] = None -class LegacyEmbedSingleItemInput(CommandInput): - """Input for the pre-1.6 embed_single_item command kept for queued jobs.""" - - item_id: str - item_type: Literal["source", "note", "insight"] - - -class LegacyEmbedSingleItemOutput(CommandOutput): - """Output matching the pre-1.6 embed_single_item command shape.""" - - success: bool - item_id: str - item_type: str - chunks_created: int = 0 - processing_time: float - error_message: Optional[str] = None - - -class LegacyEmbedChunkInput(CommandInput): - """Input for the pre-1.6 per-chunk embedding command kept for queued jobs.""" - - source_id: str - chunk_index: int - chunk_text: str - - -class LegacyEmbedChunkOutput(CommandOutput): - """Output matching the pre-1.6 embed_chunk command shape.""" - - success: bool - source_id: str - chunk_index: int - error_message: Optional[str] = None - - -class LegacyVectorizeSourceInput(CommandInput): - """Input for the pre-1.6 vectorize_source command kept for queued jobs.""" - - source_id: str - - -class LegacyVectorizeSourceOutput(CommandOutput): - """Output matching the pre-1.6 vectorize_source command shape.""" - - success: bool - source_id: str - total_chunks: int - jobs_submitted: int - processing_time: float - error_message: Optional[str] = None - - @command( "embed_note", app="open_notebook", @@ -501,218 +449,6 @@ async def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutpu raise -@command( - "embed_single_item", - app="open_notebook", - retry={ - "max_attempts": 5, - "wait_strategy": "exponential_jitter", - "wait_min": 1, - "wait_max": 60, - "stop_on": [ValueError, ConfigurationError], - "retry_log_level": "debug", - }, -) -async def legacy_embed_single_item_command( - input_data: LegacyEmbedSingleItemInput, -) -> LegacyEmbedSingleItemOutput: - """ - Compatibility handler for pre-1.6 queued embed_single_item jobs. - - New code submits embed_source, embed_note, or embed_insight directly. This - alias lets workers drain older queues after an upgrade. - """ - start_time = time.time() - - try: - logger.info( - f"Processing legacy embed_single_item for " - f"{input_data.item_type}: {input_data.item_id}" - ) - - if input_data.item_type == "source": - result = await embed_source_command( - EmbedSourceInput( - source_id=input_data.item_id, - execution_context=input_data.execution_context, - ) - ) - chunks_created = result.chunks_created - elif input_data.item_type == "note": - result = await embed_note_command( - EmbedNoteInput( - note_id=input_data.item_id, - execution_context=input_data.execution_context, - ) - ) - chunks_created = 0 - elif input_data.item_type == "insight": - result = await embed_insight_command( - EmbedInsightInput( - insight_id=input_data.item_id, - execution_context=input_data.execution_context, - ) - ) - chunks_created = 0 - else: - raise ValueError(f"Invalid item_type: {input_data.item_type}") - - return LegacyEmbedSingleItemOutput( - success=result.success, - item_id=input_data.item_id, - item_type=input_data.item_type, - chunks_created=chunks_created, - processing_time=time.time() - start_time, - error_message=result.error_message, - ) - - except ValueError as e: - processing_time = time.time() - start_time - logger.error( - f"Failed legacy embed_single_item for " - f"{input_data.item_type} {input_data.item_id}: {e}" - ) - return LegacyEmbedSingleItemOutput( - success=False, - item_id=input_data.item_id, - item_type=input_data.item_type, - processing_time=processing_time, - error_message=str(e), - ) - except Exception as e: - logger.debug( - f"Transient error in legacy embed_single_item for " - f"{input_data.item_type} {input_data.item_id}: {e}" - ) - raise - - -@command( - "embed_chunk", - app="open_notebook", - retry={ - "max_attempts": 5, - "wait_strategy": "exponential_jitter", - "wait_min": 1, - "wait_max": 60, - "stop_on": [ValueError, ConfigurationError], - "retry_log_level": "debug", - }, -) -async def legacy_embed_chunk_command( - input_data: LegacyEmbedChunkInput, -) -> LegacyEmbedChunkOutput: - """ - Compatibility handler for pre-1.6 queued embed_chunk jobs. - - The legacy vectorizer stored the full chunk payload in each job. Keeping this - command registered prevents upgraded workers from crashing on stale queues. - """ - try: - logger.debug( - f"Processing legacy chunk {input_data.chunk_index} " - f"for source {input_data.source_id}" - ) - - cmd_id = get_command_id(input_data) - embedding = await generate_embedding( - input_data.chunk_text, - content_type=ContentType.PLAIN, - command_id=cmd_id, - ) - - await repo_query( - """ - CREATE source_embedding CONTENT { - "source": $source_id, - "order": $order, - "content": $content, - "embedding": $embedding, - }; - """, - { - "source_id": ensure_record_id(input_data.source_id), - "order": input_data.chunk_index, - "content": input_data.chunk_text, - "embedding": embedding, - }, - ) - - return LegacyEmbedChunkOutput( - success=True, - source_id=input_data.source_id, - chunk_index=input_data.chunk_index, - ) - - except ValueError as e: - logger.error( - f"Failed legacy embed_chunk for source {input_data.source_id} " - f"chunk {input_data.chunk_index}: {e}" - ) - return LegacyEmbedChunkOutput( - success=False, - source_id=input_data.source_id, - chunk_index=input_data.chunk_index, - error_message=str(e), - ) - except Exception as e: - logger.debug( - f"Transient error in legacy embed_chunk for source " - f"{input_data.source_id} chunk {input_data.chunk_index}: {e}" - ) - raise - - -@command("vectorize_source", app="open_notebook", retry=None) -async def legacy_vectorize_source_command( - input_data: LegacyVectorizeSourceInput, -) -> LegacyVectorizeSourceOutput: - """ - Compatibility handler for pre-1.6 queued vectorize_source jobs. - - The old command submitted one job per chunk. Current embed_source does the - same source embedding work in one batch-aware command. - """ - start_time = time.time() - - try: - logger.info(f"Processing legacy vectorize_source for {input_data.source_id}") - result = await embed_source_command( - EmbedSourceInput( - source_id=input_data.source_id, - execution_context=input_data.execution_context, - ) - ) - jobs_submitted = 1 if result.success else 0 - - return LegacyVectorizeSourceOutput( - success=result.success, - source_id=input_data.source_id, - total_chunks=result.chunks_created, - jobs_submitted=jobs_submitted, - processing_time=time.time() - start_time, - error_message=result.error_message, - ) - - except ValueError as e: - processing_time = time.time() - start_time - logger.error(f"Failed legacy vectorize_source for {input_data.source_id}: {e}") - return LegacyVectorizeSourceOutput( - success=False, - source_id=input_data.source_id, - total_chunks=0, - jobs_submitted=0, - processing_time=processing_time, - error_message=str(e), - ) - except Exception as e: - logger.debug( - f"Transient error in legacy vectorize_source for " - f"{input_data.source_id}: {e}" - ) - raise - - @command( "create_insight", app="open_notebook", diff --git a/pyproject.toml b/pyproject.toml index d51663c4..b509cdaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,17 +92,6 @@ ignore = [ "F841", # local variable assigned but never used ] -[tool.ruff.lint.per-file-ignores] -# Streamlit files need nest_asyncio.apply() before imports -"app_home.py" = ["E402"] -"pages/**/*.py" = ["E402"] - -[tool.mypy] -# Exclude Streamlit UI pages from type checking -[[tool.mypy.overrides]] -module = "pages.*" -ignore_errors = true - [tool.uv] # Pillow < 12.2.0 has open security advisories (PSD OOB write, FITS # decompression bomb, PDF trailer DoS). The only thing holding it back is diff --git a/tests/test_embedding_commands.py b/tests/test_embedding_commands.py deleted file mode 100644 index d3068605..00000000 --- a/tests/test_embedding_commands.py +++ /dev/null @@ -1,102 +0,0 @@ -from unittest.mock import AsyncMock - -import pytest -from surreal_commands import registry - -import commands -import commands.embedding_commands as embedding_commands - - -def test_legacy_embedding_commands_are_registered(): - app_commands = registry.list_commands()["open_notebook"] - - assert "embed_chunk" in app_commands - assert "embed_single_item" in app_commands - assert "vectorize_source" in app_commands - - -@pytest.mark.asyncio -async def test_legacy_embed_chunk_processes_stale_queue_payload(monkeypatch): - mock_generate_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3]) - mock_repo_query = AsyncMock() - - monkeypatch.setattr( - embedding_commands, "generate_embedding", mock_generate_embedding - ) - monkeypatch.setattr(embedding_commands, "repo_query", mock_repo_query) - monkeypatch.setattr( - embedding_commands, "ensure_record_id", lambda value: f"record:{value}" - ) - - result = await embedding_commands.legacy_embed_chunk_command( - embedding_commands.LegacyEmbedChunkInput( - source_id="source:abc", - chunk_index=2, - chunk_text="queued legacy chunk", - ) - ) - - assert result.success is True - assert result.source_id == "source:abc" - assert result.chunk_index == 2 - mock_generate_embedding.assert_awaited_once_with( - "queued legacy chunk", - content_type=embedding_commands.ContentType.PLAIN, - command_id="unknown", - ) - mock_repo_query.assert_awaited_once() - assert mock_repo_query.await_args is not None - assert mock_repo_query.await_args.args[1] == { - "source_id": "record:source:abc", - "order": 2, - "content": "queued legacy chunk", - "embedding": [0.1, 0.2, 0.3], - } - - -@pytest.mark.asyncio -async def test_legacy_vectorize_source_delegates_to_embed_source(monkeypatch): - async def fake_embed_source(input_data): - assert input_data.source_id == "source:abc" - return embedding_commands.EmbedSourceOutput( - success=True, - source_id=input_data.source_id, - chunks_created=3, - processing_time=0.1, - ) - - monkeypatch.setattr(embedding_commands, "embed_source_command", fake_embed_source) - - result = await embedding_commands.legacy_vectorize_source_command( - embedding_commands.LegacyVectorizeSourceInput(source_id="source:abc") - ) - - assert result.success is True - assert result.source_id == "source:abc" - assert result.total_chunks == 3 - assert result.jobs_submitted == 1 - - -@pytest.mark.asyncio -async def test_legacy_embed_single_item_routes_insights(monkeypatch): - async def fake_embed_insight(input_data): - assert input_data.insight_id == "source_insight:abc" - return embedding_commands.EmbedInsightOutput( - success=True, - insight_id=input_data.insight_id, - processing_time=0.1, - ) - - monkeypatch.setattr(embedding_commands, "embed_insight_command", fake_embed_insight) - - result = await embedding_commands.legacy_embed_single_item_command( - embedding_commands.LegacyEmbedSingleItemInput( - item_id="source_insight:abc", - item_type="insight", - ) - ) - - assert result.success is True - assert result.item_id == "source_insight:abc" - assert result.item_type == "insight" - assert result.chunks_created == 0