vrr/open-notebook
2
0
Fork 0
mirror of https://github.com/lfnovo/open-notebook.git synced 2026-08-13 19:13:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor(commands): remove pre-1.6 embedding shims and dead tool config (#1056)

Browse source 
The embed_single_item, embed_chunk and vectorize_source command handlers
existed only so jobs queued by a pre-1.6 version could drain after an
upgrade; any worker restarted on 1.6+ has no such jobs. Remove them, their
input/output models and their tests.

Also drop dead tooling config from pyproject.toml: the [tool.mypy] block
(mypy.ini takes precedence and is the real config) and the Streamlit-era
ruff per-file-ignores for app_home.py and pages/**, which no longer exist.
This commit is contained in:
Luis Novo 2026-07-11 18:25:26 -03:00 • committed by GitHub
parent 5b253d7637
commit 8bcfe01f4c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 1 additions and 377 deletions
Download patch file Download diff file Expand all files Collapse all files

1
CHANGELOG.md
View file

@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Removed
- Dead Streamlit-era service layer (~2,000 lines): `api/client.py` (a synchronous HTTP client that called the app's own API) and 13 `api/*_service.py` wrappers that consumed the app's own HTTP API — none were imported by any router, command or test. Also removed the toy `process_text`/`analyze_data` demo commands (`commands/example_commands.py`) from the background worker
- Pre-1.6 embedding job compatibility shims (the `embed_single_item`, `embed_chunk` and `vectorize_source` command handlers) — they existed only so jobs queued by a pre-1.6 version could drain after an upgrade, and any worker restarted on 1.6+ has no such jobs. **Upgrade note:** if you are upgrading from a version older than 1.6 with embedding jobs still queued, drain the queue on a 1.x release before upgrading past this change. Also removed dead tooling config from `pyproject.toml`: the `[tool.mypy]` block (the real config is `mypy.ini`) and Streamlit-era ruff per-file-ignores for files that no longer exist
## [1.11.0] - 2026-07-11

264
commands/embedding_commands.py
View file

@ -118,58 +118,6 @@ class EmbedSourceOutput(CommandOutput):
error_message: Optional[str] = None
class LegacyEmbedSingleItemInput(CommandInput):
"""Input for the pre-1.6 embed_single_item command kept for queued jobs."""
item_id: str
item_type: Literal["source", "note", "insight"]
class LegacyEmbedSingleItemOutput(CommandOutput):
"""Output matching the pre-1.6 embed_single_item command shape."""
success: bool
item_id: str
item_type: str
chunks_created: int = 0
processing_time: float
error_message: Optional[str] = None
class LegacyEmbedChunkInput(CommandInput):
"""Input for the pre-1.6 per-chunk embedding command kept for queued jobs."""
source_id: str
chunk_index: int
chunk_text: str
class LegacyEmbedChunkOutput(CommandOutput):
"""Output matching the pre-1.6 embed_chunk command shape."""
success: bool
source_id: str
chunk_index: int
error_message: Optional[str] = None
class LegacyVectorizeSourceInput(CommandInput):
"""Input for the pre-1.6 vectorize_source command kept for queued jobs."""
source_id: str
class LegacyVectorizeSourceOutput(CommandOutput):
"""Output matching the pre-1.6 vectorize_source command shape."""
success: bool
source_id: str
total_chunks: int
jobs_submitted: int
processing_time: float
error_message: Optional[str] = None
@command(
"embed_note",
app="open_notebook",
@ -501,218 +449,6 @@ async def embed_source_command(input_data: EmbedSourceInput) -> EmbedSourceOutpu
raise
@command(
"embed_single_item",
app="open_notebook",
retry={
"max_attempts": 5,
"wait_strategy": "exponential_jitter",
"wait_min": 1,
"wait_max": 60,
"stop_on": [ValueError, ConfigurationError],
"retry_log_level": "debug",
},
)
async def legacy_embed_single_item_command(
input_data: LegacyEmbedSingleItemInput,
) -> LegacyEmbedSingleItemOutput:
"""
Compatibility handler for pre-1.6 queued embed_single_item jobs.
New code submits embed_source, embed_note, or embed_insight directly. This
alias lets workers drain older queues after an upgrade.
"""
start_time = time.time()
try:
logger.info(
f"Processing legacy embed_single_item for "
f"{input_data.item_type}: {input_data.item_id}"
)
if input_data.item_type == "source":
result = await embed_source_command(
EmbedSourceInput(
source_id=input_data.item_id,
execution_context=input_data.execution_context,
)
)
chunks_created = result.chunks_created
elif input_data.item_type == "note":
result = await embed_note_command(
EmbedNoteInput(
note_id=input_data.item_id,
execution_context=input_data.execution_context,
)
)
chunks_created = 0
elif input_data.item_type == "insight":
result = await embed_insight_command(
EmbedInsightInput(
insight_id=input_data.item_id,
execution_context=input_data.execution_context,
)
)
chunks_created = 0
else:
raise ValueError(f"Invalid item_type: {input_data.item_type}")
return LegacyEmbedSingleItemOutput(
success=result.success,
item_id=input_data.item_id,
item_type=input_data.item_type,
chunks_created=chunks_created,
processing_time=time.time() - start_time,
error_message=result.error_message,
)
except ValueError as e:
processing_time = time.time() - start_time
logger.error(
f"Failed legacy embed_single_item for "
f"{input_data.item_type} {input_data.item_id}: {e}"
)
return LegacyEmbedSingleItemOutput(
success=False,
item_id=input_data.item_id,
item_type=input_data.item_type,
processing_time=processing_time,
error_message=str(e),
)
except Exception as e:
logger.debug(
f"Transient error in legacy embed_single_item for "
f"{input_data.item_type} {input_data.item_id}: {e}"
)
raise
@command(
"embed_chunk",
app="open_notebook",
retry={
"max_attempts": 5,
"wait_strategy": "exponential_jitter",
"wait_min": 1,
"wait_max": 60,
"stop_on": [ValueError, ConfigurationError],
"retry_log_level": "debug",
},
)
async def legacy_embed_chunk_command(
input_data: LegacyEmbedChunkInput,
) -> LegacyEmbedChunkOutput:
"""
Compatibility handler for pre-1.6 queued embed_chunk jobs.
The legacy vectorizer stored the full chunk payload in each job. Keeping this
command registered prevents upgraded workers from crashing on stale queues.
"""
try:
logger.debug(
f"Processing legacy chunk {input_data.chunk_index} "
f"for source {input_data.source_id}"
)
cmd_id = get_command_id(input_data)
embedding = await generate_embedding(
input_data.chunk_text,
content_type=ContentType.PLAIN,
command_id=cmd_id,
)
await repo_query(
"""
CREATE source_embedding CONTENT {
"source": $source_id,
"order": $order,
"content": $content,
"embedding": $embedding,
};
""",
{
"source_id": ensure_record_id(input_data.source_id),
"order": input_data.chunk_index,
"content": input_data.chunk_text,
"embedding": embedding,
},
)
return LegacyEmbedChunkOutput(
success=True,
source_id=input_data.source_id,
chunk_index=input_data.chunk_index,
)
except ValueError as e:
logger.error(
f"Failed legacy embed_chunk for source {input_data.source_id} "
f"chunk {input_data.chunk_index}: {e}"
)
return LegacyEmbedChunkOutput(
success=False,
source_id=input_data.source_id,
chunk_index=input_data.chunk_index,
error_message=str(e),
)
except Exception as e:
logger.debug(
f"Transient error in legacy embed_chunk for source "
f"{input_data.source_id} chunk {input_data.chunk_index}: {e}"
)
raise
@command("vectorize_source", app="open_notebook", retry=None)
async def legacy_vectorize_source_command(
input_data: LegacyVectorizeSourceInput,
) -> LegacyVectorizeSourceOutput:
"""
Compatibility handler for pre-1.6 queued vectorize_source jobs.
The old command submitted one job per chunk. Current embed_source does the
same source embedding work in one batch-aware command.
"""
start_time = time.time()
try:
logger.info(f"Processing legacy vectorize_source for {input_data.source_id}")
result = await embed_source_command(
EmbedSourceInput(
source_id=input_data.source_id,
execution_context=input_data.execution_context,
)
)
jobs_submitted = 1 if result.success else 0
return LegacyVectorizeSourceOutput(
success=result.success,
source_id=input_data.source_id,
total_chunks=result.chunks_created,
jobs_submitted=jobs_submitted,
processing_time=time.time() - start_time,
error_message=result.error_message,
)
except ValueError as e:
processing_time = time.time() - start_time
logger.error(f"Failed legacy vectorize_source for {input_data.source_id}: {e}")
return LegacyVectorizeSourceOutput(
success=False,
source_id=input_data.source_id,
total_chunks=0,
jobs_submitted=0,
processing_time=processing_time,
error_message=str(e),
)
except Exception as e:
logger.debug(
f"Transient error in legacy vectorize_source for "
f"{input_data.source_id}: {e}"
)
raise
@command(
"create_insight",
app="open_notebook",

11
pyproject.toml
View file

@ -92,17 +92,6 @@ ignore = [
"F841", # local variable assigned but never used
]
[tool.ruff.lint.per-file-ignores]
# Streamlit files need nest_asyncio.apply() before imports
"app_home.py" = ["E402"]
"pages/**/*.py" = ["E402"]
[tool.mypy]
# Exclude Streamlit UI pages from type checking
[[tool.mypy.overrides]]
module = "pages.*"
ignore_errors = true
[tool.uv]
# Pillow < 12.2.0 has open security advisories (PSD OOB write, FITS
# decompression bomb, PDF trailer DoS). The only thing holding it back is

102
tests/test_embedding_commands.py
View file

@ -1,102 +0,0 @@
from unittest.mock import AsyncMock
import pytest
from surreal_commands import registry
import commands
import commands.embedding_commands as embedding_commands
def test_legacy_embedding_commands_are_registered():
app_commands = registry.list_commands()["open_notebook"]
assert "embed_chunk" in app_commands
assert "embed_single_item" in app_commands
assert "vectorize_source" in app_commands
@pytest.mark.asyncio
async def test_legacy_embed_chunk_processes_stale_queue_payload(monkeypatch):
mock_generate_embedding = AsyncMock(return_value=[0.1, 0.2, 0.3])
mock_repo_query = AsyncMock()
monkeypatch.setattr(
embedding_commands, "generate_embedding", mock_generate_embedding
)
monkeypatch.setattr(embedding_commands, "repo_query", mock_repo_query)
monkeypatch.setattr(
embedding_commands, "ensure_record_id", lambda value: f"record:{value}"
)
result = await embedding_commands.legacy_embed_chunk_command(
embedding_commands.LegacyEmbedChunkInput(
source_id="source:abc",
chunk_index=2,
chunk_text="queued legacy chunk",
)
)
assert result.success is True
assert result.source_id == "source:abc"
assert result.chunk_index == 2
mock_generate_embedding.assert_awaited_once_with(
"queued legacy chunk",
content_type=embedding_commands.ContentType.PLAIN,
command_id="unknown",
)
mock_repo_query.assert_awaited_once()
assert mock_repo_query.await_args is not None
assert mock_repo_query.await_args.args[1] == {
"source_id": "record:source:abc",
"order": 2,
"content": "queued legacy chunk",
"embedding": [0.1, 0.2, 0.3],
}
@pytest.mark.asyncio
async def test_legacy_vectorize_source_delegates_to_embed_source(monkeypatch):
async def fake_embed_source(input_data):
assert input_data.source_id == "source:abc"
return embedding_commands.EmbedSourceOutput(
success=True,
source_id=input_data.source_id,
chunks_created=3,
processing_time=0.1,
)
monkeypatch.setattr(embedding_commands, "embed_source_command", fake_embed_source)
result = await embedding_commands.legacy_vectorize_source_command(
embedding_commands.LegacyVectorizeSourceInput(source_id="source:abc")
)
assert result.success is True
assert result.source_id == "source:abc"
assert result.total_chunks == 3
assert result.jobs_submitted == 1
@pytest.mark.asyncio
async def test_legacy_embed_single_item_routes_insights(monkeypatch):
async def fake_embed_insight(input_data):
assert input_data.insight_id == "source_insight:abc"
return embedding_commands.EmbedInsightOutput(
success=True,
insight_id=input_data.insight_id,
processing_time=0.1,
)
monkeypatch.setattr(embedding_commands, "embed_insight_command", fake_embed_insight)
result = await embedding_commands.legacy_embed_single_item_command(
embedding_commands.LegacyEmbedSingleItemInput(
item_id="source_insight:abc",
item_type="insight",
)
)
assert result.success is True
assert result.item_id == "source_insight:abc"
assert result.item_type == "insight"
assert result.chunks_created == 0