mirror of
https://github.com/lfnovo/open-notebook.git
synced 2026-04-28 19:40:50 +00:00
d8006ff5cb
* feat: content-type aware chunking and unified embedding - Add chunking.py with HTML, Markdown, and plain text detection - Add embedding.py with mean pooling for large content - Create dedicated commands: embed_note, embed_insight, embed_source - Use fire-and-forget pattern for embedding via submit_command() - Refactor rebuild_embeddings_command to delegate to individual commands - Remove legacy commands and needs_embedding() methods - Reduce chunk size to 1500 chars for Ollama compatibility - Update CLAUDE.md documentation for new architecture Fixes #350, #142 * fix: address code review issues - Note.save() now returns command_id for tracking embedding jobs - Add length check after generate_embeddings() to fail fast on mismatch - Add numpy as explicit dependency (was transitive) - Remove hardcoded chunk sizes from docstrings * docs: address code review comments - Rename "SYNC PATH" to "DOMAIN MODEL PATH" in embedding router - Add test_chunking.py and test_embedding.py to Testing Strategy - Clarify auto-embedding behavior for each domain model * fix: clean thinking tags from prompt graph output Adds clean_thinking_content() to prompt.py to handle extended thinking models that return <think>...</think> tags. This fixes empty titles when saving notes from chat. * chore: remove local docker-compose from git * fix(frontend): handle null parent_id in search results Add defensive check for null parent_id in search results to prevent "Cannot read properties of null (reading 'split')" error. This can happen with orphaned records in the database. * fix: cascade delete embeddings and insights when source is deleted When deleting a Source, now also deletes associated: - source_embedding records - source_insight records This prevents orphaned records that cause null parent_id errors in vector search results. * fix: add cleanup for orphan embedding/insight records in migration 10 Deletes source_embedding and source_insight records where the linked source no longer exists (source.id = NONE). * chore: bump esperanto to 2.16 Increases ctx_num for Ollama models to accommodate larger notebook context windows. See: https://github.com/lfnovo/esperanto/pull/69
45 lines
1.4 KiB
Python
45 lines
1.4 KiB
Python
from typing import Any, Optional
|
|
|
|
from ai_prompter import Prompter
|
|
from langchain_core.messages import HumanMessage, SystemMessage
|
|
from langchain_core.runnables import RunnableConfig
|
|
from langgraph.graph import END, START, StateGraph
|
|
from typing_extensions import TypedDict
|
|
|
|
from open_notebook.ai.provision import provision_langchain_model
|
|
from open_notebook.utils.text_utils import clean_thinking_content
|
|
|
|
|
|
class PatternChainState(TypedDict):
|
|
prompt: str
|
|
parser: Optional[Any]
|
|
input_text: str
|
|
output: str
|
|
|
|
|
|
async def call_model(state: dict, config: RunnableConfig) -> dict:
|
|
content = state["input_text"]
|
|
system_prompt = Prompter(
|
|
template_text=state["prompt"], parser=state.get("parser")
|
|
).render(data=state)
|
|
payload = [SystemMessage(content=system_prompt)] + [HumanMessage(content=content)]
|
|
chain = await provision_langchain_model(
|
|
str(payload),
|
|
config.get("configurable", {}).get("model_id"),
|
|
"transformation",
|
|
max_tokens=5000,
|
|
)
|
|
|
|
response = await chain.ainvoke(payload)
|
|
|
|
# Clean thinking tags from response (handles extended thinking models)
|
|
output = clean_thinking_content(str(response.content))
|
|
return {"output": output}
|
|
|
|
|
|
agent_state = StateGraph(PatternChainState)
|
|
agent_state.add_node("agent", call_model) # type: ignore[type-var]
|
|
agent_state.add_edge(START, "agent")
|
|
agent_state.add_edge("agent", END)
|
|
|
|
graph = agent_state.compile()
|