feat: Fixed Document Summary Content across connectors and processors

surfsense_backend/app/tasks/document_processors/base.py

@@ -6,9 +6,7 @@ from langchain_community.document_transformers import MarkdownifyTransformer
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 
-from app.config import config
-from app.db import Chunk, Document
-from app.prompts import SUMMARY_PROMPT_TEMPLATE
+from app.db import Document
 
 # Initialize markdown transformer
 md = MarkdownifyTransformer()
@@ -31,44 +29,3 @@ async def check_duplicate_document(
         select(Document).where(Document.content_hash == content_hash)
     )
     return existing_doc_result.scalars().first()
-
-
-async def create_document_chunks(content: str) -> list[Chunk]:
-    """
-    Create chunks from document content.
-
-    Args:
-        content: Document content to chunk
-
-    Returns:
-        List of Chunk objects with embeddings
-    """
-    return [
-        Chunk(
-            content=chunk.text,
-            embedding=config.embedding_model_instance.embed(chunk.text),
-        )
-        for chunk in config.chunker_instance.chunk(content)
-    ]
-
-
-async def generate_document_summary(
-    content: str, user_llm, document_title: str = ""
-) -> tuple[str, list[float]]:
-    """
-    Generate summary and embedding for document content.
-
-    Args:
-        content: Document content
-        user_llm: User's LLM instance
-        document_title: Optional document title for context
-
-    Returns:
-        Tuple of (summary_content, summary_embedding)
-    """
-    summary_chain = SUMMARY_PROMPT_TEMPLATE | user_llm
-    summary_result = await summary_chain.ainvoke({"document": content})
-    summary_content = summary_result.content
-    summary_embedding = config.embedding_model_instance.embed(summary_content)
-
-    return summary_content, summary_embedding