Merge pull request #271 from MODSetter/dev

fix: Streaming Duplicate Sources & Delete Search Space UI
2025-09-01 10:09:08 +00:00 · 2025-08-19 11:14:21 -07:00 · 2025-08-19 11:14:21 -07:00 · fde2bc1534
commit fde2bc1534
parent ca9614cd5e d74f135891
5 changed files with 84 additions and 83 deletions
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -74,14 +74,14 @@ repos:
        stages: [pre-commit]

      # Biome check for surfsense_browser_extension
-      - id: biome-check-extension
-        name: biome-check-extension
-        entry: bash -c 'cd surfsense_browser_extension && npx @biomejs/biome check --diagnostic-level=error .'
-        language: system
-        files: ^surfsense_browser_extension/
-        pass_filenames: false
-        always_run: true
-        stages: [pre-commit]
+      # - id: biome-check-extension
+      #   name: biome-check-extension
+      #   entry: bash -c 'cd surfsense_browser_extension && npx @biomejs/biome check --diagnostic-level=error .'
+      #   language: system
+      #   files: ^surfsense_browser_extension/
+      #   pass_filenames: false
+      #   always_run: true
+      #   stages: [pre-commit]

  # Commit message linting
  - repo: https://github.com/commitizen-tools/commitizen
--- a/surfsense_backend/app/agents/researcher/nodes.py
+++ b/surfsense_backend/app/agents/researcher/nodes.py
@ -1708,7 +1708,7 @@ async def handle_qna_workflow(
    )

    # Use a reasonable top_k for QNA - not too many documents to avoid overwhelming the LLM
-    top_k = 15
+    top_k = 20

    relevant_documents = []
    user_selected_documents = []
--- a/surfsense_backend/app/agents/researcher/utils.py
+++ b/surfsense_backend/app/agents/researcher/utils.py
@ -91,7 +91,7 @@ def format_document_for_citation(document: dict[str, Any]) -> str:
    """Format a single document for citation in the standard XML format."""
    content = document.get("content", "")
    doc_info = document.get("document", {})
-    document_id = doc_info.get("id", "")
+    document_id = document.get("chunk_id", "")
    document_type = doc_info.get("document_type", "CRAWLED_URL")

    return f"""<document>
--- a/surfsense_backend/app/services/connector_service.py
+++ b/surfsense_backend/app/services/connector_service.py
@ -109,7 +109,7 @@ class ConnectorService:

                # Create a source entry
                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": document.get("title", "Untitled Document"),
                    "description": metadata.get(
                        "og:description",
@ -183,7 +183,7 @@ class ConnectorService:

                # Create a source entry
                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": document.get("title", "Untitled Document"),
                    "description": metadata.get(
                        "og:description",
@ -312,7 +312,7 @@ class ConnectorService:
            documents = []

            async with self.counter_lock:
-                for i, result in enumerate(tavily_results):
+                for _i, result in enumerate(tavily_results):
                    # Create a source entry
                    source = {
                        "id": self.source_id_counter,
@ -324,7 +324,7 @@ class ConnectorService:

                    # Create a document entry
                    document = {
-                        "chunk_id": f"tavily_chunk_{i}",
+                        "chunk_id": self.source_id_counter,
                        "content": result.get("content", ""),
                        "score": result.get("score", 0.0),
                        "document": {
@ -432,7 +432,7 @@ class ConnectorService:
                    url = f"https://slack.com/app_redirect?channel={channel_id}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -529,7 +529,7 @@ class ConnectorService:
                    url = f"https://notion.so/{page_id.replace('-', '')}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -650,7 +650,7 @@ class ConnectorService:
                        pass

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": webpage_url,
@ -747,7 +747,7 @@ class ConnectorService:
                url = f"https://www.youtube.com/watch?v={video_id}" if video_id else ""

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -820,7 +820,7 @@ class ConnectorService:

                # Create a source entry
                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": document.get(
                        "title", "GitHub Document"
                    ),  # Use specific title if available
@ -930,7 +930,7 @@ class ConnectorService:
                    url = f"https://linear.app/issue/{issue_identifier}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -1049,7 +1049,7 @@ class ConnectorService:
                    url = f"{metadata.get('base_url')}/browse/{issue_key}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -1183,7 +1183,7 @@ class ConnectorService:
                    url = f"https://calendar.google.com/calendar/event?eid={event_id}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -1310,7 +1310,7 @@ class ConnectorService:
                    url = f"https://mail.google.com/mail/u/0/#inbox/{message_id}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -1412,7 +1412,7 @@ class ConnectorService:
                    url = f"{metadata.get('base_url')}/pages/{page_id}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
@ -1521,7 +1521,7 @@ class ConnectorService:
            )

            source = {
-                "id": document.get("id", self.source_id_counter),
+                "id": chunk.get("chunk_id", self.source_id_counter),
                "title": task_name,
                "description": description,
                "url": task_url,
@ -1604,7 +1604,7 @@ class ConnectorService:
            documents = []

            async with self.counter_lock:
-                for i, result in enumerate(linkup_results):
+                for _i, result in enumerate(linkup_results):
                    # Only process results that have content
                    if not hasattr(result, "content") or not result.content:
                        continue
@ -1624,7 +1624,7 @@ class ConnectorService:

                    # Create a document entry
                    document = {
-                        "chunk_id": f"linkup_chunk_{i}",
+                        "chunk_id": self.source_id_counter,
                        "content": result.content if hasattr(result, "content") else "",
                        "score": 1.0,  # Default score since not provided by Linkup
                        "document": {
@ -1745,7 +1745,7 @@ class ConnectorService:
                    url = f"https://discord.com/channels/@me/{channel_id}"

                source = {
-                    "id": document.get("id", self.source_id_counter),
+                    "id": chunk.get("chunk_id", self.source_id_counter),
                    "title": title,
                    "description": description,
                    "url": url,
--- a/surfsense_web/app/dashboard/page.tsx
+++ b/surfsense_web/app/dashboard/page.tsx
@ -266,29 +266,29 @@ const DashboardPage = () => {
 						{searchSpaces &&
 							searchSpaces.length > 0 &&
 							searchSpaces.map((space) => (
-								<Link href={`/dashboard/${space.id}/documents`} key={space.id}>
-									<motion.div key={space.id} variants={itemVariants} className="aspect-[4/3]">
-										<Tilt
-											rotationFactor={6}
-											isRevese
+								<motion.div key={space.id} variants={itemVariants} className="aspect-[4/3]">
+									<Tilt
+										rotationFactor={6}
+										isRevese
+										springOptions={{
+											stiffness: 26.7,
+											damping: 4.1,
+											mass: 0.2,
+										}}
+										className="group relative rounded-lg h-full"
+									>
+										<Spotlight
+											className="z-10 from-blue-500/20 via-blue-300/10 to-blue-200/5 blur-2xl"
+											size={248}
 											springOptions={{
 												stiffness: 26.7,
 												damping: 4.1,
 												mass: 0.2,
 											}}
-											className="group relative rounded-lg h-full"
-										>
-											<Spotlight
-												className="z-10 from-blue-500/20 via-blue-300/10 to-blue-200/5 blur-2xl"
-												size={248}
-												springOptions={{
-													stiffness: 26.7,
-													damping: 4.1,
-													mass: 0.2,
-												}}
-											/>
-											<div className="flex flex-col h-full overflow-hidden rounded-xl border bg-muted/30 backdrop-blur-sm transition-all hover:border-primary/50">
-												<div className="relative h-32 w-full overflow-hidden">
+										/>
+										<div className="flex flex-col h-full overflow-hidden rounded-xl border bg-muted/30 backdrop-blur-sm transition-all hover:border-primary/50">
+											<div className="relative h-32 w-full overflow-hidden">
+												<Link href={`/dashboard/${space.id}/documents`} key={space.id}>
 													<Image
 														src="https://images.unsplash.com/photo-1519389950473-47ba0277781c?ixlib=rb-4.0.3&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=1740&q=80"
 														alt={space.name}
@ -297,42 +297,43 @@ const DashboardPage = () => {
 														height={248}
 													/>
 													<div className="absolute inset-0 bg-gradient-to-t from-background/80 to-transparent" />
-													<div className="absolute top-2 right-2">
-														<div>
-															<AlertDialog>
-																<AlertDialogTrigger asChild>
-																	<Button
-																		variant="ghost"
-																		size="icon"
-																		className="h-8 w-8 rounded-full bg-background/50 backdrop-blur-sm hover:bg-destructive/90 cursor-pointer"
+												</Link>
+												<div className="absolute top-2 right-2">
+													<div>
+														<AlertDialog>
+															<AlertDialogTrigger asChild>
+																<Button
+																	variant="ghost"
+																	size="icon"
+																	className="h-8 w-8 rounded-full bg-background/50 backdrop-blur-sm hover:bg-destructive/90 cursor-pointer"
+																>
+																	<Trash2 className="h-4 w-4" />
+																</Button>
+															</AlertDialogTrigger>
+															<AlertDialogContent>
+																<AlertDialogHeader>
+																	<AlertDialogTitle>Delete Search Space</AlertDialogTitle>
+																	<AlertDialogDescription>
+																		Are you sure you want to delete &quot;{space.name}&quot;? This
+																		action cannot be undone. All documents, chats, and podcasts in
+																		this search space will be permanently deleted.
+																	</AlertDialogDescription>
+																</AlertDialogHeader>
+																<AlertDialogFooter>
+																	<AlertDialogCancel>Cancel</AlertDialogCancel>
+																	<AlertDialogAction
+																		onClick={() => handleDeleteSearchSpace(space.id)}
+																		className="bg-destructive hover:bg-destructive/90"
 																	>
-																		<Trash2 className="h-4 w-4" />
-																	</Button>
-																</AlertDialogTrigger>
-																<AlertDialogContent>
-																	<AlertDialogHeader>
-																		<AlertDialogTitle>Delete Search Space</AlertDialogTitle>
-																		<AlertDialogDescription>
-																			Are you sure you want to delete &quot;{space.name}&quot;? This
-																			action cannot be undone. All documents, chats, and podcasts in
-																			this search space will be permanently deleted.
-																		</AlertDialogDescription>
-																	</AlertDialogHeader>
-																	<AlertDialogFooter>
-																		<AlertDialogCancel>Cancel</AlertDialogCancel>
-																		<AlertDialogAction
-																			onClick={() => handleDeleteSearchSpace(space.id)}
-																			className="bg-destructive hover:bg-destructive/90"
-																		>
-																			Delete
-																		</AlertDialogAction>
-																	</AlertDialogFooter>
-																</AlertDialogContent>
-															</AlertDialog>
-														</div>
+																		Delete
+																	</AlertDialogAction>
+																</AlertDialogFooter>
+															</AlertDialogContent>
+														</AlertDialog>
 													</div>
 												</div>
-
+											</div>
+											<Link href={`/dashboard/${space.id}/documents`} key={space.id}>
 												<div className="flex flex-1 flex-col justify-between p-4">
 													<div>
 														<h3 className="font-medium text-lg">{space.name}</h3>
@ -345,10 +346,10 @@ const DashboardPage = () => {
 														<span>Created {formatDate(space.created_at)}</span>
 													</div>
 												</div>
-											</div>
-										</Tilt>
-									</motion.div>
-								</Link>
+											</Link>
+										</div>
+									</Tilt>
+								</motion.div>
 							))}

 						{searchSpaces.length === 0 && (