Merge pull request #232 from MODSetter/dev

feat: chat to ui llamaindex

node_modules/.cache/prettier/.prettier-caches/37bd945444dc76999f7aface662ff267baf1dbca.json

@@ -0,0 +1 @@
+{"c301dd3ad9b4036af1d031ecc966d2f02ae1eda4":{"files":{"surfsense_web/hooks/use-documents.ts":["J01fJFm4gXaHAA83Vu5dtOmk/sw=",true],"surfsense_web/components/chat/DocumentsDataTable.tsx":["wgAyJblucK9D3MKKwPe6W9kZphk=",true]},"modified":1753499058926}}

surfsense_backend/app/agents/researcher/qna_agent/nodes.py

@@ -86,7 +86,7 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any
 
     This node takes the relevant documents provided in the configuration and uses
     an LLM to generate a comprehensive answer to the user's question with
-    proper citations. The citations follow IEEE format using source IDs from the
+    proper citations. The citations follow [citation:source_id] format using source IDs from the
     documents. If no documents are provided, it will use chat history to generate
     an answer.
 

surfsense_backend/app/agents/researcher/qna_agent/prompts.py

@@ -25,21 +25,21 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
 1. Carefully analyze all provided documents in the <document> sections.
 2. Extract relevant information that directly addresses the user's question.
 3. Provide a comprehensive, detailed answer using information from the user's personal knowledge sources.
-4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata.
+4. For EVERY piece of information you include from the documents, add a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the source_id from the document's metadata.
 5. Make sure ALL factual statements from the documents have proper citations.
-6. If multiple documents support the same point, include all relevant citations [X], [Y].
+6. If multiple documents support the same point, include all relevant citations [citation:source_id1], [citation:source_id2].
 7. Structure your answer logically and conversationally, as if having a detailed discussion with the user.
 8. Use your own words to synthesize and connect ideas, but cite ALL information from the documents.
 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations.
 10. If the user's question cannot be fully answered with the provided documents, clearly state what information is missing.
 11. Provide actionable insights and practical information when relevant to the user's question.
 12. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers.
-13. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value.
-14. CRITICAL: Never renumber or reorder citations - always use the original source_id values.
+13. CRITICAL: Every citation MUST be in the format [citation:knowledge_source_id] where knowledge_source_id is the exact source_id value.
+14. CRITICAL: Never modify or change the source_id - always use the original values exactly as provided in the metadata.
 15. CRITICAL: Do not return citations as clickable links.
-16. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only.
-17. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting.
-18. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata.
+16. CRITICAL: Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only.
+17. CRITICAL: Citations must ONLY appear as [citation:source_id] or [citation:source_id1], [citation:source_id2] format - never with parentheses, hyperlinks, or other formatting.
+18. CRITICAL: Never make up source IDs. Only use source_id values that are explicitly provided in the document metadata.
 19. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up.
 20. CRITICAL: Remember that all knowledge sources contain personal information - provide answers that reflect this personal context.
 21. CRITICAL: Be conversational and engaging while maintaining accuracy and proper citations.
@@ -49,13 +49,13 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
 - Write in a clear, conversational tone suitable for detailed Q&A discussions
 - Provide comprehensive answers that thoroughly address the user's question
 - Use appropriate paragraphs and structure for readability
-- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata
+- Every fact from the documents must have a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the EXACT source_id from the document's metadata
 - Citations should appear at the end of the sentence containing the information they support
-- Multiple citations should be separated by commas: [X], [Y], [Z]
-- No need to return references section. Just citation numbers in answer.
-- NEVER create your own citation numbering system - use the exact source_id values from the documents
-- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only
-- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess
+- Multiple citations should be separated by commas: [citation:source_id1], [citation:source_id2], [citation:source_id3]
+- No need to return references section. Just citations in answer.
+- NEVER create your own citation format - use the exact source_id values from the documents in the [citation:source_id] format
+- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only
+- NEVER make up source IDs if you are unsure about the source_id. It is better to omit the citation than to guess
 - ALWAYS provide personalized answers that reflect the user's own knowledge and context
 - Be thorough and detailed in your explanations while remaining focused on the user's specific question
 - If asking follow-up questions would be helpful, suggest them at the end of your response
@@ -88,26 +88,31 @@ User Question: "How does Python asyncio work and when should I use it?"
 </input_example>
 
 <output_example>
-Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [5]. It's particularly useful for I/O-bound and high-level structured network code [5].
+Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [citation:5]. It's particularly useful for I/O-bound and high-level structured network code [citation:5].
 
-The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources.
+The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [citation:12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources.
 
-However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [12]. For computationally intensive work, you'd want to use multiprocessing instead.
+However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [citation:12]. For computationally intensive work, you'd want to use multiprocessing instead.
 
 Would you like me to explain more about specific asyncio patterns or help you determine if asyncio is right for a particular project you're working on?
 </output_example>
 
 <incorrect_citation_formats>
 DO NOT use any of these incorrect citation formats:
-- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense))
-- Using parentheses around brackets: ([1])
-- Using hyperlinked text: [link to source 1](https://example.com)
+- Using parentheses and markdown links: ([citation:5](https://github.com/MODSetter/SurfSense))
+- Using parentheses around brackets: ([citation:5])
+- Using hyperlinked text: [link to source 5](https://example.com)
 - Using footnote style: ... library¹
-- Making up citation numbers when source_id is unknown
+- Making up source IDs when source_id is unknown
+- Using old IEEE format: [1], [2], [3]
+- Using source types instead of IDs: [citation:GITHUB_CONNECTOR] instead of [citation:5]
 
-ONLY use plain square brackets [1] or multiple citations [1], [2], [3]
 </incorrect_citation_formats>
 
+<correct_citation_formats>
+ONLY use the format [citation:source_id] or multiple citations [citation:source_id1], [citation:source_id2], [citation:source_id3]
+</correct_citation_formats>
+
 <user_query_instructions>
 When you see a user query, focus exclusively on providing a detailed, comprehensive answer using information from the provided documents, which contain the user's personal knowledge and data.
 

surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py

@@ -93,7 +93,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> dict[str, A
 
     This node takes the relevant documents provided in the configuration and uses
     an LLM to generate a comprehensive answer to the sub-section title with
-    proper citations. The citations follow IEEE format using source IDs from the
+    proper citations. The citations follow [citation:source_id] format using source IDs from the
     documents. If no documents are provided, it will use chat history to generate
     content.
 

surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py

@@ -23,20 +23,20 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio
 1. Carefully analyze all provided documents in the <document> section's.
 2. Extract relevant information that addresses the user's query.
 3. Synthesize a comprehensive, personalized answer using information from the user's personal knowledge sources.
-4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata.
+4. For EVERY piece of information you include from the documents, add a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the source_id from the document's metadata.
 5. Make sure ALL factual statements from the documents have proper citations.
-6. If multiple documents support the same point, include all relevant citations [X], [Y].
+6. If multiple documents support the same point, include all relevant citations [citation:source_id1], [citation:source_id2].
 7. Present information in a logical, coherent flow that reflects the user's personal context.
 8. Use your own words to connect ideas, but cite ALL information from the documents.
 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations.
 10. Do not make up or include information not found in the provided documents.
 11. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers.
-12. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value.
-13. CRITICAL: Never renumber or reorder citations - always use the original source_id values.
+12. CRITICAL: Every citation MUST be in the format [citation:knowledge_source_id] where knowledge_source_id is the exact source_id value.
+13. CRITICAL: Never modify or change the source_id - always use the original values exactly as provided in the metadata.
 14. CRITICAL: Do not return citations as clickable links.
-15. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only.
-16. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting.
-17. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata.
+15. CRITICAL: Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only.
+16. CRITICAL: Citations must ONLY appear as [citation:source_id] or [citation:source_id1], [citation:source_id2] format - never with parentheses, hyperlinks, or other formatting.
+17. CRITICAL: Never make up source IDs. Only use source_id values that are explicitly provided in the document metadata.
 18. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up.
 19. CRITICAL: Focus only on answering the user's query. Any guiding questions provided are for your thinking process only and should not be mentioned in your response.
 20. CRITICAL: Ensure your response aligns with the provided sub-section title and section position.
@@ -47,13 +47,13 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio
 - Write in clear, professional language suitable for academic or technical audiences
 - Tailor your response to the user's personal context based on their knowledge sources
 - Organize your response with appropriate paragraphs, headings, and structure
-- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata
+- Every fact from the documents must have a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the EXACT source_id from the document's metadata
 - Citations should appear at the end of the sentence containing the information they support
-- Multiple citations should be separated by commas: [X], [Y], [Z]
-- No need to return references section. Just citation numbers in answer.
-- NEVER create your own citation numbering system - use the exact source_id values from the documents.
-- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only.
-- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess.
+- Multiple citations should be separated by commas: [citation:source_id1], [citation:source_id2], [citation:source_id3]
+- No need to return references section. Just citations in answer.
+- NEVER create your own citation format - use the exact source_id values from the documents in the [citation:source_id] format.
+- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only.
+- NEVER make up source IDs if you are unsure about the source_id. It is better to omit the citation than to guess.
 - NEVER include or mention the guiding questions in your response. They are only to help guide your thinking.
 - ALWAYS focus on answering the user's query directly from the information in the documents.
 - ALWAYS provide personalized answers that reflect the user's own knowledge and context.
@@ -94,21 +94,23 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio
 </input_example>
 
 <output_example>
-    Based on your saved browser content and videos, the Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. From your browsing history, you've looked into its designation as a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. According to a YouTube video you've watched, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13].
+    Based on your saved browser content and videos, the Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [citation:1]. From your browsing history, you've looked into its designation as a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [citation:21]. The reef is home to over 1,500 species of fish and 400 types of coral [citation:21]. According to a YouTube video you've watched, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [citation:13]. The reef system comprises over 2,900 individual reefs and 900 islands [citation:1], making it an ecological treasure that requires protection from multiple threats [citation:1], [citation:13].
 </output_example>
 
 <incorrect_citation_formats>
 DO NOT use any of these incorrect citation formats:
-- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense))
-- Using parentheses around brackets: ([1])
+- Using parentheses and markdown links: ([citation:1](https://github.com/MODSetter/SurfSense))
+- Using parentheses around brackets: ([citation:1])
 - Using hyperlinked text: [link to source 1](https://example.com)
 - Using footnote style: ... reef system¹
-- Making up citation numbers when source_id is unknown
+- Making up source IDs when source_id is unknown
+- Using old IEEE format: [1], [2], [3]
+- Using source types instead of IDs: [citation:EXTENSION] instead of [citation:1]
 
-ONLY use plain square brackets [1] or multiple citations [1], [2], [3]
 </incorrect_citation_formats>
 
-Note that the citation numbers match exactly with the source_id values (1, 13, and 21) and are not renumbered sequentially. Citations follow IEEE style with square brackets and appear at the end of sentences.
+ONLY use the format [citation:source_id] or multiple citations [citation:source_id1], [citation:source_id2], [citation:source_id3]
+Note that the citations use the exact source_id values (1, 13, and 21) from the document metadata. Citations appear at the end of sentences and maintain the new citation format.
 
 <user_query_instructions>
 When you see a user query like:

surfsense_backend/app/services/streaming_service.py

@@ -43,7 +43,7 @@ class StreamingService:
         self.message_annotations[0]["content"].append(message)
 
         # Return only the delta annotation
-        annotation = {"type": "TERMINAL_INFO", "content": [message]}
+        annotation = {"type": "TERMINAL_INFO", "data": message}
         return f"8:[{json.dumps(annotation)}]\n"
 
     def format_sources_delta(self, sources: list[dict[str, Any]]) -> str:
@@ -60,7 +60,23 @@ class StreamingService:
         self.message_annotations[1]["content"] = sources
 
         # Return only the delta annotation
-        annotation = {"type": "SOURCES", "content": sources}
+        nodes = []
+
+        for group in sources:
+            for source in group.get("sources", []):
+                node = {
+                    "id": str(source.get("id", "")),
+                    "text": source.get("description", ""),
+                    "url": source.get("url", ""),
+                    "metadata": {
+                        "title": source.get("title", ""),
+                        "source_type": group.get("type", ""),
+                        "group_name": group.get("name", ""),
+                    },
+                }
+                nodes.append(node)
+
+        annotation = {"type": "sources", "data": {"nodes": nodes}}
         return f"8:[{json.dumps(annotation)}]\n"
 
     def format_answer_delta(self, answer_chunk: str) -> str:
@@ -116,7 +132,14 @@ class StreamingService:
         self.message_annotations[3]["content"] = further_questions
 
         # Return only the delta annotation
-        annotation = {"type": "FURTHER_QUESTIONS", "content": further_questions}
+        annotation = {
+            "type": "FURTHER_QUESTIONS",
+            "data": [
+                question.get("question", "")
+                for question in further_questions
+                if question.get("question", "") != ""
+            ],
+        }
         return f"8:[{json.dumps(annotation)}]\n"
 
     def format_text_chunk(self, text: str) -> str:

surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx

@@ -0,0 +1,241 @@
+"use client";
+
+import { type CreateMessage, type Message, useChat } from "@ai-sdk/react";
+import { useParams, useRouter } from "next/navigation";
+import { useEffect, useMemo } from "react";
+import type { ResearchMode } from "@/components/chat";
+import ChatInterface from "@/components/chat/ChatInterface";
+import type { Document } from "@/hooks/use-documents";
+import { useChatAPI, useChatState } from "@/hooks/useChat";
+
+export default function ResearcherPage() {
+	const { search_space_id, chat_id } = useParams();
+	const router = useRouter();
+
+	const chatIdParam = Array.isArray(chat_id) ? chat_id[0] : chat_id;
+	const isNewChat = !chatIdParam;
+
+	const {
+		token,
+		isLoading,
+		setIsLoading,
+		searchMode,
+		setSearchMode,
+		researchMode,
+		setResearchMode,
+		selectedConnectors,
+		setSelectedConnectors,
+		selectedDocuments,
+		setSelectedDocuments,
+	} = useChatState({
+		search_space_id: search_space_id as string,
+		chat_id: chatIdParam,
+	});
+
+	const { fetchChatDetails, updateChat, createChat } = useChatAPI({
+		token,
+		search_space_id: search_space_id as string,
+	});
+
+	// Memoize document IDs to prevent infinite re-renders
+	const documentIds = useMemo(() => {
+		return selectedDocuments.map((doc) => doc.id);
+	}, [selectedDocuments]);
+
+	// Memoize connector types to prevent infinite re-renders
+	const connectorTypes = useMemo(() => {
+		return selectedConnectors;
+	}, [selectedConnectors]);
+
+	// Unified localStorage management for chat state
+	interface ChatState {
+		selectedDocuments: Document[];
+		selectedConnectors: string[];
+		searchMode: "DOCUMENTS" | "CHUNKS";
+		researchMode: ResearchMode;
+	}
+
+	const getChatStateStorageKey = (searchSpaceId: string, chatId: string) =>
+		`surfsense_chat_state_${searchSpaceId}_${chatId}`;
+
+	const storeChatState = (
+		searchSpaceId: string,
+		chatId: string,
+		state: ChatState,
+	) => {
+		const key = getChatStateStorageKey(searchSpaceId, chatId);
+		localStorage.setItem(key, JSON.stringify(state));
+	};
+
+	const restoreChatState = (
+		searchSpaceId: string,
+		chatId: string,
+	): ChatState | null => {
+		const key = getChatStateStorageKey(searchSpaceId, chatId);
+		const stored = localStorage.getItem(key);
+		if (stored) {
+			localStorage.removeItem(key); // Clean up after restoration
+			try {
+				return JSON.parse(stored);
+			} catch (error) {
+				console.error("Error parsing stored chat state:", error);
+				return null;
+			}
+		}
+		return null;
+	};
+
+	const handler = useChat({
+		api: `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chat`,
+		streamProtocol: "data",
+		initialMessages: [],
+		headers: {
+			...(token && { Authorization: `Bearer ${token}` }),
+		},
+		body: {
+			data: {
+				search_space_id: search_space_id,
+				selected_connectors: connectorTypes,
+				research_mode: researchMode,
+				search_mode: searchMode,
+				document_ids_to_add_in_context: documentIds,
+			},
+		},
+		onError: (error) => {
+			console.error("Chat error:", error);
+		},
+	});
+
+	const customHandlerAppend = async (
+		message: Message | CreateMessage,
+		chatRequestOptions?: { data?: any },
+	) => {
+		const newChatId = await createChat(
+			message.content,
+			researchMode,
+			selectedConnectors,
+		);
+		if (newChatId) {
+			// Store chat state before navigation
+			storeChatState(search_space_id as string, newChatId, {
+				selectedDocuments,
+				selectedConnectors,
+				searchMode,
+				researchMode,
+			});
+			router.replace(`/dashboard/${search_space_id}/researcher/${newChatId}`);
+		}
+		return newChatId;
+	};
+
+	useEffect(() => {
+		if (token && !isNewChat && chatIdParam) {
+			setIsLoading(true);
+			loadChatData(chatIdParam);
+		}
+	}, [token, isNewChat, chatIdParam]);
+
+	// Restore chat state from localStorage on page load
+	useEffect(() => {
+		if (chatIdParam && search_space_id) {
+			const restoredState = restoreChatState(
+				search_space_id as string,
+				chatIdParam,
+			);
+			if (restoredState) {
+				setSelectedDocuments(restoredState.selectedDocuments);
+				setSelectedConnectors(restoredState.selectedConnectors);
+				setSearchMode(restoredState.searchMode);
+				setResearchMode(restoredState.researchMode);
+			}
+		}
+	}, [
+		chatIdParam,
+		search_space_id,
+		setSelectedDocuments,
+		setSelectedConnectors,
+		setSearchMode,
+		setResearchMode,
+	]);
+
+	const loadChatData = async (chatId: string) => {
+		try {
+			const chatData = await fetchChatDetails(chatId);
+			if (!chatData) return;
+
+			// Update configuration from chat data
+			if (chatData.type) {
+				setResearchMode(chatData.type as ResearchMode);
+			}
+
+			if (
+				chatData.initial_connectors &&
+				Array.isArray(chatData.initial_connectors)
+			) {
+				setSelectedConnectors(chatData.initial_connectors);
+			}
+
+			// Load existing messages
+			if (chatData.messages && Array.isArray(chatData.messages)) {
+				if (
+					chatData.messages.length === 1 &&
+					chatData.messages[0].role === "user"
+				) {
+					// Single user message - append to trigger LLM response
+					handler.append({
+						role: "user",
+						content: chatData.messages[0].content,
+					});
+				} else if (chatData.messages.length > 1) {
+					// Multiple messages - set them all
+					handler.setMessages(chatData.messages);
+				}
+			}
+		} finally {
+			setIsLoading(false);
+		}
+	};
+
+	// Auto-update chat when messages change (only for existing chats)
+	useEffect(() => {
+		if (
+			!isNewChat &&
+			chatIdParam &&
+			handler.status === "ready" &&
+			handler.messages.length > 0 &&
+			handler.messages[handler.messages.length - 1]?.role === "assistant"
+		) {
+			updateChat(
+				chatIdParam,
+				handler.messages,
+				researchMode,
+				selectedConnectors,
+			);
+		}
+	}, [handler.messages, handler.status, chatIdParam, isNewChat]);
+
+	if (isLoading) {
+		return (
+			<div className="flex items-center justify-center h-full">
+				<div>Loading...</div>
+			</div>
+		);
+	}
+
+	return (
+		<ChatInterface
+			handler={{
+				...handler,
+				append: isNewChat ? customHandlerAppend : handler.append,
+			}}
+			onDocumentSelectionChange={setSelectedDocuments}
+			selectedDocuments={selectedDocuments}
+			onConnectorSelectionChange={setSelectedConnectors}
+			selectedConnectors={selectedConnectors}
+			searchMode={searchMode}
+			onSearchModeChange={setSearchMode}
+			researchMode={researchMode}
+			onResearchModeChange={setResearchMode}
+		/>
+	);
+}

surfsense_web/app/dashboard/[search_space_id]/researcher/page.tsx

@@ -1,80 +0,0 @@
-"use client";
-import React, { useEffect } from 'react';
-import { useParams, useRouter } from 'next/navigation';
-import { Loader2 } from 'lucide-react';
-
-const ResearcherPage = () => {
-  const router = useRouter();
-  const { search_space_id } = useParams();
-  const [isCreating, setIsCreating] = React.useState(true);
-  const [error, setError] = React.useState<string | null>(null);
-
-  useEffect(() => {
-    const createChat = async () => {
-      try {
-        // Get token from localStorage
-        const token = localStorage.getItem('surfsense_bearer_token');
-        
-        if (!token) {
-          setError('Authentication token not found');
-          setIsCreating(false);
-          return;
-        }
-
-        // Create a new chat
-        const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/`, {
-          method: 'POST',
-          headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${token}`
-          },
-          body: JSON.stringify({
-            type: "QNA",
-            title: "Untitled Chat", // Empty title initially
-            initial_connectors: [], // No default connectors
-            messages: [],
-            search_space_id: Number(search_space_id)
-          })
-        });
-
-        if (!response.ok) {
-          throw new Error(`Failed to create chat: ${response.statusText}`);
-        }
-
-        const data = await response.json();
-        
-        // Redirect to the new chat page
-        router.push(`/dashboard/${search_space_id}/researcher/${data.id}`);
-      } catch (err) {
-        console.error('Error creating chat:', err);
-        setError(err instanceof Error ? err.message : 'Failed to create chat');
-        setIsCreating(false);
-      }
-    };
-
-    createChat();
-  }, [search_space_id, router]);
-
-  if (error) {
-    return (
-      <div className="flex flex-col items-center justify-center min-h-[calc(100vh-4rem)]">
-        <div className="text-red-500 mb-4">Error: {error}</div>
-        <button 
-          onClick={() => location.reload()}
-          className="px-4 py-2 bg-primary text-white rounded-md"
-        >
-          Try Again
-        </button>
-      </div>
-    );
-  }
-
-  return (
-    <div className="flex flex-col items-center justify-center min-h-[calc(100vh-4rem)]">
-      <Loader2 className="h-8 w-8 animate-spin text-primary mb-4" />
-      <p className="text-muted-foreground">Creating new research chat...</p>
-    </div>
-  );
-};
-
-export default ResearcherPage;

surfsense_web/app/globals.css

@@ -156,3 +156,5 @@
 button {
   cursor: pointer;
 }
+
+@source '../node_modules/@llamaindex/chat-ui/**/*.{ts,tsx}'

surfsense_web/components/chat/ChatCitation.tsx

@@ -0,0 +1,62 @@
+"use client";
+
+import React from "react";
+import {
+	Popover,
+	PopoverContent,
+	PopoverTrigger,
+} from "@/components/ui/popover";
+import { ExternalLink } from "lucide-react";
+
+export const CitationDisplay: React.FC<{ index: number; node: any }> = ({
+	index,
+	node,
+}) => {
+	const truncateText = (text: string, maxLength: number = 200) => {
+		if (text.length <= maxLength) return text;
+		return text.substring(0, maxLength) + "...";
+	};
+
+	const handleUrlClick = (e: React.MouseEvent, url: string) => {
+		e.preventDefault();
+		e.stopPropagation();
+		window.open(url, "_blank", "noopener,noreferrer");
+	};
+
+	return (
+		<Popover>
+			<PopoverTrigger asChild>
+				<span className="text-[10px] font-bold bg-slate-500 hover:bg-slate-600 text-white rounded-full w-4 h-4 inline-flex items-center justify-center align-super cursor-pointer transition-colors">
+					{index + 1}
+				</span>
+			</PopoverTrigger>
+			<PopoverContent className="w-80 p-4 space-y-3 relative" align="start">
+				{/* External Link Button - Top Right */}
+				{node?.url && (
+					<button
+						onClick={(e) => handleUrlClick(e, node.url)}
+						className="absolute top-3 right-3 inline-flex items-center justify-center w-6 h-6 text-blue-600 hover:text-blue-800 dark:text-blue-400 dark:hover:text-blue-200 hover:bg-blue-50 dark:hover:bg-blue-900/20 rounded transition-colors"
+						title="Open in new tab"
+					>
+						<ExternalLink size={14} />
+					</button>
+				)}
+
+				{/* Heading */}
+				<div className="text-sm font-semibold text-slate-900 dark:text-slate-100 pr-8">
+					{node?.metadata?.group_name || "Source"}
+				</div>
+
+				{/* Source */}
+				<div className="text-xs text-slate-600 dark:text-slate-400 font-medium">
+					{node?.metadata?.title || "Untitled"}
+				</div>
+
+				{/* Body */}
+				<div className="text-xs text-slate-700 dark:text-slate-300 leading-relaxed">
+					{truncateText(node?.text || "No content available")}
+				</div>
+			</PopoverContent>
+		</Popover>
+	);
+};

surfsense_web/components/chat/ChatFurtherQuestions.tsx

@@ -0,0 +1,45 @@
+"use client";
+
+import { SuggestedQuestions } from "@llamaindex/chat-ui/widgets";
+import { getAnnotationData, Message, useChatUI } from "@llamaindex/chat-ui";
+import {
+	Accordion,
+	AccordionContent,
+	AccordionItem,
+	AccordionTrigger,
+} from "@/components/ui/accordion";
+
+export const ChatFurtherQuestions: React.FC<{ message: Message }> = ({
+	message,
+}) => {
+	const annotations: string[][] = getAnnotationData(
+		message,
+		"FURTHER_QUESTIONS",
+	);
+	const { append, requestData } = useChatUI();
+
+	if (annotations.length !== 1 || annotations[0].length === 0) {
+		return <></>;
+	}
+
+	return (
+		<Accordion
+			type="single"
+			collapsible
+			className="w-full px-2 border-2 rounded-lg shadow-lg"
+		>
+			<AccordionItem value="suggested-questions">
+				<AccordionTrigger className="text-sm font-semibold">
+					Suggested Questions
+				</AccordionTrigger>
+				<AccordionContent>
+					<SuggestedQuestions
+						questions={annotations[0]}
+						append={append}
+						requestData={requestData}
+					/>
+				</AccordionContent>
+			</AccordionItem>
+		</Accordion>
+	);
+};

surfsense_web/components/chat/ChatInputGroup.tsx

@@ -0,0 +1,617 @@
+"use client";
+
+import { ChatInput } from "@llamaindex/chat-ui";
+import { FolderOpen, Check, Zap, Brain } from "lucide-react";
+import { Button } from "@/components/ui/button";
+import {
+	Dialog,
+	DialogContent,
+	DialogDescription,
+	DialogTitle,
+	DialogTrigger,
+	DialogFooter,
+} from "@/components/ui/dialog";
+import {
+	Select,
+	SelectContent,
+	SelectItem,
+	SelectTrigger,
+	SelectValue,
+} from "@/components/ui/select";
+import { Badge } from "@/components/ui/badge";
+import { Suspense, useState, useCallback } from "react";
+import { useParams } from "next/navigation";
+import { useDocuments, Document } from "@/hooks/use-documents";
+import { DocumentsDataTable } from "@/components/chat/DocumentsDataTable";
+import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors";
+import {
+	getConnectorIcon,
+	ConnectorButton as ConnectorButtonComponent,
+} from "@/components/chat/ConnectorComponents";
+import { ResearchMode } from "@/components/chat";
+import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs";
+import React from "react";
+
+const DocumentSelector = React.memo(
+	({
+		onSelectionChange,
+		selectedDocuments = [],
+	}: {
+		onSelectionChange?: (documents: Document[]) => void;
+		selectedDocuments?: Document[];
+	}) => {
+		const { search_space_id } = useParams();
+		const [isOpen, setIsOpen] = useState(false);
+
+		const { documents, loading, isLoaded, fetchDocuments } = useDocuments(
+			Number(search_space_id),
+			true,
+		);
+
+		const handleOpenChange = useCallback(
+			(open: boolean) => {
+				setIsOpen(open);
+				if (open && !isLoaded) {
+					fetchDocuments();
+				}
+			},
+			[fetchDocuments, isLoaded],
+		);
+
+		const handleSelectionChange = useCallback(
+			(documents: Document[]) => {
+				onSelectionChange?.(documents);
+			},
+			[onSelectionChange],
+		);
+
+		const handleDone = useCallback(() => {
+			setIsOpen(false);
+		}, []);
+
+		const selectedCount = React.useMemo(
+			() => selectedDocuments.length,
+			[selectedDocuments.length],
+		);
+
+		return (
+			<Dialog open={isOpen} onOpenChange={handleOpenChange}>
+				<DialogTrigger asChild>
+					<Button variant="outline" className="relative">
+						<FolderOpen className="w-4 h-4" />
+						{selectedCount > 0 && (
+							<span className="absolute -top-2 -right-2 h-5 w-5 rounded-full bg-primary text-primary-foreground text-xs flex items-center justify-center">
+								{selectedCount}
+							</span>
+						)}
+					</Button>
+				</DialogTrigger>
+
+				<DialogContent className="max-w-[95vw] md:max-w-5xl h-[90vh] md:h-[85vh] p-0 flex flex-col">
+					<div className="flex flex-col h-full">
+						<div className="px-4 md:px-6 py-4 border-b flex-shrink-0">
+							<DialogTitle className="text-lg md:text-xl">
+								Select Documents
+							</DialogTitle>
+							<DialogDescription className="mt-1 text-sm">
+								Choose documents to include in your research context
+							</DialogDescription>
+						</div>
+
+						<div className="flex-1 min-h-0 p-4 md:p-6">
+							{loading ? (
+								<div className="flex items-center justify-center h-full">
+									<div className="text-center space-y-2">
+										<div className="animate-spin h-8 w-8 border-2 border-primary border-t-transparent rounded-full mx-auto" />
+										<p className="text-sm text-muted-foreground">
+											Loading documents...
+										</p>
+									</div>
+								</div>
+							) : isLoaded ? (
+								<DocumentsDataTable
+									documents={documents}
+									onSelectionChange={handleSelectionChange}
+									onDone={handleDone}
+									initialSelectedDocuments={selectedDocuments}
+								/>
+							) : null}
+						</div>
+					</div>
+				</DialogContent>
+			</Dialog>
+		);
+	},
+);
+
+DocumentSelector.displayName = "DocumentSelector";
+
+const ConnectorSelector = React.memo(
+	({
+		onSelectionChange,
+		selectedConnectors = [],
+	}: {
+		onSelectionChange?: (connectorTypes: string[]) => void;
+		selectedConnectors?: string[];
+	}) => {
+		const [isOpen, setIsOpen] = useState(false);
+
+		const { connectorSourceItems, isLoading, isLoaded, fetchConnectors } =
+			useSearchSourceConnectors(true);
+
+		const handleOpenChange = useCallback(
+			(open: boolean) => {
+				setIsOpen(open);
+				if (open && !isLoaded) {
+					fetchConnectors();
+				}
+			},
+			[fetchConnectors, isLoaded],
+		);
+
+		const handleConnectorToggle = useCallback(
+			(connectorType: string) => {
+				const isSelected = selectedConnectors.includes(connectorType);
+				const newSelection = isSelected
+					? selectedConnectors.filter((type) => type !== connectorType)
+					: [...selectedConnectors, connectorType];
+				onSelectionChange?.(newSelection);
+			},
+			[selectedConnectors, onSelectionChange],
+		);
+
+		const handleSelectAll = useCallback(() => {
+			onSelectionChange?.(connectorSourceItems.map((c) => c.type));
+		}, [connectorSourceItems, onSelectionChange]);
+
+		const handleClearAll = useCallback(() => {
+			onSelectionChange?.([]);
+		}, [onSelectionChange]);
+
+		return (
+			<Dialog open={isOpen} onOpenChange={handleOpenChange}>
+				<DialogTrigger asChild>
+					<ConnectorButtonComponent
+						selectedConnectors={selectedConnectors}
+						onClick={() => setIsOpen(true)}
+						connectorSources={connectorSourceItems}
+					/>
+				</DialogTrigger>
+
+				<DialogContent className="sm:max-w-md">
+					<DialogTitle>Select Connectors</DialogTitle>
+					<DialogDescription>
+						Choose which data sources to include in your research
+					</DialogDescription>
+
+					{/* Connector selection grid */}
+					<div className="grid grid-cols-2 gap-4 py-4">
+						{isLoading ? (
+							<div className="col-span-2 flex justify-center py-4">
+								<div className="animate-spin h-6 w-6 border-2 border-primary border-t-transparent rounded-full" />
+							</div>
+						) : (
+							connectorSourceItems.map((connector) => {
+								const isSelected = selectedConnectors.includes(connector.type);
+
+								return (
+									<div
+										key={connector.id}
+										className={`flex items-center gap-2 p-2 rounded-md border cursor-pointer transition-colors ${
+											isSelected
+												? "border-primary bg-primary/10"
+												: "border-border hover:border-primary/50 hover:bg-muted"
+										}`}
+										onClick={() => handleConnectorToggle(connector.type)}
+										role="checkbox"
+										aria-checked={isSelected}
+										tabIndex={0}
+									>
+										<div className="flex-shrink-0 w-6 h-6 flex items-center justify-center rounded-full bg-muted">
+											{getConnectorIcon(connector.type)}
+										</div>
+										<span className="flex-1 text-sm font-medium">
+											{connector.name}
+										</span>
+										{isSelected && <Check className="h-4 w-4 text-primary" />}
+									</div>
+								);
+							})
+						)}
+					</div>
+
+					<DialogFooter className="flex justify-between items-center">
+						<div className="flex gap-2">
+							<Button variant="outline" onClick={handleClearAll}>
+								Clear All
+							</Button>
+							<Button onClick={handleSelectAll}>Select All</Button>
+						</div>
+					</DialogFooter>
+				</DialogContent>
+			</Dialog>
+		);
+	},
+);
+
+ConnectorSelector.displayName = "ConnectorSelector";
+
+const SearchModeSelector = React.memo(
+	({
+		searchMode,
+		onSearchModeChange,
+	}: {
+		searchMode?: "DOCUMENTS" | "CHUNKS";
+		onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void;
+	}) => {
+		const handleDocumentsClick = React.useCallback(() => {
+			onSearchModeChange?.("DOCUMENTS");
+		}, [onSearchModeChange]);
+
+		const handleChunksClick = React.useCallback(() => {
+			onSearchModeChange?.("CHUNKS");
+		}, [onSearchModeChange]);
+
+		return (
+			<div className="flex items-center gap-1 sm:gap-2">
+				<span className="text-xs text-muted-foreground hidden sm:block">
+					Scope:
+				</span>
+				<div className="flex rounded-md border border-border overflow-hidden">
+					<Button
+						variant={searchMode === "DOCUMENTS" ? "default" : "ghost"}
+						size="sm"
+						className="rounded-none border-r h-8 px-2 sm:px-3 text-xs transition-all duration-200 hover:bg-muted/80"
+						onClick={handleDocumentsClick}
+					>
+						<span className="hidden sm:inline">Documents</span>
+						<span className="sm:hidden">Docs</span>
+					</Button>
+					<Button
+						variant={searchMode === "CHUNKS" ? "default" : "ghost"}
+						size="sm"
+						className="rounded-none h-8 px-2 sm:px-3 text-xs transition-all duration-200 hover:bg-muted/80"
+						onClick={handleChunksClick}
+					>
+						Chunks
+					</Button>
+				</div>
+			</div>
+		);
+	},
+);
+
+SearchModeSelector.displayName = "SearchModeSelector";
+
+const ResearchModeSelector = React.memo(
+	({
+		researchMode,
+		onResearchModeChange,
+	}: {
+		researchMode?: ResearchMode;
+		onResearchModeChange?: (mode: ResearchMode) => void;
+	}) => {
+		const handleValueChange = React.useCallback(
+			(value: string) => {
+				onResearchModeChange?.(value as ResearchMode);
+			},
+			[onResearchModeChange],
+		);
+
+		// Memoize mode options to prevent recreation
+		const modeOptions = React.useMemo(
+			() => [
+				{ value: "QNA", label: "Q&A", shortLabel: "Q&A" },
+				{
+					value: "REPORT_GENERAL",
+					label: "General Report",
+					shortLabel: "General",
+				},
+				{
+					value: "REPORT_DEEP",
+					label: "Deep Report",
+					shortLabel: "Deep",
+				},
+				{
+					value: "REPORT_DEEPER",
+					label: "Deeper Report",
+					shortLabel: "Deeper",
+				},
+			],
+			[],
+		);
+
+		return (
+			<div className="flex items-center gap-1 sm:gap-2">
+				<span className="text-xs text-muted-foreground hidden sm:block">
+					Mode:
+				</span>
+				<Select value={researchMode} onValueChange={handleValueChange}>
+					<SelectTrigger className="w-auto min-w-[80px] sm:min-w-[120px] h-8 text-xs border-border bg-background hover:bg-muted/50 transition-colors duration-200 focus:ring-2 focus:ring-primary/20">
+						<SelectValue placeholder="Mode" className="text-xs" />
+					</SelectTrigger>
+					<SelectContent align="end" className="min-w-[140px]">
+						<div className="px-2 py-1.5 text-xs font-medium text-muted-foreground border-b bg-muted/30">
+							Research Mode
+						</div>
+						{modeOptions.map((option) => (
+							<SelectItem
+								key={option.value}
+								value={option.value}
+								className="px-3 py-2 cursor-pointer hover:bg-accent/50 focus:bg-accent"
+							>
+								<span className="hidden sm:inline">{option.label}</span>
+								<span className="sm:hidden">{option.shortLabel}</span>
+							</SelectItem>
+						))}
+					</SelectContent>
+				</Select>
+			</div>
+		);
+	},
+);
+
+ResearchModeSelector.displayName = "ResearchModeSelector";
+
+const LLMSelector = React.memo(() => {
+	const { llmConfigs, loading: llmLoading, error } = useLLMConfigs();
+	const {
+		preferences,
+		updatePreferences,
+		loading: preferencesLoading,
+	} = useLLMPreferences();
+
+	const isLoading = llmLoading || preferencesLoading;
+
+	// Memoize the selected config to avoid repeated lookups
+	const selectedConfig = React.useMemo(() => {
+		if (!preferences.fast_llm_id || !llmConfigs.length) return null;
+		return (
+			llmConfigs.find((config) => config.id === preferences.fast_llm_id) || null
+		);
+	}, [preferences.fast_llm_id, llmConfigs]);
+
+	// Memoize the display value for the trigger
+	const displayValue = React.useMemo(() => {
+		if (!selectedConfig) return null;
+		return (
+			<div className="flex items-center gap-1">
+				<span className="font-medium text-xs">{selectedConfig.provider}</span>
+				<span className="text-muted-foreground">•</span>
+				<span className="hidden sm:inline text-muted-foreground text-xs truncate max-w-[60px]">
+					{selectedConfig.name}
+				</span>
+			</div>
+		);
+	}, [selectedConfig]);
+
+	const handleValueChange = React.useCallback(
+		(value: string) => {
+			const llmId = value ? parseInt(value, 10) : undefined;
+			updatePreferences({ fast_llm_id: llmId });
+		},
+		[updatePreferences],
+	);
+
+	// Loading skeleton
+	if (isLoading) {
+		return (
+			<div className="h-8 min-w-[100px] sm:min-w-[120px]">
+				<div className="h-8 rounded-md bg-muted animate-pulse flex items-center px-3">
+					<div className="w-3 h-3 rounded bg-muted-foreground/20 mr-2" />
+					<div className="h-3 w-16 rounded bg-muted-foreground/20" />
+				</div>
+			</div>
+		);
+	}
+
+	// Error state
+	if (error) {
+		return (
+			<div className="h-8 min-w-[100px] sm:min-w-[120px]">
+				<Button
+					variant="outline"
+					size="sm"
+					className="h-8 px-3 text-xs text-destructive border-destructive/50 hover:bg-destructive/10"
+					disabled
+				>
+					<span className="text-xs">Error</span>
+				</Button>
+			</div>
+		);
+	}
+
+	return (
+		<div className="h-8 min-w-0">
+			<Select
+				value={preferences.fast_llm_id?.toString() || ""}
+				onValueChange={handleValueChange}
+				disabled={isLoading}
+			>
+				<SelectTrigger className="h-8 w-auto min-w-[100px] sm:min-w-[120px] px-3 text-xs border-border bg-background hover:bg-muted/50 transition-colors duration-200 focus:ring-2 focus:ring-primary/20">
+					<div className="flex items-center gap-2 min-w-0">
+						<Zap className="h-3 w-3 text-primary flex-shrink-0" />
+						<SelectValue placeholder="Fast LLM" className="text-xs">
+							{displayValue || (
+								<span className="text-muted-foreground">Select LLM</span>
+							)}
+						</SelectValue>
+					</div>
+				</SelectTrigger>
+
+				<SelectContent align="end" className="w-[300px] max-h-[400px]">
+					<div className="px-3 py-2 text-xs font-medium text-muted-foreground border-b bg-muted/30">
+						<div className="flex items-center gap-2">
+							<Zap className="h-3 w-3" />
+							Fast LLM Selection
+						</div>
+					</div>
+
+					{llmConfigs.length === 0 ? (
+						<div className="px-4 py-6 text-center">
+							<div className="mx-auto w-12 h-12 rounded-full bg-muted flex items-center justify-center mb-3">
+								<Brain className="h-5 w-5 text-muted-foreground" />
+							</div>
+							<h4 className="text-sm font-medium mb-1">
+								No LLM configurations
+							</h4>
+							<p className="text-xs text-muted-foreground mb-3">
+								Configure AI models to get started
+							</p>
+							<Button
+								variant="outline"
+								size="sm"
+								className="text-xs"
+								onClick={() => window.open("/settings", "_blank")}
+							>
+								Open Settings
+							</Button>
+						</div>
+					) : (
+						<div className="py-1">
+							{llmConfigs.map((config) => (
+								<SelectItem
+									key={config.id}
+									value={config.id.toString()}
+									className="px-3 py-2 cursor-pointer hover:bg-accent/50 focus:bg-accent"
+								>
+									<div className="flex items-center justify-between w-full min-w-0">
+										<div className="flex items-center gap-3 min-w-0 flex-1">
+											<div className="flex h-8 w-8 items-center justify-center rounded-md bg-primary/10 flex-shrink-0">
+												<Brain className="h-4 w-4 text-primary" />
+											</div>
+											<div className="min-w-0 flex-1">
+												<div className="flex items-center gap-2 mb-1">
+													<span className="font-medium text-sm truncate">
+														{config.name}
+													</span>
+													<Badge
+														variant="outline"
+														className="text-xs px-1.5 py-0.5 flex-shrink-0"
+													>
+														{config.provider}
+													</Badge>
+												</div>
+												<p className="text-xs text-muted-foreground font-mono truncate">
+													{config.model_name}
+												</p>
+											</div>
+										</div>
+										{preferences.fast_llm_id === config.id && (
+											<div className="flex h-5 w-5 items-center justify-center rounded-full bg-primary ml-2 flex-shrink-0">
+												<Check className="h-3 w-3 text-primary-foreground" />
+											</div>
+										)}
+									</div>
+								</SelectItem>
+							))}
+						</div>
+					)}
+				</SelectContent>
+			</Select>
+		</div>
+	);
+});
+
+LLMSelector.displayName = "LLMSelector";
+
+const CustomChatInputOptions = React.memo(
+	({
+		onDocumentSelectionChange,
+		selectedDocuments,
+		onConnectorSelectionChange,
+		selectedConnectors,
+		searchMode,
+		onSearchModeChange,
+		researchMode,
+		onResearchModeChange,
+	}: {
+		onDocumentSelectionChange?: (documents: Document[]) => void;
+		selectedDocuments?: Document[];
+		onConnectorSelectionChange?: (connectorTypes: string[]) => void;
+		selectedConnectors?: string[];
+		searchMode?: "DOCUMENTS" | "CHUNKS";
+		onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void;
+		researchMode?: ResearchMode;
+		onResearchModeChange?: (mode: ResearchMode) => void;
+	}) => {
+		// Memoize the loading fallback to prevent recreation
+		const loadingFallback = React.useMemo(
+			() => (
+				<div className="h-8 min-w-[100px] animate-pulse bg-muted rounded-md" />
+			),
+			[],
+		);
+
+		return (
+			<div className="flex flex-wrap gap-2 sm:gap-3 items-center justify-start">
+				<Suspense fallback={loadingFallback}>
+					<DocumentSelector
+						onSelectionChange={onDocumentSelectionChange}
+						selectedDocuments={selectedDocuments}
+					/>
+				</Suspense>
+				<Suspense fallback={loadingFallback}>
+					<ConnectorSelector
+						onSelectionChange={onConnectorSelectionChange}
+						selectedConnectors={selectedConnectors}
+					/>
+				</Suspense>
+				<SearchModeSelector
+					searchMode={searchMode}
+					onSearchModeChange={onSearchModeChange}
+				/>
+				<ResearchModeSelector
+					researchMode={researchMode}
+					onResearchModeChange={onResearchModeChange}
+				/>
+				<LLMSelector />
+			</div>
+		);
+	},
+);
+
+CustomChatInputOptions.displayName = "CustomChatInputOptions";
+
+export const ChatInputUI = React.memo(
+	({
+		onDocumentSelectionChange,
+		selectedDocuments,
+		onConnectorSelectionChange,
+		selectedConnectors,
+		searchMode,
+		onSearchModeChange,
+		researchMode,
+		onResearchModeChange,
+	}: {
+		onDocumentSelectionChange?: (documents: Document[]) => void;
+		selectedDocuments?: Document[];
+		onConnectorSelectionChange?: (connectorTypes: string[]) => void;
+		selectedConnectors?: string[];
+		searchMode?: "DOCUMENTS" | "CHUNKS";
+		onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void;
+		researchMode?: ResearchMode;
+		onResearchModeChange?: (mode: ResearchMode) => void;
+	}) => {
+		return (
+			<ChatInput>
+				<ChatInput.Form className="flex gap-2">
+					<ChatInput.Field className="flex-1" />
+					<ChatInput.Submit />
+				</ChatInput.Form>
+				<CustomChatInputOptions
+					onDocumentSelectionChange={onDocumentSelectionChange}
+					selectedDocuments={selectedDocuments}
+					onConnectorSelectionChange={onConnectorSelectionChange}
+					selectedConnectors={selectedConnectors}
+					searchMode={searchMode}
+					onSearchModeChange={onSearchModeChange}
+					researchMode={researchMode}
+					onResearchModeChange={onResearchModeChange}
+				/>
+			</ChatInput>
+		);
+	},
+);
+
+ChatInputUI.displayName = "ChatInputUI";

surfsense_web/components/chat/ChatInterface.tsx

@@ -0,0 +1,55 @@
+"use client";
+
+import React from "react";
+import {
+	ChatSection as LlamaIndexChatSection,
+	ChatHandler,
+} from "@llamaindex/chat-ui";
+import { Document } from "@/hooks/use-documents";
+import { ChatInputUI } from "@/components/chat/ChatInputGroup";
+import { ResearchMode } from "@/components/chat";
+import { ChatMessagesUI } from "@/components/chat/ChatMessages";
+
+interface ChatInterfaceProps {
+	handler: ChatHandler;
+	onDocumentSelectionChange?: (documents: Document[]) => void;
+	selectedDocuments?: Document[];
+	onConnectorSelectionChange?: (connectorTypes: string[]) => void;
+	selectedConnectors?: string[];
+	searchMode?: "DOCUMENTS" | "CHUNKS";
+	onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void;
+	researchMode?: ResearchMode;
+	onResearchModeChange?: (mode: ResearchMode) => void;
+}
+
+export default function ChatInterface({
+	handler,
+	onDocumentSelectionChange,
+	selectedDocuments = [],
+	onConnectorSelectionChange,
+	selectedConnectors = [],
+	searchMode,
+	onSearchModeChange,
+	researchMode,
+	onResearchModeChange,
+}: ChatInterfaceProps) {
+	return (
+		<LlamaIndexChatSection handler={handler} className="flex h-full">
+			<div className="flex flex-1 flex-col">
+				<ChatMessagesUI />
+				<div className="border-t p-4">
+					<ChatInputUI
+						onDocumentSelectionChange={onDocumentSelectionChange}
+						selectedDocuments={selectedDocuments}
+						onConnectorSelectionChange={onConnectorSelectionChange}
+						selectedConnectors={selectedConnectors}
+						searchMode={searchMode}
+						onSearchModeChange={onSearchModeChange}
+						researchMode={researchMode}
+						onResearchModeChange={onResearchModeChange}
+					/>
+				</div>
+			</div>
+		</LlamaIndexChatSection>
+	);
+}

surfsense_web/components/chat/ChatMessages.tsx

@@ -0,0 +1,78 @@
+"use client";
+
+import React from "react";
+import {
+	ChatMessage as LlamaIndexChatMessage,
+	ChatMessages as LlamaIndexChatMessages,
+	Message,
+	useChatUI,
+} from "@llamaindex/chat-ui";
+import TerminalDisplay from "@/components/chat/ChatTerminal";
+import ChatSourcesDisplay from "@/components/chat/ChatSources";
+import { CitationDisplay } from "@/components/chat/ChatCitation";
+import { ChatFurtherQuestions } from "@/components/chat/ChatFurtherQuestions";
+
+export function ChatMessagesUI() {
+	const { messages } = useChatUI();
+
+	return (
+		<LlamaIndexChatMessages className="flex-1">
+			<LlamaIndexChatMessages.Empty heading="Welcome to Surfsense!" subheading="Ask me anything from your documents" />
+			<LlamaIndexChatMessages.List className="p-4">
+				{messages.map((message, index) => (
+					<ChatMessageUI
+						key={`Message-${index}`}
+						message={message}
+						isLast={index === messages.length - 1}
+					/>
+				))}
+			</LlamaIndexChatMessages.List>
+			<LlamaIndexChatMessages.Loading />
+		</LlamaIndexChatMessages>
+	);
+}
+
+function ChatMessageUI({
+	message,
+	isLast,
+}: {
+	message: Message;
+	isLast: boolean;
+}) {
+	const bottomRef = React.useRef<HTMLDivElement>(null);
+
+	React.useEffect(() => {
+		if (isLast && bottomRef.current) {
+			bottomRef.current.scrollIntoView({ behavior: "smooth" });
+		}
+	}, [message]);
+
+	return (
+		<LlamaIndexChatMessage
+			message={message}
+			isLast={isLast}
+			className="flex flex-col "
+		>
+			{message.role === "assistant" ? (
+				<div className="flex-1 flex flex-col space-y-4">
+					<TerminalDisplay message={message} open={isLast} />
+					<ChatSourcesDisplay message={message} />
+					<LlamaIndexChatMessage.Content className="flex-1">
+						<LlamaIndexChatMessage.Content.Markdown
+							citationComponent={CitationDisplay}
+						/>
+					</LlamaIndexChatMessage.Content>
+					<div ref={bottomRef} />
+					<div className="flex flex-row justify-end gap-2">
+						{isLast && <ChatFurtherQuestions message={message} />}
+						<LlamaIndexChatMessage.Actions className="flex-1 flex-col" />
+					</div>
+				</div>
+			) : (
+				<LlamaIndexChatMessage.Content className="flex-1">
+					<LlamaIndexChatMessage.Content.Markdown />
+				</LlamaIndexChatMessage.Content>
+			)}
+		</LlamaIndexChatMessage>
+	);
+}

surfsense_web/components/chat/ChatSources.tsx

@@ -0,0 +1,224 @@
+"use client";
+
+import { useState } from "react";
+import { getAnnotationData, Message } from "@llamaindex/chat-ui";
+import { Button } from "@/components/ui/button";
+import {
+	Dialog,
+	DialogContent,
+	DialogHeader,
+	DialogTitle,
+	DialogTrigger,
+} from "@/components/ui/dialog";
+import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
+import {
+	Card,
+	CardContent,
+	CardDescription,
+	CardHeader,
+	CardTitle,
+} from "@/components/ui/card";
+import { Badge } from "@/components/ui/badge";
+import { ExternalLink, FileText, Globe } from "lucide-react";
+import { IconBrandGithub } from "@tabler/icons-react";
+
+interface Source {
+	id: string;
+	title: string;
+	description: string;
+	url: string;
+}
+
+interface SourceGroup {
+	id: number;
+	name: string;
+	type: string;
+	sources: Source[];
+}
+
+// New interfaces for the updated data format
+interface NodeMetadata {
+	title: string;
+	source_type: string;
+	group_name: string;
+}
+
+interface SourceNode {
+	id: string;
+	text: string;
+	url: string;
+	metadata: NodeMetadata;
+}
+
+interface NodesResponse {
+	nodes: SourceNode[];
+}
+
+function getSourceIcon(type: string) {
+	switch (type) {
+		case "USER_SELECTED_GITHUB_CONNECTOR":
+		case "GITHUB_CONNECTOR":
+			return <IconBrandGithub className="h-4 w-4" />;
+		case "USER_SELECTED_NOTION_CONNECTOR":
+		case "NOTION_CONNECTOR":
+			return <FileText className="h-4 w-4" />;
+		case "USER_SELECTED_FILE":
+		case "FILE":
+			return <FileText className="h-4 w-4" />;
+		default:
+			return <Globe className="h-4 w-4" />;
+	}
+}
+
+function SourceCard({ source }: { source: Source }) {
+	const hasUrl = source.url && source.url.trim() !== "";
+
+	return (
+		<Card className="mb-3">
+			<CardHeader className="pb-2">
+				<div className="flex items-start justify-between gap-2">
+					<CardTitle className="text-sm md:text-base font-medium leading-tight">
+						{source.title}
+					</CardTitle>
+					{hasUrl && (
+						<Button
+							variant="ghost"
+							size="sm"
+							className="h-6 w-6 md:h-8 md:w-8 p-0 flex-shrink-0"
+							onClick={() => window.open(source.url, "_blank")}
+						>
+							<ExternalLink className="h-3 w-3 md:h-4 md:w-4" />
+						</Button>
+					)}
+				</div>
+			</CardHeader>
+			<CardContent className="pt-0">
+				<CardDescription className="text-xs md:text-sm line-clamp-3 md:line-clamp-4 leading-relaxed">
+					{source.description}
+				</CardDescription>
+			</CardContent>
+		</Card>
+	);
+}
+
+export default function ChatSourcesDisplay({ message }: { message: Message }) {
+	const [open, setOpen] = useState(false);
+	const annotations = getAnnotationData(message, "sources");
+
+	// Transform the new data format to the expected SourceGroup format
+	const sourceGroups: SourceGroup[] = [];
+
+	if (Array.isArray(annotations) && annotations.length > 0) {
+		// Extract all nodes from the response
+		const allNodes: SourceNode[] = [];
+
+		annotations.forEach((item) => {
+			if (
+				item &&
+				typeof item === "object" &&
+				"nodes" in item &&
+				Array.isArray(item.nodes)
+			) {
+				allNodes.push(...item.nodes);
+			}
+		});
+
+		// Group nodes by source_type
+		const groupedByType = allNodes.reduce(
+			(acc, node) => {
+				const sourceType = node.metadata.source_type;
+				if (!acc[sourceType]) {
+					acc[sourceType] = [];
+				}
+				acc[sourceType].push(node);
+				return acc;
+			},
+			{} as Record<string, SourceNode[]>,
+		);
+
+		// Convert grouped nodes to SourceGroup format
+		Object.entries(groupedByType).forEach(([sourceType, nodes], index) => {
+			if (nodes.length > 0) {
+				const firstNode = nodes[0];
+				sourceGroups.push({
+					id: index + 100, // Generate unique ID
+					name: firstNode.metadata.group_name,
+					type: sourceType,
+					sources: nodes.map((node) => ({
+						id: node.id,
+						title: node.metadata.title,
+						description: node.text,
+						url: node.url || "",
+					})),
+				});
+			}
+		});
+	}
+
+	if (sourceGroups.length === 0) {
+		return null;
+	}
+
+	const totalSources = sourceGroups.reduce(
+		(acc, group) => acc + group.sources.length,
+		0,
+	);
+
+	return (
+		<Dialog open={open} onOpenChange={setOpen}>
+			<DialogTrigger asChild>
+				<Button variant="outline" size="sm" className="w-fit">
+					<FileText className="h-4 w-4 mr-2" />
+					View Sources ({totalSources})
+				</Button>
+			</DialogTrigger>
+			<DialogContent className="max-w-4xl md:h-[80vh] h-[90vh] w-[95vw] md:w-auto flex flex-col">
+				<DialogHeader className="flex-shrink-0">
+					<DialogTitle>Sources</DialogTitle>
+				</DialogHeader>
+				<Tabs
+					defaultValue={sourceGroups[0]?.type}
+					className="flex-1 flex flex-col min-h-0"
+				>
+					<div className="flex-shrink-0 w-full overflow-x-auto">
+						<TabsList className="flex w-max min-w-full">
+							{sourceGroups.map((group) => (
+								<TabsTrigger
+									key={group.type}
+									value={group.type}
+									className="flex items-center gap-2 whitespace-nowrap px-3 md:px-4"
+								>
+									{getSourceIcon(group.type)}
+									<span className="truncate max-w-[100px] md:max-w-none">
+										{group.name}
+									</span>
+									<Badge
+										variant="secondary"
+										className="ml-1 h-5 text-xs flex-shrink-0"
+									>
+										{group.sources.length}
+									</Badge>
+								</TabsTrigger>
+							))}
+						</TabsList>
+					</div>
+					{sourceGroups.map((group) => (
+						<TabsContent
+							key={group.type}
+							value={group.type}
+							className="flex-1 min-h-0 mt-4"
+						>
+							<div className="h-full overflow-y-auto pr-2">
+								<div className="space-y-3">
+									{group.sources.map((source) => (
+										<SourceCard key={source.id} source={source} />
+									))}
+								</div>
+							</div>
+						</TabsContent>
+					))}
+				</Tabs>
+			</DialogContent>
+		</Dialog>
+	);
+}

surfsense_web/components/chat/ChatTerminal.tsx

@@ -0,0 +1,115 @@
+"use client";
+
+import React from "react";
+import { getAnnotationData, Message } from "@llamaindex/chat-ui";
+
+export default function TerminalDisplay({
+	message,
+	open,
+}: {
+	message: Message;
+	open: boolean;
+}) {
+	const [isCollapsed, setIsCollapsed] = React.useState(!open);
+
+	const bottomRef = React.useRef<HTMLDivElement>(null);
+
+	// Get the last assistant message that's not being typed
+	if (!message) {
+		return <></>;
+	}
+
+	interface TerminalInfo {
+		id: number;
+		text: string;
+		type: string;
+	}
+
+	const events = getAnnotationData(message, "TERMINAL_INFO") as TerminalInfo[];
+
+	if (events.length === 0) {
+		return <></>;
+	}
+
+	React.useEffect(() => {
+		if (bottomRef.current) {
+			bottomRef.current.scrollTo({
+				top: bottomRef.current.scrollHeight,
+				behavior: "smooth",
+			});
+		}
+	}, [events]);
+
+	return (
+		<div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden font-mono text-sm shadow-lg">
+			{/* Terminal Header */}
+			<div
+				className="bg-gray-800 px-4 py-2 flex items-center gap-2 border-b border-gray-700 cursor-pointer hover:bg-gray-750 transition-colors"
+				onClick={() => setIsCollapsed(!isCollapsed)}
+			>
+				<div className="flex gap-2">
+					<div className="w-3 h-3 rounded-full bg-red-500"></div>
+					<div className="w-3 h-3 rounded-full bg-yellow-500"></div>
+					<div className="w-3 h-3 rounded-full bg-green-500"></div>
+				</div>
+				<div className="text-gray-400 text-xs ml-2 flex-1">
+					Agent Process Terminal ({events.length} events)
+				</div>
+				<div className="text-gray-400">
+					{isCollapsed ? (
+						<svg
+							className="w-4 h-4"
+							fill="none"
+							stroke="currentColor"
+							viewBox="0 0 24 24"
+						>
+							<path
+								strokeLinecap="round"
+								strokeLinejoin="round"
+								strokeWidth={2}
+								d="M19 9l-7 7-7-7"
+							/>
+						</svg>
+					) : (
+						<svg
+							className="w-4 h-4"
+							fill="none"
+							stroke="currentColor"
+							viewBox="0 0 24 24"
+						>
+							<path
+								strokeLinecap="round"
+								strokeLinejoin="round"
+								strokeWidth={2}
+								d="M5 15l7-7 7 7"
+							/>
+						</svg>
+					)}
+				</div>
+			</div>
+
+			{/* Terminal Content */}
+			{!isCollapsed && (
+				<div
+					ref={bottomRef}
+					className="h-64 overflow-y-auto p-4 space-y-1 bg-gray-900"
+				>
+					{events.map((event, index) => (
+						<div key={`${event.id}-${index}`} className="text-green-400">
+							<span className="text-blue-400">$</span>
+							<span className="text-yellow-400 ml-2">[{event.type || ""}]</span>
+							<span className="text-gray-300 ml-4 mt-1 pl-2 border-l-2 border-gray-600">
+								{event.text || ""}...
+							</span>
+						</div>
+					))}
+					{events.length === 0 && (
+						<div className="text-gray-500 italic">
+							No agent events to display...
+						</div>
+					)}
+				</div>
+			)}
+		</div>
+	);
+}

surfsense_web/components/chat/DocumentsDataTable.tsx

@@ -0,0 +1,502 @@
+"use client";
+
+import * as React from "react";
+import {
+  ColumnDef,
+  ColumnFiltersState,
+  flexRender,
+  getCoreRowModel,
+  getFilteredRowModel,
+  getPaginationRowModel,
+  getSortedRowModel,
+  SortingState,
+  useReactTable,
+  VisibilityState,
+} from "@tanstack/react-table";
+import { ArrowUpDown, Calendar, FileText, Search } from "lucide-react";
+
+import { Button } from "@/components/ui/button";
+import { Checkbox } from "@/components/ui/checkbox";
+import {
+  Select,
+  SelectContent,
+  SelectItem,
+  SelectTrigger,
+  SelectValue,
+} from "@/components/ui/select";
+import { Input } from "@/components/ui/input";
+import {
+  Table,
+  TableBody,
+  TableCell,
+  TableHead,
+  TableHeader,
+  TableRow,
+} from "@/components/ui/table";
+import { Badge } from "@/components/ui/badge";
+import { Document, DocumentType } from "@/hooks/use-documents";
+
+interface DocumentsDataTableProps {
+  documents: Document[];
+  onSelectionChange: (documents: Document[]) => void;
+  onDone: () => void;
+  initialSelectedDocuments?: Document[];
+}
+
+const DOCUMENT_TYPES: (DocumentType | "ALL")[] = [
+  "ALL",
+  "FILE",
+  "EXTENSION",
+  "CRAWLED_URL",
+  "YOUTUBE_VIDEO",
+  "SLACK_CONNECTOR",
+  "NOTION_CONNECTOR",
+  "GITHUB_CONNECTOR",
+  "LINEAR_CONNECTOR",
+  "DISCORD_CONNECTOR",
+  "JIRA_CONNECTOR",
+];
+
+const getDocumentTypeColor = (type: DocumentType) => {
+  const colors = {
+    FILE: "bg-blue-50 text-blue-700 border-blue-200",
+    EXTENSION: "bg-green-50 text-green-700 border-green-200",
+    CRAWLED_URL: "bg-purple-50 text-purple-700 border-purple-200",
+    YOUTUBE_VIDEO: "bg-red-50 text-red-700 border-red-200",
+    SLACK_CONNECTOR: "bg-yellow-50 text-yellow-700 border-yellow-200",
+    NOTION_CONNECTOR: "bg-indigo-50 text-indigo-700 border-indigo-200",
+    GITHUB_CONNECTOR: "bg-gray-50 text-gray-700 border-gray-200",
+    LINEAR_CONNECTOR: "bg-pink-50 text-pink-700 border-pink-200",
+    DISCORD_CONNECTOR: "bg-violet-50 text-violet-700 border-violet-200",
+    JIRA_CONNECTOR: "bg-orange-50 text-orange-700 border-orange-200",
+  };
+  return colors[type] || "bg-gray-50 text-gray-700 border-gray-200";
+};
+
+const columns: ColumnDef<Document>[] = [
+  {
+    id: "select",
+    header: ({ table }) => (
+      <Checkbox
+        checked={
+          table.getIsAllPageRowsSelected() ||
+          (table.getIsSomePageRowsSelected() && "indeterminate")
+        }
+        onCheckedChange={(value) => table.toggleAllPageRowsSelected(!!value)}
+        aria-label="Select all"
+      />
+    ),
+    cell: ({ row }) => (
+      <Checkbox
+        checked={row.getIsSelected()}
+        onCheckedChange={(value) => row.toggleSelected(!!value)}
+        aria-label="Select row"
+      />
+    ),
+    enableSorting: false,
+    enableHiding: false,
+    size: 40,
+  },
+  {
+    accessorKey: "title",
+    header: ({ column }) => (
+      <Button
+        variant="ghost"
+        onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
+        className="h-8 px-1 sm:px-2 font-medium text-left justify-start"
+      >
+        <FileText className="mr-1 sm:mr-2 h-3 w-3 sm:h-4 sm:w-4 flex-shrink-0" />
+        <span className="hidden sm:inline">Title</span>
+        <span className="sm:hidden">Doc</span>
+        <ArrowUpDown className="ml-1 sm:ml-2 h-3 w-3 sm:h-4 sm:w-4 flex-shrink-0" />
+      </Button>
+    ),
+    cell: ({ row }) => {
+      const title = row.getValue("title") as string;
+      return (
+        <div
+          className="font-medium max-w-[120px] sm:max-w-[250px] truncate text-xs sm:text-sm"
+          title={title}
+        >
+          {title}
+        </div>
+      );
+    },
+  },
+  {
+    accessorKey: "document_type",
+    header: "Type",
+    cell: ({ row }) => {
+      const type = row.getValue("document_type") as DocumentType;
+      return (
+        <Badge
+          variant="outline"
+          className={`${getDocumentTypeColor(
+            type,
+          )} text-[10px] sm:text-xs px-1 sm:px-2`}
+        >
+          <span className="hidden sm:inline">{type.replace(/_/g, " ")}</span>
+          <span className="sm:hidden">{type.split("_")[0]}</span>
+        </Badge>
+      );
+    },
+    size: 80,
+    meta: {
+      className: "hidden sm:table-cell",
+    },
+  },
+  {
+    accessorKey: "content",
+    header: "Preview",
+    cell: ({ row }) => {
+      const content = row.getValue("content") as string;
+      return (
+        <div
+          className="text-muted-foreground max-w-[150px] sm:max-w-[350px] truncate text-[10px] sm:text-sm"
+          title={content}
+        >
+          <span className="sm:hidden">{content.substring(0, 30)}...</span>
+          <span className="hidden sm:inline">
+            {content.substring(0, 100)}...
+          </span>
+        </div>
+      );
+    },
+    enableSorting: false,
+    meta: {
+      className: "hidden md:table-cell",
+    },
+  },
+  {
+    accessorKey: "created_at",
+    header: ({ column }) => (
+      <Button
+        variant="ghost"
+        onClick={() => column.toggleSorting(column.getIsSorted() === "asc")}
+        className="h-8 px-1 sm:px-2 font-medium"
+      >
+        <Calendar className="mr-1 sm:mr-2 h-3 w-3 sm:h-4 sm:w-4 flex-shrink-0" />
+        <span className="hidden sm:inline">Created</span>
+        <span className="sm:hidden">Date</span>
+        <ArrowUpDown className="ml-1 sm:ml-2 h-3 w-3 sm:h-4 sm:w-4 flex-shrink-0" />
+      </Button>
+    ),
+    cell: ({ row }) => {
+      const date = new Date(row.getValue("created_at"));
+      return (
+        <div className="text-xs sm:text-sm whitespace-nowrap">
+          <span className="hidden sm:inline">
+            {date.toLocaleDateString("en-US", {
+              month: "short",
+              day: "numeric",
+              year: "numeric",
+            })}
+          </span>
+          <span className="sm:hidden">
+            {date.toLocaleDateString("en-US", {
+              month: "numeric",
+              day: "numeric",
+            })}
+          </span>
+        </div>
+      );
+    },
+    size: 80,
+  },
+];
+
+export function DocumentsDataTable({
+  documents,
+  onSelectionChange,
+  onDone,
+  initialSelectedDocuments = [],
+}: DocumentsDataTableProps) {
+  const [sorting, setSorting] = React.useState<SortingState>([]);
+  const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>(
+    [],
+  );
+  const [columnVisibility, setColumnVisibility] =
+    React.useState<VisibilityState>({});
+  const [documentTypeFilter, setDocumentTypeFilter] = React.useState<
+    DocumentType | "ALL"
+  >("ALL");
+
+  // Memoize initial row selection to prevent infinite loops
+  const initialRowSelection = React.useMemo(() => {
+    if (!documents.length || !initialSelectedDocuments.length) return {};
+
+    const selection: Record<string, boolean> = {};
+    initialSelectedDocuments.forEach((selectedDoc) => {
+      selection[selectedDoc.id] = true;
+    });
+    return selection;
+  }, [documents, initialSelectedDocuments]);
+
+  const [rowSelection, setRowSelection] = React.useState<
+    Record<string, boolean>
+  >({});
+
+  // Only update row selection when initialRowSelection actually changes and is not empty
+  React.useEffect(() => {
+    const hasChanges =
+      JSON.stringify(rowSelection) !== JSON.stringify(initialRowSelection);
+    if (hasChanges && Object.keys(initialRowSelection).length > 0) {
+      setRowSelection(initialRowSelection);
+    }
+  }, [initialRowSelection]);
+
+  // Initialize row selection on mount
+  React.useEffect(() => {
+    if (
+      Object.keys(rowSelection).length === 0 &&
+      Object.keys(initialRowSelection).length > 0
+    ) {
+      setRowSelection(initialRowSelection);
+    }
+  }, []);
+
+  const filteredDocuments = React.useMemo(() => {
+    if (documentTypeFilter === "ALL") return documents;
+    return documents.filter((doc) => doc.document_type === documentTypeFilter);
+  }, [documents, documentTypeFilter]);
+
+  const table = useReactTable({
+    data: filteredDocuments,
+    columns,
+    getRowId: (row) => row.id.toString(),
+    onSortingChange: setSorting,
+    onColumnFiltersChange: setColumnFilters,
+    getCoreRowModel: getCoreRowModel(),
+    getPaginationRowModel: getPaginationRowModel(),
+    getSortedRowModel: getSortedRowModel(),
+    getFilteredRowModel: getFilteredRowModel(),
+    onColumnVisibilityChange: setColumnVisibility,
+    onRowSelectionChange: setRowSelection,
+    initialState: { pagination: { pageSize: 10 } },
+    state: { sorting, columnFilters, columnVisibility, rowSelection },
+  });
+
+  React.useEffect(() => {
+    const selectedRows = table.getFilteredSelectedRowModel().rows;
+    const selectedDocuments = selectedRows.map((row) => row.original);
+    onSelectionChange(selectedDocuments);
+  }, [rowSelection, onSelectionChange, table]);
+
+  const handleClearAll = () => setRowSelection({});
+
+  const handleSelectPage = () => {
+    const currentPageRows = table.getRowModel().rows;
+    const newSelection = { ...rowSelection };
+    currentPageRows.forEach((row) => {
+      newSelection[row.id] = true;
+    });
+    setRowSelection(newSelection);
+  };
+
+  const handleSelectAllFiltered = () => {
+    const allFilteredRows = table.getFilteredRowModel().rows;
+    const newSelection: Record<string, boolean> = {};
+    allFilteredRows.forEach((row) => {
+      newSelection[row.id] = true;
+    });
+    setRowSelection(newSelection);
+  };
+
+  const selectedCount = table.getFilteredSelectedRowModel().rows.length;
+  const totalFiltered = table.getFilteredRowModel().rows.length;
+
+  return (
+    <div className="flex flex-col h-full space-y-3 md:space-y-4">
+      {/* Header Controls */}
+      <div className="space-y-3 md:space-y-4 flex-shrink-0">
+        {/* Search and Filter Row */}
+        <div className="flex flex-col sm:flex-row gap-3 sm:gap-4">
+          <div className="relative flex-1 max-w-full sm:max-w-sm">
+            <Search className="absolute left-3 top-1/2 transform -translate-y-1/2 h-4 w-4 text-muted-foreground" />
+            <Input
+              placeholder="Search documents..."
+              value={
+                (table.getColumn("title")?.getFilterValue() as string) ?? ""
+              }
+              onChange={(event) =>
+                table.getColumn("title")?.setFilterValue(event.target.value)
+              }
+              className="pl-10 text-sm"
+            />
+          </div>
+          <Select
+            value={documentTypeFilter}
+            onValueChange={(value) =>
+              setDocumentTypeFilter(value as DocumentType | "ALL")
+            }
+          >
+            <SelectTrigger className="w-full sm:w-[180px]">
+              <SelectValue />
+            </SelectTrigger>
+            <SelectContent>
+              {DOCUMENT_TYPES.map((type) => (
+                <SelectItem key={type} value={type}>
+                  {type === "ALL" ? "All Types" : type.replace(/_/g, " ")}
+                </SelectItem>
+              ))}
+            </SelectContent>
+          </Select>
+        </div>
+
+        {/* Action Controls Row */}
+        <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3">
+          <div className="flex flex-col sm:flex-row sm:items-center gap-2">
+            <span className="text-sm text-muted-foreground whitespace-nowrap">
+              {selectedCount} of {totalFiltered} selected
+            </span>
+            <div className="hidden sm:block h-4 w-px bg-border mx-2" />
+            <div className="flex items-center gap-2 flex-wrap">
+              <Button
+                variant="ghost"
+                size="sm"
+                onClick={handleClearAll}
+                disabled={selectedCount === 0}
+                className="text-xs sm:text-sm"
+              >
+                Clear All
+              </Button>
+              <Button
+                variant="ghost"
+                size="sm"
+                onClick={handleSelectPage}
+                className="text-xs sm:text-sm"
+              >
+                Select Page
+              </Button>
+              <Button
+                variant="ghost"
+                size="sm"
+                onClick={handleSelectAllFiltered}
+                className="text-xs sm:text-sm hidden sm:inline-flex"
+              >
+                Select All Filtered
+              </Button>
+              <Button
+                variant="ghost"
+                size="sm"
+                onClick={handleSelectAllFiltered}
+                className="text-xs sm:hidden"
+              >
+                Select All
+              </Button>
+            </div>
+          </div>
+          <Button
+            onClick={onDone}
+            disabled={selectedCount === 0}
+            className="w-full sm:w-auto sm:min-w-[100px]"
+          >
+            Done ({selectedCount})
+          </Button>
+        </div>
+      </div>
+
+      {/* Table Container */}
+      <div className="border rounded-lg flex-1 min-h-0 overflow-hidden bg-background">
+        <div className="overflow-auto h-full">
+          <Table>
+            <TableHeader className="sticky top-0 bg-background/95 backdrop-blur supports-[backdrop-filter]:bg-background/60 z-10">
+              {table.getHeaderGroups().map((headerGroup) => (
+                <TableRow key={headerGroup.id} className="border-b">
+                  {headerGroup.headers.map((header) => (
+                    <TableHead
+                      key={header.id}
+                      className="h-12 text-xs sm:text-sm"
+                    >
+                      {header.isPlaceholder
+                        ? null
+                        : flexRender(
+                            header.column.columnDef.header,
+                            header.getContext(),
+                          )}
+                    </TableHead>
+                  ))}
+                </TableRow>
+              ))}
+            </TableHeader>
+            <TableBody>
+              {table.getRowModel().rows?.length ? (
+                table.getRowModel().rows.map((row) => (
+                  <TableRow
+                    key={row.id}
+                    data-state={row.getIsSelected() && "selected"}
+                    className="hover:bg-muted/30"
+                  >
+                    {row.getVisibleCells().map((cell) => (
+                      <TableCell
+                        key={cell.id}
+                        className="py-3 text-xs sm:text-sm"
+                      >
+                        {flexRender(
+                          cell.column.columnDef.cell,
+                          cell.getContext(),
+                        )}
+                      </TableCell>
+                    ))}
+                  </TableRow>
+                ))
+              ) : (
+                <TableRow>
+                  <TableCell
+                    colSpan={columns.length}
+                    className="h-32 text-center text-muted-foreground text-sm"
+                  >
+                    No documents found.
+                  </TableCell>
+                </TableRow>
+              )}
+            </TableBody>
+          </Table>
+        </div>
+      </div>
+
+      {/* Footer Pagination */}
+      <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between gap-3 text-xs sm:text-sm text-muted-foreground border-t pt-3 md:pt-4 flex-shrink-0">
+        <div className="text-center sm:text-left">
+          Showing{" "}
+          {table.getState().pagination.pageIndex *
+            table.getState().pagination.pageSize +
+            1}{" "}
+          to{" "}
+          {Math.min(
+            (table.getState().pagination.pageIndex + 1) *
+              table.getState().pagination.pageSize,
+            table.getFilteredRowModel().rows.length,
+          )}{" "}
+          of {table.getFilteredRowModel().rows.length} documents
+        </div>
+        <div className="flex items-center justify-center sm:justify-end space-x-2">
+          <Button
+            variant="outline"
+            size="sm"
+            onClick={() => table.previousPage()}
+            disabled={!table.getCanPreviousPage()}
+            className="text-xs sm:text-sm"
+          >
+            Previous
+          </Button>
+          <div className="flex items-center space-x-1 text-xs sm:text-sm">
+            <span>Page</span>
+            <strong>{table.getState().pagination.pageIndex + 1}</strong>
+            <span>of</span>
+            <strong>{table.getPageCount()}</strong>
+          </div>
+          <Button
+            variant="outline"
+            size="sm"
+            onClick={() => table.nextPage()}
+            disabled={!table.getCanNextPage()}
+            className="text-xs sm:text-sm"
+          >
+            Next
+          </Button>
+        </div>
+      </div>
+    </div>
+  );
+}

surfsense_web/components/chat/connector-sources.ts

@@ -1,23 +0,0 @@
-// Connector sources
-export const connectorSourcesMenu = [
-    {
-      id: 1,
-      name: "Crawled URL",
-      type: "CRAWLED_URL",
-    },
-    {
-      id: 2,
-      name: "File",
-      type: "FILE",
-    }, 
-    {
-      id: 3,
-      name: "Extension",
-      type: "EXTENSION",
-    },
-    {
-      id: 4,
-      name: "Youtube Video",
-      type: "YOUTUBE_VIDEO",
-    }
-  ];

surfsense_web/components/ui/accordion.tsx

@@ -2,59 +2,65 @@
 
 import * as React from "react"
 import * as AccordionPrimitive from "@radix-ui/react-accordion"
-import { ChevronDown } from "lucide-react"
+import { ChevronDownIcon } from "lucide-react"
 
 import { cn } from "@/lib/utils"
 
-const Accordion = AccordionPrimitive.Root
+function Accordion({
+  ...props
+}: React.ComponentProps<typeof AccordionPrimitive.Root>) {
+  return <AccordionPrimitive.Root data-slot="accordion" {...props} />
+}
 
-const AccordionItem = React.forwardRef<
-  React.ElementRef<typeof AccordionPrimitive.Item>,
-  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Item>
->(({ className, ...props }, ref) => (
+function AccordionItem({
+  className,
+  ...props
+}: React.ComponentProps<typeof AccordionPrimitive.Item>) {
+  return (
     <AccordionPrimitive.Item
-    ref={ref}
-    className={cn("border-b", className)}
+      data-slot="accordion-item"
+      className={cn("border-b last:border-b-0", className)}
       {...props}
     />
-))
-AccordionItem.displayName = "AccordionItem"
+  )
+}
 
-const AccordionTrigger = React.forwardRef<
-  React.ElementRef<typeof AccordionPrimitive.Trigger>,
-  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Trigger>
->(({ className, children, ...props }, ref) => (
+function AccordionTrigger({
+  className,
+  children,
+  ...props
+}: React.ComponentProps<typeof AccordionPrimitive.Trigger>) {
+  return (
     <AccordionPrimitive.Header className="flex">
       <AccordionPrimitive.Trigger
-      ref={ref}
+        data-slot="accordion-trigger"
         className={cn(
-        "flex flex-1 items-center justify-between py-4 font-medium transition-all hover:underline [&[data-state=open]>svg]:rotate-180",
+          "focus-visible:border-ring focus-visible:ring-ring/50 flex flex-1 items-start justify-between gap-4 rounded-md py-4 text-left text-sm font-medium transition-all outline-none hover:underline focus-visible:ring-[3px] disabled:pointer-events-none disabled:opacity-50 [&[data-state=open]>svg]:rotate-180",
           className
         )}
         {...props}
       >
         {children}
-      <ChevronDown className="h-4 w-4 shrink-0 transition-transform duration-200" />
+        <ChevronDownIcon className="text-muted-foreground pointer-events-none size-4 shrink-0 translate-y-0.5 transition-transform duration-200" />
       </AccordionPrimitive.Trigger>
     </AccordionPrimitive.Header>
-))
-AccordionTrigger.displayName = "AccordionTrigger"
+  )
+}
 
-const AccordionContent = React.forwardRef<
-  React.ElementRef<typeof AccordionPrimitive.Content>,
-  React.ComponentPropsWithoutRef<typeof AccordionPrimitive.Content>
->(({ className, children, ...props }, ref) => (
+function AccordionContent({
+  className,
+  children,
+  ...props
+}: React.ComponentProps<typeof AccordionPrimitive.Content>) {
+  return (
     <AccordionPrimitive.Content
-    ref={ref}
-    className={cn(
-      "overflow-hidden text-sm transition-all data-[state=closed]:animate-accordion-up data-[state=open]:animate-accordion-down",
-      className
-    )}
+      data-slot="accordion-content"
+      className="data-[state=closed]:animate-accordion-up data-[state=open]:animate-accordion-down overflow-hidden text-sm"
       {...props}
     >
-    <div className="pb-4 pt-0">{children}</div>
+      <div className={cn("pt-0 pb-4", className)}>{children}</div>
     </AccordionPrimitive.Content>
-))
-AccordionContent.displayName = "AccordionContent"
+  )
+}
 
 export { Accordion, AccordionItem, AccordionTrigger, AccordionContent }

surfsense_web/components/ui/table.tsx

@@ -1,95 +1,116 @@
-import * as React from "react";
+"use client"
 
-import { cn } from "@/lib/utils";
+import * as React from "react"
 
-const Table = React.forwardRef<HTMLTableElement, React.HTMLAttributes<HTMLTableElement>>(
-  ({ className, ...props }, ref) => (
-    <div className="relative w-full overflow-auto">
-      <table ref={ref} className={cn("w-full caption-bottom text-sm", className)} {...props} />
+import { cn } from "@/lib/utils"
+
+function Table({ className, ...props }: React.ComponentProps<"table">) {
+  return (
+    <div
+      data-slot="table-container"
+      className="relative w-full overflow-x-auto"
+    >
+      <table
+        data-slot="table"
+        className={cn("w-full caption-bottom text-sm", className)}
+        {...props}
+      />
     </div>
-  ),
-);
-Table.displayName = "Table";
+  )
+}
 
-const TableHeader = React.forwardRef<
-  HTMLTableSectionElement,
-  React.HTMLAttributes<HTMLTableSectionElement>
->(({ className, ...props }, ref) => <thead ref={ref} className={cn(className)} {...props} />);
-TableHeader.displayName = "TableHeader";
+function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
+  return (
+    <thead
+      data-slot="table-header"
+      className={cn("[&_tr]:border-b", className)}
+      {...props}
+    />
+  )
+}
 
-const TableBody = React.forwardRef<
-  HTMLTableSectionElement,
-  React.HTMLAttributes<HTMLTableSectionElement>
->(({ className, ...props }, ref) => (
-  <tbody ref={ref} className={cn("[&_tr:last-child]:border-0", className)} {...props} />
-));
-TableBody.displayName = "TableBody";
+function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
+  return (
+    <tbody
+      data-slot="table-body"
+      className={cn("[&_tr:last-child]:border-0", className)}
+      {...props}
+    />
+  )
+}
 
-const TableFooter = React.forwardRef<
-  HTMLTableSectionElement,
-  React.HTMLAttributes<HTMLTableSectionElement>
->(({ className, ...props }, ref) => (
+function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
+  return (
     <tfoot
-    ref={ref}
+      data-slot="table-footer"
       className={cn(
-      "border-t border-border bg-muted/50 font-medium [&>tr]:last:border-b-0",
-      className,
+        "bg-muted/50 border-t font-medium [&>tr]:last:border-b-0",
+        className
       )}
       {...props}
     />
-));
-TableFooter.displayName = "TableFooter";
+  )
+}
 
-const TableRow = React.forwardRef<HTMLTableRowElement, React.HTMLAttributes<HTMLTableRowElement>>(
-  ({ className, ...props }, ref) => (
+function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
+  return (
     <tr
-      ref={ref}
+      data-slot="table-row"
       className={cn(
-        "border-b border-border transition-colors hover:bg-muted/50 data-[state=selected]:bg-muted",
-        className,
+        "hover:bg-muted/50 data-[state=selected]:bg-muted border-b transition-colors",
+        className
       )}
       {...props}
     />
-  ),
-);
-TableRow.displayName = "TableRow";
+  )
+}
 
-const TableHead = React.forwardRef<
-  HTMLTableCellElement,
-  React.ThHTMLAttributes<HTMLTableCellElement>
->(({ className, ...props }, ref) => (
+function TableHead({ className, ...props }: React.ComponentProps<"th">) {
+  return (
     <th
-    ref={ref}
+      data-slot="table-head"
       className={cn(
-      "h-12 px-3 text-left align-middle font-medium text-muted-foreground [&:has([role=checkbox])]:w-px [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-0.5",
-      className,
+        "text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+        className
       )}
       {...props}
     />
-));
-TableHead.displayName = "TableHead";
+  )
+}
 
-const TableCell = React.forwardRef<
-  HTMLTableCellElement,
-  React.TdHTMLAttributes<HTMLTableCellElement>
->(({ className, ...props }, ref) => (
+function TableCell({ className, ...props }: React.ComponentProps<"td">) {
+  return (
     <td
-    ref={ref}
+      data-slot="table-cell"
       className={cn(
-      "p-3 align-middle [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-0.5",
-      className,
+        "p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+        className
       )}
       {...props}
     />
-));
-TableCell.displayName = "TableCell";
+  )
+}
 
-const TableCaption = React.forwardRef<
-  HTMLTableCaptionElement,
-  React.HTMLAttributes<HTMLTableCaptionElement>
->(({ className, ...props }, ref) => (
-  <caption ref={ref} className={cn("mt-4 text-sm text-muted-foreground", className)} {...props} />
-));
-TableCaption.displayName = "TableCaption";
+function TableCaption({
+  className,
+  ...props
+}: React.ComponentProps<"caption">) {
+  return (
+    <caption
+      data-slot="table-caption"
+      className={cn("text-muted-foreground mt-4 text-sm", className)}
+      {...props}
+    />
+  )
+}
 
-export { Table, TableBody, TableCaption, TableCell, TableFooter, TableHead, TableHeader, TableRow };
+export {
+  Table,
+  TableHeader,
+  TableBody,
+  TableFooter,
+  TableHead,
+  TableRow,
+  TableCell,
+  TableCaption,
+}

surfsense_web/components/ui/textarea.tsx

@@ -0,0 +1,18 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
+  return (
+    <textarea
+      data-slot="textarea"
+      className={cn(
+        "border-input placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 flex field-sizing-content min-h-16 w-full rounded-md border bg-transparent px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
+        className
+      )}
+      {...props}
+    />
+  )
+}
+
+export { Textarea }

surfsense_web/hooks/use-documents.ts

@@ -1,98 +1,95 @@
-"use client"
-import { useState, useEffect } from 'react';
-import { toast } from 'sonner';
+"use client";
+import { useState, useEffect, useCallback } from "react";
+import { toast } from "sonner";
 
 export interface Document {
   id: number;
   title: string;
-  document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE";
+  document_type: DocumentType;
   document_metadata: any;
   content: string;
   created_at: string;
   search_space_id: number;
 }
 
-export function useDocuments(searchSpaceId: number) {
+export type DocumentType =
+  | "EXTENSION"
+  | "CRAWLED_URL"
+  | "SLACK_CONNECTOR"
+  | "NOTION_CONNECTOR"
+  | "FILE"
+  | "YOUTUBE_VIDEO"
+  | "GITHUB_CONNECTOR"
+  | "LINEAR_CONNECTOR"
+  | "DISCORD_CONNECTOR"
+  | "JIRA_CONNECTOR";
+
+export function useDocuments(searchSpaceId: number, lazy: boolean = false) {
   const [documents, setDocuments] = useState<Document[]>([]);
-  const [loading, setLoading] = useState(true);
+  const [loading, setLoading] = useState(!lazy); // Don't show loading initially for lazy mode
   const [error, setError] = useState<string | null>(null);
+  const [isLoaded, setIsLoaded] = useState(false); // Memoization flag
+
+  const fetchDocuments = useCallback(async () => {
+    if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode
+
+    try {
+      setLoading(true);
+      const response = await fetch(
+        `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`,
+        {
+          headers: {
+            Authorization: `Bearer ${localStorage.getItem(
+              "surfsense_bearer_token",
+            )}`,
+          },
+          method: "GET",
+        },
+      );
+
+      if (!response.ok) {
+        toast.error("Failed to fetch documents");
+        throw new Error("Failed to fetch documents");
+      }
+
+      const data = await response.json();
+      setDocuments(data);
+      setError(null);
+      setIsLoaded(true);
+    } catch (err: any) {
+      setError(err.message || "Failed to fetch documents");
+      console.error("Error fetching documents:", err);
+    } finally {
+      setLoading(false);
+    }
+  }, [searchSpaceId, isLoaded, lazy]);
 
   useEffect(() => {
-    const fetchDocuments = async () => {
-      try {
-        setLoading(true);
-        const response = await fetch(
-          `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`, 
-          {
-            headers: {
-              Authorization: `Bearer ${localStorage.getItem('surfsense_bearer_token')}`,
-            },
-            method: "GET",
-          }
-        );
-        
-        if (!response.ok) {
-          toast.error("Failed to fetch documents");
-          throw new Error("Failed to fetch documents");
-        }
-        
-        const data = await response.json();
-        setDocuments(data);
-        setError(null);
-      } catch (err: any) {
-        setError(err.message || 'Failed to fetch documents');
-        console.error('Error fetching documents:', err);
-      } finally {
-        setLoading(false);
-      }
-    };
-
-    if (searchSpaceId) {
+    if (!lazy && searchSpaceId) {
       fetchDocuments();
     }
-  }, [searchSpaceId]);
+  }, [searchSpaceId, lazy, fetchDocuments]);
 
   // Function to refresh the documents list
-  const refreshDocuments = async () => {
-    setLoading(true);
-    try {
-      const response = await fetch(
-        `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents?search_space_id=${searchSpaceId}`, 
-        {
-          headers: {
-            Authorization: `Bearer ${localStorage.getItem('surfsense_bearer_token')}`,
-          },
-          method: "GET",
-        }
-      );
-      
-      if (!response.ok) {
-        toast.error("Failed to fetch documents");
-        throw new Error("Failed to fetch documents");
-      }
-      
-      const data = await response.json();
-      setDocuments(data);
-      setError(null);
-    } catch (err: any) {
-      setError(err.message || 'Failed to fetch documents');
-      console.error('Error fetching documents:', err);
-    } finally {
-      setLoading(false);
-    }
-  };
+  const refreshDocuments = useCallback(async () => {
+    setIsLoaded(false); // Reset memoization flag to allow refetch
+    await fetchDocuments();
+  }, [fetchDocuments]);
 
   // Function to delete a document
-  const deleteDocument = async (documentId: number) => {
+  const deleteDocument = useCallback(
+    async (documentId: number) => {
       try {
         const response = await fetch(
           `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/documents/${documentId}`,
           {
             headers: {
-            Authorization: `Bearer ${localStorage.getItem('surfsense_bearer_token')}`,
+              Authorization: `Bearer ${localStorage.getItem(
+                "surfsense_bearer_token",
+              )}`,
             },
             method: "DELETE",
-        }
+          },
         );
 
         if (!response.ok) {
@@ -102,14 +99,24 @@ export function useDocuments(searchSpaceId: number) {
 
         toast.success("Document deleted successfully");
         // Update the local state after successful deletion
-      setDocuments(documents.filter(doc => doc.id !== documentId));
+        setDocuments(documents.filter((doc) => doc.id !== documentId));
         return true;
       } catch (err: any) {
-      toast.error(err.message || 'Failed to delete document');
-      console.error('Error deleting document:', err);
+        toast.error(err.message || "Failed to delete document");
+        console.error("Error deleting document:", err);
         return false;
       }
-  };
+    },
+    [documents],
+  );
 
-  return { documents, loading, error, refreshDocuments, deleteDocument };
+  return {
+    documents,
+    loading,
+    error,
+    isLoaded,
+    fetchDocuments, // Manual fetch function for lazy mode
+    refreshDocuments,
+    deleteDocument,
+  };
 }

surfsense_web/hooks/useChat.ts

@@ -0,0 +1,193 @@
+import { useState, useEffect, useCallback } from "react";
+import { Message } from "@ai-sdk/react";
+import { ResearchMode } from "@/components/chat";
+import { Document } from "@/hooks/use-documents";
+
+interface UseChatStateProps {
+    search_space_id: string;
+    chat_id?: string;
+}
+
+export function useChatState({ search_space_id, chat_id }: UseChatStateProps) {
+    const [token, setToken] = useState<string | null>(null);
+    const [isLoading, setIsLoading] = useState(false);
+    const [currentChatId, setCurrentChatId] = useState<string | null>(
+        chat_id || null
+    );
+
+    // Chat configuration state
+    const [searchMode, setSearchMode] = useState<"DOCUMENTS" | "CHUNKS">(
+        "DOCUMENTS"
+    );
+    const [researchMode, setResearchMode] = useState<ResearchMode>("QNA");
+    const [selectedConnectors, setSelectedConnectors] = useState<string[]>([]);
+    const [selectedDocuments, setSelectedDocuments] = useState<Document[]>([]);
+
+    useEffect(() => {
+        const bearerToken = localStorage.getItem("surfsense_bearer_token");
+        setToken(bearerToken);
+    }, []);
+
+    return {
+        token,
+        setToken,
+        isLoading,
+        setIsLoading,
+        currentChatId,
+        setCurrentChatId,
+        searchMode,
+        setSearchMode,
+        researchMode,
+        setResearchMode,
+        selectedConnectors,
+        setSelectedConnectors,
+        selectedDocuments,
+        setSelectedDocuments,
+    };
+}
+
+interface UseChatAPIProps {
+    token: string | null;
+    search_space_id: string;
+}
+
+export function useChatAPI({ token, search_space_id }: UseChatAPIProps) {
+    const fetchChatDetails = useCallback(
+        async (chatId: string) => {
+            if (!token) return null;
+
+            try {
+                const response = await fetch(
+                    `${
+                        process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL
+                    }/api/v1/chats/${Number(chatId)}`,
+                    {
+                        method: "GET",
+                        headers: {
+                            "Content-Type": "application/json",
+                            Authorization: `Bearer ${token}`,
+                        },
+                    }
+                );
+
+                if (!response.ok) {
+                    throw new Error(
+                        `Failed to fetch chat details: ${response.statusText}`
+                    );
+                }
+
+                return await response.json();
+            } catch (err) {
+                console.error("Error fetching chat details:", err);
+                return null;
+            }
+        },
+        [token]
+    );
+
+    const createChat = useCallback(
+        async (
+            initialMessage: string,
+            researchMode: ResearchMode,
+            selectedConnectors: string[]
+        ): Promise<string | null> => {
+            if (!token) {
+                console.error("Authentication token not found");
+                return null;
+            }
+
+            try {
+                const response = await fetch(
+                    `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/`,
+                    {
+                        method: "POST",
+                        headers: {
+                            "Content-Type": "application/json",
+                            Authorization: `Bearer ${token}`,
+                        },
+                        body: JSON.stringify({
+                            type: researchMode,
+                            title: "Untitled Chat",
+                            initial_connectors: selectedConnectors,
+                            messages: [
+                                {
+                                    role: "user",
+                                    content: initialMessage,
+                                },
+                            ],
+                            search_space_id: Number(search_space_id),
+                        }),
+                    }
+                );
+
+                if (!response.ok) {
+                    throw new Error(
+                        `Failed to create chat: ${response.statusText}`
+                    );
+                }
+
+                const data = await response.json();
+                return data.id;
+            } catch (err) {
+                console.error("Error creating chat:", err);
+                return null;
+            }
+        },
+        [token, search_space_id]
+    );
+
+    const updateChat = useCallback(
+        async (
+            chatId: string,
+            messages: Message[],
+            researchMode: ResearchMode,
+            selectedConnectors: string[]
+        ) => {
+            if (!token) return;
+
+            try {
+                const userMessages = messages.filter(
+                    (msg) => msg.role === "user"
+                );
+                if (userMessages.length === 0) return;
+
+                const title = userMessages[0].content;
+
+                const response = await fetch(
+                    `${
+                        process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL
+                    }/api/v1/chats/${Number(chatId)}`,
+                    {
+                        method: "PUT",
+                        headers: {
+                            "Content-Type": "application/json",
+                            Authorization: `Bearer ${token}`,
+                        },
+                        body: JSON.stringify({
+                            type: researchMode,
+                            title: title,
+                            initial_connectors: selectedConnectors,
+                            messages: messages,
+                            search_space_id: Number(search_space_id),
+                        }),
+                    }
+                );
+
+                if (!response.ok) {
+                    throw new Error(
+                        `Failed to update chat: ${response.statusText}`
+                    );
+                }
+            } catch (err) {
+                console.error("Error updating chat:", err);
+            }
+        },
+        [token, search_space_id]
+    );
+
+    return {
+        fetchChatDetails,
+        createChat,
+        updateChat,
+    };
+}

surfsense_web/hooks/useSearchSourceConnectors.ts

@@ -1,4 +1,4 @@
-import { useState, useEffect, useCallback } from 'react';
+import { useState, useEffect, useCallback } from "react";
 
 export interface SearchSourceConnector {
     id: number;
@@ -21,11 +21,14 @@ export interface ConnectorSourceItem {
 /**
  * Hook to fetch search source connectors from the API
  */
-export const useSearchSourceConnectors = () => {
+export const useSearchSourceConnectors = (lazy: boolean = false) => {
     const [connectors, setConnectors] = useState<SearchSourceConnector[]>([]);
-  const [isLoading, setIsLoading] = useState(true);
+    const [isLoading, setIsLoading] = useState(!lazy); // Don't show loading initially for lazy mode
+    const [isLoaded, setIsLoaded] = useState(false); // Memoization flag
     const [error, setError] = useState<Error | null>(null);
-  const [connectorSourceItems, setConnectorSourceItems] = useState<ConnectorSourceItem[]>([
+    const [connectorSourceItems, setConnectorSourceItems] = useState<
+        ConnectorSourceItem[]
+    >([
         {
             id: 1,
             name: "Crawled URL",
@@ -49,52 +52,72 @@ export const useSearchSourceConnectors = () => {
             name: "Youtube Video",
             type: "YOUTUBE_VIDEO",
             sources: [],
-    }
+        },
     ]);
 
-  useEffect(() => {
-    const fetchConnectors = async () => {
+    const fetchConnectors = useCallback(async () => {
+        if (isLoaded && lazy) return; // Avoid redundant calls in lazy mode
+
         try {
             setIsLoading(true);
-        const token = localStorage.getItem('surfsense_bearer_token');
+            setError(null);
+            const token = localStorage.getItem("surfsense_bearer_token");
 
             if (!token) {
-          throw new Error('No authentication token found');
+                throw new Error("No authentication token found");
             }
 
             const response = await fetch(
                 `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`,
                 {
-            method: 'GET',
+                    method: "GET",
                     headers: {
-              'Content-Type': 'application/json',
-              'Authorization': `Bearer ${token}`
-            }
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${token}`,
+                    },
                 }
             );
 
             if (!response.ok) {
-          throw new Error(`Failed to fetch connectors: ${response.statusText}`);
+                throw new Error(
+                    `Failed to fetch connectors: ${response.statusText}`
+                );
             }
 
             const data = await response.json();
             setConnectors(data);
+            setIsLoaded(true);
 
             // Update connector source items when connectors change
             updateConnectorSourceItems(data);
         } catch (err) {
-        setError(err instanceof Error ? err : new Error('An unknown error occurred'));
-        console.error('Error fetching search source connectors:', err);
+            setError(
+                err instanceof Error
+                    ? err
+                    : new Error("An unknown error occurred")
+            );
+            console.error("Error fetching search source connectors:", err);
         } finally {
             setIsLoading(false);
         }
-    };
+    }, [isLoaded, lazy]);
 
+    useEffect(() => {
+        if (!lazy) {
             fetchConnectors();
-  }, []);
+        }
+    }, [lazy, fetchConnectors]);
+
+    // Function to refresh the connectors list
+    const refreshConnectors = useCallback(async () => {
+        setIsLoaded(false); // Reset memoization flag to allow refetch
+        await fetchConnectors();
+    }, [fetchConnectors]);
 
     // Update connector source items when connectors change
-  const updateConnectorSourceItems = (currentConnectors: SearchSourceConnector[]) => {
+    const updateConnectorSourceItems = (
+        currentConnectors: SearchSourceConnector[]
+    ) => {
         // Start with the default hardcoded connectors
         const defaultConnectors: ConnectorSourceItem[] = [
             {
@@ -120,16 +143,18 @@ export const useSearchSourceConnectors = () => {
                 name: "Youtube Video",
                 type: "YOUTUBE_VIDEO",
                 sources: [],
-      }
+            },
         ];
 
         // Add the API connectors
-    const apiConnectors: ConnectorSourceItem[] = currentConnectors.map((connector, index) => ({
+        const apiConnectors: ConnectorSourceItem[] = currentConnectors.map(
+            (connector, index) => ({
                 id: 1000 + index, // Use a high ID to avoid conflicts with hardcoded IDs
                 name: connector.name,
                 type: connector.connector_type,
                 sources: [],
-    }));
+            })
+        );
 
         setConnectorSourceItems([...defaultConnectors, ...apiConnectors]);
     };
@@ -137,28 +162,35 @@ export const useSearchSourceConnectors = () => {
     /**
      * Create a new search source connector
      */
-  const createConnector = async (connectorData: Omit<SearchSourceConnector, 'id' | 'user_id' | 'created_at'>) => {
+    const createConnector = async (
+        connectorData: Omit<
+            SearchSourceConnector,
+            "id" | "user_id" | "created_at"
+        >
+    ) => {
         try {
-      const token = localStorage.getItem('surfsense_bearer_token');
+            const token = localStorage.getItem("surfsense_bearer_token");
 
             if (!token) {
-        throw new Error('No authentication token found');
+                throw new Error("No authentication token found");
             }
 
             const response = await fetch(
                 `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/`,
                 {
-          method: 'POST',
+                    method: "POST",
                     headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${token}`
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${token}`,
                     },
-          body: JSON.stringify(connectorData)
+                    body: JSON.stringify(connectorData),
                 }
             );
 
             if (!response.ok) {
-        throw new Error(`Failed to create connector: ${response.statusText}`);
+                throw new Error(
+                    `Failed to create connector: ${response.statusText}`
+                );
             }
 
             const newConnector = await response.json();
@@ -167,7 +199,7 @@ export const useSearchSourceConnectors = () => {
             updateConnectorSourceItems(updatedConnectors);
             return newConnector;
         } catch (err) {
-      console.error('Error creating search source connector:', err);
+            console.error("Error creating search source connector:", err);
             throw err;
         }
     };
@@ -177,40 +209,44 @@ export const useSearchSourceConnectors = () => {
      */
     const updateConnector = async (
         connectorId: number,
-    connectorData: Partial<Omit<SearchSourceConnector, 'id' | 'user_id' | 'created_at'>>
+        connectorData: Partial<
+            Omit<SearchSourceConnector, "id" | "user_id" | "created_at">
+        >
     ) => {
         try {
-      const token = localStorage.getItem('surfsense_bearer_token');
+            const token = localStorage.getItem("surfsense_bearer_token");
 
             if (!token) {
-        throw new Error('No authentication token found');
+                throw new Error("No authentication token found");
             }
 
             const response = await fetch(
                 `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`,
                 {
-          method: 'PUT',
+                    method: "PUT",
                     headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${token}`
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${token}`,
                     },
-          body: JSON.stringify(connectorData)
+                    body: JSON.stringify(connectorData),
                 }
             );
 
             if (!response.ok) {
-        throw new Error(`Failed to update connector: ${response.statusText}`);
+                throw new Error(
+                    `Failed to update connector: ${response.statusText}`
+                );
             }
 
             const updatedConnector = await response.json();
-      const updatedConnectors = connectors.map(connector => 
+            const updatedConnectors = connectors.map((connector) =>
                 connector.id === connectorId ? updatedConnector : connector
             );
             setConnectors(updatedConnectors);
             updateConnectorSourceItems(updatedConnectors);
             return updatedConnector;
         } catch (err) {
-      console.error('Error updating search source connector:', err);
+            console.error("Error updating search source connector:", err);
             throw err;
         }
     };
@@ -220,32 +256,36 @@ export const useSearchSourceConnectors = () => {
      */
     const deleteConnector = async (connectorId: number) => {
         try {
-      const token = localStorage.getItem('surfsense_bearer_token');
+            const token = localStorage.getItem("surfsense_bearer_token");
 
             if (!token) {
-        throw new Error('No authentication token found');
+                throw new Error("No authentication token found");
             }
 
             const response = await fetch(
                 `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}`,
                 {
-          method: 'DELETE',
+                    method: "DELETE",
                     headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${token}`
-          }
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${token}`,
+                    },
                 }
             );
 
             if (!response.ok) {
-        throw new Error(`Failed to delete connector: ${response.statusText}`);
+                throw new Error(
+                    `Failed to delete connector: ${response.statusText}`
+                );
             }
 
-      const updatedConnectors = connectors.filter(connector => connector.id !== connectorId);
+            const updatedConnectors = connectors.filter(
+                (connector) => connector.id !== connectorId
+            );
             setConnectors(updatedConnectors);
             updateConnectorSourceItems(updatedConnectors);
         } catch (err) {
-      console.error('Error deleting search source connector:', err);
+            console.error("Error deleting search source connector:", err);
             throw err;
         }
     };
@@ -260,49 +300,58 @@ export const useSearchSourceConnectors = () => {
         endDate?: string
     ) => {
         try {
-      const token = localStorage.getItem('surfsense_bearer_token');
+            const token = localStorage.getItem("surfsense_bearer_token");
 
             if (!token) {
-        throw new Error('No authentication token found');
+                throw new Error("No authentication token found");
             }
 
             // Build query parameters
-      const params = new URLSearchParams({ search_space_id: searchSpaceId.toString() });
+            const params = new URLSearchParams({
+                search_space_id: searchSpaceId.toString(),
+            });
             if (startDate) {
-        params.append('start_date', startDate);
+                params.append("start_date", startDate);
             }
             if (endDate) {
-        params.append('end_date', endDate);
+                params.append("end_date", endDate);
             }
 
             const response = await fetch(
-        `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/search-source-connectors/${connectorId}/index?${params.toString()}`,
+                `${
+                    process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL
+                }/api/v1/search-source-connectors/${connectorId}/index?${params.toString()}`,
                 {
-          method: 'POST',
+                    method: "POST",
                     headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${token}`
-          }
+                        "Content-Type": "application/json",
+                        Authorization: `Bearer ${token}`,
+                    },
                 }
             );
 
             if (!response.ok) {
-        throw new Error(`Failed to index connector content: ${response.statusText}`);
+                throw new Error(
+                    `Failed to index connector content: ${response.statusText}`
+                );
             }
 
             const result = await response.json();
 
             // Update the connector's last_indexed_at timestamp
-      const updatedConnectors = connectors.map(connector => 
+            const updatedConnectors = connectors.map((connector) =>
                 connector.id === connectorId
-          ? { ...connector, last_indexed_at: new Date().toISOString() } 
+                    ? {
+                          ...connector,
+                          last_indexed_at: new Date().toISOString(),
+                      }
                     : connector
             );
             setConnectors(updatedConnectors);
 
             return result;
         } catch (err) {
-      console.error('Error indexing connector content:', err);
+            console.error("Error indexing connector content:", err);
             throw err;
         }
     };
@@ -317,12 +366,15 @@ export const useSearchSourceConnectors = () => {
     return {
         connectors,
         isLoading,
+        isLoaded,
         error,
+        fetchConnectors,
         createConnector,
         updateConnector,
         deleteConnector,
         indexConnector,
         getConnectorSourceItems,
-    connectorSourceItems
+        connectorSourceItems,
+        refreshConnectors,
     };
 };

surfsense_web/package.json

@@ -17,6 +17,7 @@
   "dependencies": {
     "@ai-sdk/react": "^1.1.21",
     "@hookform/resolvers": "^4.1.3",
+    "@llamaindex/chat-ui": "^0.5.17",
     "@radix-ui/react-accordion": "^1.2.3",
     "@radix-ui/react-alert-dialog": "^1.1.6",
     "@radix-ui/react-avatar": "^1.1.3",
@@ -26,11 +27,14 @@
     "@radix-ui/react-dropdown-menu": "^2.1.6",
     "@radix-ui/react-label": "^2.1.2",
     "@radix-ui/react-popover": "^1.1.6",
+    "@radix-ui/react-scroll-area": "^1.2.9",
     "@radix-ui/react-select": "^2.1.6",
     "@radix-ui/react-separator": "^1.1.2",
     "@radix-ui/react-slider": "^1.3.4",
     "@radix-ui/react-slot": "^1.1.2",
     "@radix-ui/react-tabs": "^1.1.3",
+    "@radix-ui/react-toggle": "^1.1.9",
+    "@radix-ui/react-toggle-group": "^1.1.10",
     "@radix-ui/react-tooltip": "^1.1.8",
     "@tabler/icons-react": "^3.30.0",
     "@tanstack/react-table": "^8.21.2",