diff --git a/node_modules/.cache/prettier/.prettier-caches/37bd945444dc76999f7aface662ff267baf1dbca.json b/node_modules/.cache/prettier/.prettier-caches/37bd945444dc76999f7aface662ff267baf1dbca.json new file mode 100644 index 0000000..e94f98f --- /dev/null +++ b/node_modules/.cache/prettier/.prettier-caches/37bd945444dc76999f7aface662ff267baf1dbca.json @@ -0,0 +1 @@ +{"c301dd3ad9b4036af1d031ecc966d2f02ae1eda4":{"files":{"surfsense_web/hooks/use-documents.ts":["J01fJFm4gXaHAA83Vu5dtOmk/sw=",true],"surfsense_web/components/chat/DocumentsDataTable.tsx":["wgAyJblucK9D3MKKwPe6W9kZphk=",true]},"modified":1753499058926}} \ No newline at end of file diff --git a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py index 4bcc042..6661cdf 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/nodes.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/nodes.py @@ -86,7 +86,7 @@ async def answer_question(state: State, config: RunnableConfig) -> dict[str, Any This node takes the relevant documents provided in the configuration and uses an LLM to generate a comprehensive answer to the user's question with - proper citations. The citations follow IEEE format using source IDs from the + proper citations. The citations follow [citation:source_id] format using source IDs from the documents. If no documents are provided, it will use chat history to generate an answer. diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index 3f4d975..eed8e55 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -25,21 +25,21 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel 1. Carefully analyze all provided documents in the sections. 2. Extract relevant information that directly addresses the user's question. 3. Provide a comprehensive, detailed answer using information from the user's personal knowledge sources. -4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata. +4. For EVERY piece of information you include from the documents, add a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the source_id from the document's metadata. 5. Make sure ALL factual statements from the documents have proper citations. -6. If multiple documents support the same point, include all relevant citations [X], [Y]. +6. If multiple documents support the same point, include all relevant citations [citation:source_id1], [citation:source_id2]. 7. Structure your answer logically and conversationally, as if having a detailed discussion with the user. 8. Use your own words to synthesize and connect ideas, but cite ALL information from the documents. 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations. 10. If the user's question cannot be fully answered with the provided documents, clearly state what information is missing. 11. Provide actionable insights and practical information when relevant to the user's question. 12. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers. -13. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value. -14. CRITICAL: Never renumber or reorder citations - always use the original source_id values. +13. CRITICAL: Every citation MUST be in the format [citation:knowledge_source_id] where knowledge_source_id is the exact source_id value. +14. CRITICAL: Never modify or change the source_id - always use the original values exactly as provided in the metadata. 15. CRITICAL: Do not return citations as clickable links. -16. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only. -17. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting. -18. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata. +16. CRITICAL: Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only. +17. CRITICAL: Citations must ONLY appear as [citation:source_id] or [citation:source_id1], [citation:source_id2] format - never with parentheses, hyperlinks, or other formatting. +18. CRITICAL: Never make up source IDs. Only use source_id values that are explicitly provided in the document metadata. 19. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up. 20. CRITICAL: Remember that all knowledge sources contain personal information - provide answers that reflect this personal context. 21. CRITICAL: Be conversational and engaging while maintaining accuracy and proper citations. @@ -49,13 +49,13 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel - Write in a clear, conversational tone suitable for detailed Q&A discussions - Provide comprehensive answers that thoroughly address the user's question - Use appropriate paragraphs and structure for readability -- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata +- Every fact from the documents must have a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the EXACT source_id from the document's metadata - Citations should appear at the end of the sentence containing the information they support -- Multiple citations should be separated by commas: [X], [Y], [Z] -- No need to return references section. Just citation numbers in answer. -- NEVER create your own citation numbering system - use the exact source_id values from the documents -- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only -- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess +- Multiple citations should be separated by commas: [citation:source_id1], [citation:source_id2], [citation:source_id3] +- No need to return references section. Just citations in answer. +- NEVER create your own citation format - use the exact source_id values from the documents in the [citation:source_id] format +- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only +- NEVER make up source IDs if you are unsure about the source_id. It is better to omit the citation than to guess - ALWAYS provide personalized answers that reflect the user's own knowledge and context - Be thorough and detailed in your explanations while remaining focused on the user's specific question - If asking follow-up questions would be helpful, suggest them at the end of your response @@ -88,26 +88,31 @@ User Question: "How does Python asyncio work and when should I use it?" -Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [5]. It's particularly useful for I/O-bound and high-level structured network code [5]. +Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [citation:5]. It's particularly useful for I/O-bound and high-level structured network code [citation:5]. -The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources. +The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [citation:12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources. -However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [12]. For computationally intensive work, you'd want to use multiprocessing instead. +However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [citation:12]. For computationally intensive work, you'd want to use multiprocessing instead. Would you like me to explain more about specific asyncio patterns or help you determine if asyncio is right for a particular project you're working on? DO NOT use any of these incorrect citation formats: -- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense)) -- Using parentheses around brackets: ([1]) -- Using hyperlinked text: [link to source 1](https://example.com) +- Using parentheses and markdown links: ([citation:5](https://github.com/MODSetter/SurfSense)) +- Using parentheses around brackets: ([citation:5]) +- Using hyperlinked text: [link to source 5](https://example.com) - Using footnote style: ... library¹ -- Making up citation numbers when source_id is unknown +- Making up source IDs when source_id is unknown +- Using old IEEE format: [1], [2], [3] +- Using source types instead of IDs: [citation:GITHUB_CONNECTOR] instead of [citation:5] -ONLY use plain square brackets [1] or multiple citations [1], [2], [3] + +ONLY use the format [citation:source_id] or multiple citations [citation:source_id1], [citation:source_id2], [citation:source_id3] + + When you see a user query, focus exclusively on providing a detailed, comprehensive answer using information from the provided documents, which contain the user's personal knowledge and data. diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 2dee978..734be65 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -93,7 +93,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> dict[str, A This node takes the relevant documents provided in the configuration and uses an LLM to generate a comprehensive answer to the sub-section title with - proper citations. The citations follow IEEE format using source IDs from the + proper citations. The citations follow [citation:source_id] format using source IDs from the documents. If no documents are provided, it will use chat history to generate content. diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index 02036fc..83125e6 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -23,20 +23,20 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio 1. Carefully analyze all provided documents in the section's. 2. Extract relevant information that addresses the user's query. 3. Synthesize a comprehensive, personalized answer using information from the user's personal knowledge sources. -4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata. +4. For EVERY piece of information you include from the documents, add a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the source_id from the document's metadata. 5. Make sure ALL factual statements from the documents have proper citations. -6. If multiple documents support the same point, include all relevant citations [X], [Y]. +6. If multiple documents support the same point, include all relevant citations [citation:source_id1], [citation:source_id2]. 7. Present information in a logical, coherent flow that reflects the user's personal context. 8. Use your own words to connect ideas, but cite ALL information from the documents. 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations. 10. Do not make up or include information not found in the provided documents. 11. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers. -12. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value. -13. CRITICAL: Never renumber or reorder citations - always use the original source_id values. +12. CRITICAL: Every citation MUST be in the format [citation:knowledge_source_id] where knowledge_source_id is the exact source_id value. +13. CRITICAL: Never modify or change the source_id - always use the original values exactly as provided in the metadata. 14. CRITICAL: Do not return citations as clickable links. -15. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only. -16. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting. -17. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata. +15. CRITICAL: Never format citations as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only. +16. CRITICAL: Citations must ONLY appear as [citation:source_id] or [citation:source_id1], [citation:source_id2] format - never with parentheses, hyperlinks, or other formatting. +17. CRITICAL: Never make up source IDs. Only use source_id values that are explicitly provided in the document metadata. 18. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up. 19. CRITICAL: Focus only on answering the user's query. Any guiding questions provided are for your thinking process only and should not be mentioned in your response. 20. CRITICAL: Ensure your response aligns with the provided sub-section title and section position. @@ -47,13 +47,13 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio - Write in clear, professional language suitable for academic or technical audiences - Tailor your response to the user's personal context based on their knowledge sources - Organize your response with appropriate paragraphs, headings, and structure -- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata +- Every fact from the documents must have a citation in the format [citation:knowledge_source_id] where knowledge_source_id is the EXACT source_id from the document's metadata - Citations should appear at the end of the sentence containing the information they support -- Multiple citations should be separated by commas: [X], [Y], [Z] -- No need to return references section. Just citation numbers in answer. -- NEVER create your own citation numbering system - use the exact source_id values from the documents. -- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only. -- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess. +- Multiple citations should be separated by commas: [citation:source_id1], [citation:source_id2], [citation:source_id3] +- No need to return references section. Just citations in answer. +- NEVER create your own citation format - use the exact source_id values from the documents in the [citation:source_id] format. +- NEVER format citations as clickable links or as markdown links like "([citation:5](https://example.com))". Always use plain square brackets only. +- NEVER make up source IDs if you are unsure about the source_id. It is better to omit the citation than to guess. - NEVER include or mention the guiding questions in your response. They are only to help guide your thinking. - ALWAYS focus on answering the user's query directly from the information in the documents. - ALWAYS provide personalized answers that reflect the user's own knowledge and context. @@ -94,21 +94,23 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio - Based on your saved browser content and videos, the Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. From your browsing history, you've looked into its designation as a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. According to a YouTube video you've watched, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13]. + Based on your saved browser content and videos, the Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [citation:1]. From your browsing history, you've looked into its designation as a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [citation:21]. The reef is home to over 1,500 species of fish and 400 types of coral [citation:21]. According to a YouTube video you've watched, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [citation:13]. The reef system comprises over 2,900 individual reefs and 900 islands [citation:1], making it an ecological treasure that requires protection from multiple threats [citation:1], [citation:13]. DO NOT use any of these incorrect citation formats: -- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense)) -- Using parentheses around brackets: ([1]) +- Using parentheses and markdown links: ([citation:1](https://github.com/MODSetter/SurfSense)) +- Using parentheses around brackets: ([citation:1]) - Using hyperlinked text: [link to source 1](https://example.com) - Using footnote style: ... reef system¹ -- Making up citation numbers when source_id is unknown +- Making up source IDs when source_id is unknown +- Using old IEEE format: [1], [2], [3] +- Using source types instead of IDs: [citation:EXTENSION] instead of [citation:1] -ONLY use plain square brackets [1] or multiple citations [1], [2], [3] -Note that the citation numbers match exactly with the source_id values (1, 13, and 21) and are not renumbered sequentially. Citations follow IEEE style with square brackets and appear at the end of sentences. +ONLY use the format [citation:source_id] or multiple citations [citation:source_id1], [citation:source_id2], [citation:source_id3] +Note that the citations use the exact source_id values (1, 13, and 21) from the document metadata. Citations appear at the end of sentences and maintain the new citation format. When you see a user query like: diff --git a/surfsense_backend/app/services/streaming_service.py b/surfsense_backend/app/services/streaming_service.py index dde792c..40bd430 100644 --- a/surfsense_backend/app/services/streaming_service.py +++ b/surfsense_backend/app/services/streaming_service.py @@ -43,7 +43,7 @@ class StreamingService: self.message_annotations[0]["content"].append(message) # Return only the delta annotation - annotation = {"type": "TERMINAL_INFO", "content": [message]} + annotation = {"type": "TERMINAL_INFO", "data": message} return f"8:[{json.dumps(annotation)}]\n" def format_sources_delta(self, sources: list[dict[str, Any]]) -> str: @@ -60,7 +60,23 @@ class StreamingService: self.message_annotations[1]["content"] = sources # Return only the delta annotation - annotation = {"type": "SOURCES", "content": sources} + nodes = [] + + for group in sources: + for source in group.get("sources", []): + node = { + "id": str(source.get("id", "")), + "text": source.get("description", ""), + "url": source.get("url", ""), + "metadata": { + "title": source.get("title", ""), + "source_type": group.get("type", ""), + "group_name": group.get("name", ""), + }, + } + nodes.append(node) + + annotation = {"type": "sources", "data": {"nodes": nodes}} return f"8:[{json.dumps(annotation)}]\n" def format_answer_delta(self, answer_chunk: str) -> str: @@ -116,7 +132,14 @@ class StreamingService: self.message_annotations[3]["content"] = further_questions # Return only the delta annotation - annotation = {"type": "FURTHER_QUESTIONS", "content": further_questions} + annotation = { + "type": "FURTHER_QUESTIONS", + "data": [ + question.get("question", "") + for question in further_questions + if question.get("question", "") != "" + ], + } return f"8:[{json.dumps(annotation)}]\n" def format_text_chunk(self, text: str) -> str: diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx new file mode 100644 index 0000000..bf961a9 --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[[...chat_id]]/page.tsx @@ -0,0 +1,241 @@ +"use client"; + +import { type CreateMessage, type Message, useChat } from "@ai-sdk/react"; +import { useParams, useRouter } from "next/navigation"; +import { useEffect, useMemo } from "react"; +import type { ResearchMode } from "@/components/chat"; +import ChatInterface from "@/components/chat/ChatInterface"; +import type { Document } from "@/hooks/use-documents"; +import { useChatAPI, useChatState } from "@/hooks/useChat"; + +export default function ResearcherPage() { + const { search_space_id, chat_id } = useParams(); + const router = useRouter(); + + const chatIdParam = Array.isArray(chat_id) ? chat_id[0] : chat_id; + const isNewChat = !chatIdParam; + + const { + token, + isLoading, + setIsLoading, + searchMode, + setSearchMode, + researchMode, + setResearchMode, + selectedConnectors, + setSelectedConnectors, + selectedDocuments, + setSelectedDocuments, + } = useChatState({ + search_space_id: search_space_id as string, + chat_id: chatIdParam, + }); + + const { fetchChatDetails, updateChat, createChat } = useChatAPI({ + token, + search_space_id: search_space_id as string, + }); + + // Memoize document IDs to prevent infinite re-renders + const documentIds = useMemo(() => { + return selectedDocuments.map((doc) => doc.id); + }, [selectedDocuments]); + + // Memoize connector types to prevent infinite re-renders + const connectorTypes = useMemo(() => { + return selectedConnectors; + }, [selectedConnectors]); + + // Unified localStorage management for chat state + interface ChatState { + selectedDocuments: Document[]; + selectedConnectors: string[]; + searchMode: "DOCUMENTS" | "CHUNKS"; + researchMode: ResearchMode; + } + + const getChatStateStorageKey = (searchSpaceId: string, chatId: string) => + `surfsense_chat_state_${searchSpaceId}_${chatId}`; + + const storeChatState = ( + searchSpaceId: string, + chatId: string, + state: ChatState, + ) => { + const key = getChatStateStorageKey(searchSpaceId, chatId); + localStorage.setItem(key, JSON.stringify(state)); + }; + + const restoreChatState = ( + searchSpaceId: string, + chatId: string, + ): ChatState | null => { + const key = getChatStateStorageKey(searchSpaceId, chatId); + const stored = localStorage.getItem(key); + if (stored) { + localStorage.removeItem(key); // Clean up after restoration + try { + return JSON.parse(stored); + } catch (error) { + console.error("Error parsing stored chat state:", error); + return null; + } + } + return null; + }; + + const handler = useChat({ + api: `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chat`, + streamProtocol: "data", + initialMessages: [], + headers: { + ...(token && { Authorization: `Bearer ${token}` }), + }, + body: { + data: { + search_space_id: search_space_id, + selected_connectors: connectorTypes, + research_mode: researchMode, + search_mode: searchMode, + document_ids_to_add_in_context: documentIds, + }, + }, + onError: (error) => { + console.error("Chat error:", error); + }, + }); + + const customHandlerAppend = async ( + message: Message | CreateMessage, + chatRequestOptions?: { data?: any }, + ) => { + const newChatId = await createChat( + message.content, + researchMode, + selectedConnectors, + ); + if (newChatId) { + // Store chat state before navigation + storeChatState(search_space_id as string, newChatId, { + selectedDocuments, + selectedConnectors, + searchMode, + researchMode, + }); + router.replace(`/dashboard/${search_space_id}/researcher/${newChatId}`); + } + return newChatId; + }; + + useEffect(() => { + if (token && !isNewChat && chatIdParam) { + setIsLoading(true); + loadChatData(chatIdParam); + } + }, [token, isNewChat, chatIdParam]); + + // Restore chat state from localStorage on page load + useEffect(() => { + if (chatIdParam && search_space_id) { + const restoredState = restoreChatState( + search_space_id as string, + chatIdParam, + ); + if (restoredState) { + setSelectedDocuments(restoredState.selectedDocuments); + setSelectedConnectors(restoredState.selectedConnectors); + setSearchMode(restoredState.searchMode); + setResearchMode(restoredState.researchMode); + } + } + }, [ + chatIdParam, + search_space_id, + setSelectedDocuments, + setSelectedConnectors, + setSearchMode, + setResearchMode, + ]); + + const loadChatData = async (chatId: string) => { + try { + const chatData = await fetchChatDetails(chatId); + if (!chatData) return; + + // Update configuration from chat data + if (chatData.type) { + setResearchMode(chatData.type as ResearchMode); + } + + if ( + chatData.initial_connectors && + Array.isArray(chatData.initial_connectors) + ) { + setSelectedConnectors(chatData.initial_connectors); + } + + // Load existing messages + if (chatData.messages && Array.isArray(chatData.messages)) { + if ( + chatData.messages.length === 1 && + chatData.messages[0].role === "user" + ) { + // Single user message - append to trigger LLM response + handler.append({ + role: "user", + content: chatData.messages[0].content, + }); + } else if (chatData.messages.length > 1) { + // Multiple messages - set them all + handler.setMessages(chatData.messages); + } + } + } finally { + setIsLoading(false); + } + }; + + // Auto-update chat when messages change (only for existing chats) + useEffect(() => { + if ( + !isNewChat && + chatIdParam && + handler.status === "ready" && + handler.messages.length > 0 && + handler.messages[handler.messages.length - 1]?.role === "assistant" + ) { + updateChat( + chatIdParam, + handler.messages, + researchMode, + selectedConnectors, + ); + } + }, [handler.messages, handler.status, chatIdParam, isNewChat]); + + if (isLoading) { + return ( +
+
Loading...
+
+ ); + } + + return ( + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx deleted file mode 100644 index e92db28..0000000 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx +++ /dev/null @@ -1,2106 +0,0 @@ -"use client"; -import React, { - useRef, - useEffect, - useState, - useMemo, - useCallback, -} from "react"; -import { useChat } from "@ai-sdk/react"; -import { useParams } from "next/navigation"; -import { - Loader2, - X, - Search, - ExternalLink, - ChevronLeft, - ChevronRight, - Check, - ArrowDown, - CircleUser, - Database, - SendHorizontal, - FileText, - Grid3x3, - FolderOpen, - Upload, - ChevronDown, - Filter, - Brain, - Zap, -} from "lucide-react"; -import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogHeader, - DialogTitle, - DialogTrigger, - DialogFooter, -} from "@/components/ui/dialog"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, -} from "@/components/ui/dropdown-menu"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "@/components/ui/select"; -import { Badge } from "@/components/ui/badge"; -import { Skeleton } from "@/components/ui/skeleton"; -import { - ConnectorButton as ConnectorButtonComponent, - getConnectorIcon, - getFilteredSources as getFilteredSourcesUtil, - getPaginatedDialogSources as getPaginatedDialogSourcesUtil, - useScrollToBottom, - updateScrollIndicators as updateScrollIndicatorsUtil, - useScrollIndicators, - scrollTabsLeft as scrollTabsLeftUtil, - scrollTabsRight as scrollTabsRightUtil, - Source, - ResearchMode, - ResearchModeControl, -} from "@/components/chat"; -import { MarkdownViewer } from "@/components/markdown-viewer"; -import { Logo } from "@/components/Logo"; -import { useSearchSourceConnectors } from "@/hooks"; -import { useDocuments } from "@/hooks/use-documents"; -import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; - -interface SourceItem { - id: number; - title: string; - description: string; - url: string; - connectorType?: string; -} - -interface ConnectorSource { - id: number; - name: string; - type: string; - sources: SourceItem[]; -} - -type DocumentType = - | "EXTENSION" - | "CRAWLED_URL" - | "SLACK_CONNECTOR" - | "NOTION_CONNECTOR" - | "FILE" - | "YOUTUBE_VIDEO" - | "GITHUB_CONNECTOR" - | "LINEAR_CONNECTOR" - | "JIRA_CONNECTOR" - | "DISCORD_CONNECTOR"; - -/** - * Skeleton loader for document items - */ -const DocumentSkeleton = () => ( -
- -
- - - -
- -
-); - -/** - * Enhanced document type filter dropdown - */ -const DocumentTypeFilter = ({ - value, - onChange, - counts, -}: { - value: DocumentType | "ALL"; - onChange: (value: DocumentType | "ALL") => void; - counts: Record; -}) => { - const getTypeLabel = (type: DocumentType | "ALL") => { - if (type === "ALL") return "All Types"; - return type - .replace(/_/g, " ") - .toLowerCase() - .replace(/\b\w/g, (l) => l.toUpperCase()); - }; - - const getTypeIcon = (type: DocumentType | "ALL") => { - if (type === "ALL") return ; - return getConnectorIcon(type); - }; - - return ( - - - - - - Document Types - - {Object.entries(counts).map(([type, count]) => ( - onChange(type as DocumentType | "ALL")} - className="flex items-center justify-between" - > -
- {getTypeIcon(type as DocumentType | "ALL")} - {getTypeLabel(type as DocumentType | "ALL")} -
- - {count} - - - ))} - - - ); -}; - -/** - * Button that displays selected connectors and opens connector selection dialog - */ -const ConnectorButton = ({ - selectedConnectors, - onClick, -}: { - selectedConnectors: string[]; - onClick: () => void; -}) => { - const { connectorSourceItems } = useSearchSourceConnectors(); - - return ( - - ); -}; - -/** - * Button that displays selected documents count and opens document selection dialog - */ -const DocumentSelectorButton = ({ - selectedDocuments, - onClick, - documentsCount, -}: { - selectedDocuments: number[]; - onClick: () => void; - documentsCount: number; -}) => { - return ( -
- - {selectedDocuments.length > 0 && ( - - {selectedDocuments.length > 99 ? "99+" : selectedDocuments.length} - - )} - {selectedDocuments.length === 0 && ( - - 0 - - )} -
- ); -}; - -// Create a wrapper component for the sources dialog content -const SourcesDialogContent = ({ - connector, - sourceFilter, - expandedSources, - sourcesPage, - setSourcesPage, - setSourceFilter, - setExpandedSources, - isLoadingMore, -}: { - connector: any; - sourceFilter: string; - expandedSources: boolean; - sourcesPage: number; - setSourcesPage: React.Dispatch>; - setSourceFilter: React.Dispatch>; - setExpandedSources: React.Dispatch>; - isLoadingMore: boolean; -}) => { - // Safely access sources with fallbacks - const sources = connector?.sources || []; - - // Safe versions of utility functions - const getFilteredSourcesSafe = () => { - if (!sources.length) return []; - return getFilteredSourcesUtil(connector, sourceFilter); - }; - - const getPaginatedSourcesSafe = () => { - if (!sources.length) return []; - return getPaginatedDialogSourcesUtil( - connector, - sourceFilter, - expandedSources, - sourcesPage, - 5, // SOURCES_PER_PAGE - ); - }; - - const filteredSources = getFilteredSourcesSafe() || []; - const paginatedSources = getPaginatedSourcesSafe() || []; - - // Description text - const descriptionText = sourceFilter - ? `Found ${filteredSources.length} sources matching "${sourceFilter}"` - : `Viewing ${paginatedSources.length} of ${sources.length} sources`; - - if (paginatedSources.length === 0) { - return ( -
- -

No sources found matching "{sourceFilter}"

- -
- ); - } - - return ( - <> - - - {getConnectorIcon(connector.type)} - {connector.name} Sources - - - {descriptionText} - - - -
- - { - setSourceFilter(e.target.value); - setSourcesPage(1); - setExpandedSources(false); - }} - /> - {sourceFilter && ( - - )} -
- -
- {paginatedSources.map((source: any, index: number) => ( - -
-
- {getConnectorIcon(connector.type)} -
-
-

{source.title}

-

- {source.description} -

-
- -
- - ))} - - {!expandedSources && - paginatedSources.length < filteredSources.length && ( - - )} - - {expandedSources && filteredSources.length > 10 && ( -
- Showing all {filteredSources.length} sources -
- )} -
- - ); -}; - -const ChatPage = () => { - const [token, setToken] = React.useState(null); - const [dialogOpenId, setDialogOpenId] = useState(null); - const [sourcesPage, setSourcesPage] = useState(1); - const [expandedSources, setExpandedSources] = useState(false); - const [canScrollLeft, setCanScrollLeft] = useState(false); - const [canScrollRight, setCanScrollRight] = useState(true); - const [sourceFilter, setSourceFilter] = useState(""); - const tabsListRef = useRef(null); - const [terminalExpanded, setTerminalExpanded] = useState(false); - const [selectedConnectors, setSelectedConnectors] = useState([]); - const [searchMode, setSearchMode] = useState<"DOCUMENTS" | "CHUNKS">( - "DOCUMENTS", - ); - const [researchMode, setResearchMode] = useState("QNA"); - const [currentTime, setCurrentTime] = useState(""); - const [currentDate, setCurrentDate] = useState(""); - const terminalMessagesRef = useRef(null); - const { connectorSourceItems, isLoading: isLoadingConnectors } = - useSearchSourceConnectors(); - const { llmConfigs } = useLLMConfigs(); - const { preferences, updatePreferences } = useLLMPreferences(); - - const INITIAL_SOURCES_DISPLAY = 3; - - const { search_space_id, chat_id } = useParams(); - - // Document selection state - const [selectedDocuments, setSelectedDocuments] = useState([]); - const [documentFilter, setDocumentFilter] = useState(""); - const [debouncedDocumentFilter, setDebouncedDocumentFilter] = useState(""); - const [documentTypeFilter, setDocumentTypeFilter] = useState< - DocumentType | "ALL" - >("ALL"); - const [documentsPage, setDocumentsPage] = useState(1); - const [documentsPerPage] = useState(10); - const { - documents, - loading: isLoadingDocuments, - error: documentsError, - } = useDocuments(Number(search_space_id)); - - // Debounced search effect (proper implementation) - useEffect(() => { - const handler = setTimeout(() => { - setDebouncedDocumentFilter(documentFilter); - setDocumentsPage(1); // Reset page when search changes - }, 300); - - return () => { - clearTimeout(handler); - }; - }, [documentFilter]); - - // Memoized filtered and paginated documents - const filteredDocuments = useMemo(() => { - if (!documents) return []; - - return documents.filter((doc) => { - const matchesSearch = - doc.title - .toLowerCase() - .includes(debouncedDocumentFilter.toLowerCase()) || - doc.content - .toLowerCase() - .includes(debouncedDocumentFilter.toLowerCase()); - const matchesType = - documentTypeFilter === "ALL" || - doc.document_type === documentTypeFilter; - return matchesSearch && matchesType; - }); - }, [documents, debouncedDocumentFilter, documentTypeFilter]); - - const paginatedDocuments = useMemo(() => { - const startIndex = (documentsPage - 1) * documentsPerPage; - return filteredDocuments.slice(startIndex, startIndex + documentsPerPage); - }, [filteredDocuments, documentsPage, documentsPerPage]); - - const totalPages = Math.ceil(filteredDocuments.length / documentsPerPage); - - // Document type counts for filter dropdown - const documentTypeCounts = useMemo(() => { - if (!documents) return {}; - - const counts: Record = { ALL: documents.length }; - documents.forEach((doc) => { - counts[doc.document_type] = (counts[doc.document_type] || 0) + 1; - }); - return counts; - }, [documents]); - - // Callback to handle document selection - const handleDocumentToggle = useCallback((documentId: number) => { - setSelectedDocuments((prev) => - prev.includes(documentId) - ? prev.filter((id) => id !== documentId) - : [...prev, documentId], - ); - }, []); - - // Function to scroll terminal to bottom - const scrollTerminalToBottom = () => { - if (terminalMessagesRef.current) { - terminalMessagesRef.current.scrollTop = - terminalMessagesRef.current.scrollHeight; - } - }; - - // Get token from localStorage on client side only - React.useEffect(() => { - setToken(localStorage.getItem("surfsense_bearer_token")); - }, []); - - // Set the current time only on the client side after initial render - useEffect(() => { - setCurrentDate(new Date().toISOString().split("T")[0]); - setCurrentTime(new Date().toTimeString().split(" ")[0]); - }, []); - - // Add this CSS to remove input shadow and improve the UI - useEffect(() => { - if (typeof document !== "undefined") { - const style = document.createElement("style"); - style.innerHTML = ` - .no-shadow-input { - box-shadow: none !important; - } - .no-shadow-input:focus-visible { - box-shadow: none !important; - outline: none !important; - } - .shadcn-selector { - transition: all 200ms cubic-bezier(0.4, 0, 0.2, 1); - border: 1px solid hsl(var(--border)); - background-color: transparent; - position: relative; - overflow: hidden; - } - .shadcn-selector:hover { - background-color: hsl(var(--muted)); - border-color: hsl(var(--primary) / 0.5); - } - .shadcn-selector:after { - content: ''; - position: absolute; - top: 0; - left: 0; - right: 0; - height: 0; - background: hsl(var(--primary) / 0.1); - transition: height 300ms ease; - } - .shadcn-selector:hover:after { - height: 100%; - } - .shadcn-selector-primary { - color: hsl(var(--primary)); - border-color: hsl(var(--primary) / 0.3); - } - .shadcn-selector-primary:hover { - border-color: hsl(var(--primary)); - background-color: hsl(var(--primary) / 0.1); - } - /* Fix for scrollbar layout shifts */ - html { - overflow-y: scroll; - } - body { - scrollbar-gutter: stable; - } - /* For Firefox */ - * { - scrollbar-width: thin; - } - /* For Webkit browsers */ - ::-webkit-scrollbar { - width: 8px; - height: 8px; - } - ::-webkit-scrollbar-track { - background: transparent; - } - ::-webkit-scrollbar-thumb { - background-color: rgba(155, 155, 155, 0.5); - border-radius: 20px; - } - /* Line clamp utility */ - .line-clamp-2 { - display: -webkit-box; - -webkit-line-clamp: 2; - -webkit-box-orient: vertical; - overflow: hidden; - } - /* Hide scrollbar by default, show on hover */ - .scrollbar-hover { - -ms-overflow-style: none; /* IE and Edge */ - scrollbar-width: none; /* Firefox */ - } - .scrollbar-hover::-webkit-scrollbar { - display: none; /* Chrome, Safari and Opera */ - } - .scrollbar-hover:hover { - -ms-overflow-style: auto; /* IE and Edge */ - scrollbar-width: thin; /* Firefox */ - } - .scrollbar-hover:hover::-webkit-scrollbar { - display: block; /* Chrome, Safari and Opera */ - height: 6px; - } - .scrollbar-hover:hover::-webkit-scrollbar-track { - background: hsl(var(--muted)); - border-radius: 3px; - } - .scrollbar-hover:hover::-webkit-scrollbar-thumb { - background: hsl(var(--muted-foreground) / 0.3); - border-radius: 3px; - } - .scrollbar-hover:hover::-webkit-scrollbar-thumb:hover { - background: hsl(var(--muted-foreground) / 0.5); - } - `; - document.head.appendChild(style); - - return () => { - document.head.removeChild(style); - }; - } - }, []); - - const { - messages, - input, - handleInputChange, - handleSubmit: handleChatSubmit, - status, - setMessages, - } = useChat({ - api: `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chat`, - streamProtocol: "data", - headers: { - ...(token && { Authorization: `Bearer ${token}` }), - }, - body: { - data: { - search_space_id: search_space_id, - selected_connectors: selectedConnectors, - research_mode: researchMode, - search_mode: searchMode, - document_ids_to_add_in_context: selectedDocuments, - }, - }, - onError: (error) => { - console.error("Chat error:", error); - // You can add additional error handling here if needed - }, - }); - - // Fetch chat details when component mounts - useEffect(() => { - const fetchChatDetails = async () => { - try { - if (!token) return; // Wait for token to be set - - // console.log('Fetching chat details for chat ID:', chat_id); - - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/${Number(chat_id)}`, - { - method: "GET", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - }, - ); - - if (!response.ok) { - throw new Error( - `Failed to fetch chat details: ${response.statusText}`, - ); - } - - const chatData = await response.json(); - // console.log('Chat details fetched:', chatData); - - // Set research mode from chat data - if (chatData.type) { - setResearchMode(chatData.type as ResearchMode); - } - - // Set connectors from chat data - if ( - chatData.initial_connectors && - Array.isArray(chatData.initial_connectors) - ) { - setSelectedConnectors(chatData.initial_connectors); - } - - // Set messages from chat data - if (chatData.messages && Array.isArray(chatData.messages)) { - setMessages(chatData.messages); - } - } catch (err) { - console.error("Error fetching chat details:", err); - } - }; - - if (token) { - fetchChatDetails(); - } - }, [token, chat_id, setMessages]); - - // Update chat when a conversation exchange is complete - useEffect(() => { - const updateChat = async () => { - try { - // Only update when: - // 1. Status is ready (not loading) - // 2. We have messages - // 3. Last message is from assistant (completed response) - if ( - status === "ready" && - messages.length > 0 && - messages[messages.length - 1]?.role === "assistant" - ) { - const token = localStorage.getItem("surfsense_bearer_token"); - if (!token) return; - - // Find the first user message to use as title - const userMessages = messages.filter((msg) => msg.role === "user"); - if (userMessages.length === 0) return; - - // Use the first user message as the title - const title = userMessages[0].content; - - // console.log('Updating chat with title:', title); - - // Update the chat - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/${Number(chat_id)}`, - { - method: "PUT", - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${token}`, - }, - body: JSON.stringify({ - type: researchMode, - title: title, - initial_connectors: selectedConnectors, - messages: messages, - search_space_id: Number(search_space_id), - }), - }, - ); - - if (!response.ok) { - throw new Error(`Failed to update chat: ${response.statusText}`); - } - - // console.log('Chat updated successfully'); - } - } catch (err) { - console.error("Error updating chat:", err); - } - }; - - updateChat(); - }, [ - messages, - status, - chat_id, - researchMode, - selectedConnectors, - search_space_id, - ]); - - // Check and scroll terminal when terminal info is available - useEffect(() => { - // Modified to trigger during streaming as well (removed status check) - if (messages.length === 0) return; - - // Find the latest assistant message - const assistantMessages = messages.filter( - (msg) => msg.role === "assistant", - ); - if (assistantMessages.length === 0) return; - - const latestAssistantMessage = - assistantMessages[assistantMessages.length - 1]; - if (!latestAssistantMessage?.annotations) return; - - // Check for terminal info annotations - const annotations = latestAssistantMessage.annotations as any[]; - const terminalInfoAnnotations = annotations.filter( - (a) => a.type === "TERMINAL_INFO", - ); - - if (terminalInfoAnnotations.length > 0) { - // Always scroll to bottom when terminal info is updated, even during streaming - scrollTerminalToBottom(); - } - }, [messages]); // Removed status from dependencies to ensure it triggers during streaming - - // Pure function to get connector sources for a specific message - const getMessageConnectorSources = (message: any): any[] => { - if (!message || message.role !== "assistant" || !message.annotations) - return []; - - // Find all SOURCES annotations - const annotations = message.annotations as any[]; - const sourcesAnnotations = annotations.filter((a) => a.type === "SOURCES"); - - // Get the latest SOURCES annotation - if (sourcesAnnotations.length === 0) return []; - const latestSourcesAnnotation = - sourcesAnnotations[sourcesAnnotations.length - 1]; - - if (!latestSourcesAnnotation.content) return []; - - return latestSourcesAnnotation.content; - }; - - // Custom handleSubmit function to include selected connectors and answer type - const handleSubmit = (e: React.FormEvent) => { - e.preventDefault(); - - if (!input.trim() || status !== "ready") return; - - // Validation: require at least one connector OR at least one document - // Note: Fast LLM selection updates user preferences automatically - // if (selectedConnectors.length === 0 && selectedDocuments.length === 0) { - // alert("Please select at least one connector or document"); - // return; - // } - - // Call the original handleSubmit from useChat - handleChatSubmit(e); - }; - - // Reference to the messages container for auto-scrolling - const messagesEndRef = useRef(null); - - // Function to scroll to bottom - const scrollToBottom = () => { - messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); - }; - - // Scroll to bottom when messages change - useEffect(() => { - scrollToBottom(); - }, [messages]); - - // Reset sources page when new messages arrive - useEffect(() => { - // Reset pagination when we get new messages - setSourcesPage(1); - setExpandedSources(false); - }, [messages]); - - // Scroll terminal to bottom when expanded - useEffect(() => { - if (terminalExpanded) { - setTimeout(scrollTerminalToBottom, 300); // Wait for transition to complete - } - }, [terminalExpanded]); - - // Function to check scroll position and update indicators - const updateScrollIndicators = () => { - updateScrollIndicatorsUtil( - tabsListRef as React.RefObject, - setCanScrollLeft, - setCanScrollRight, - ); - }; - - // Initialize scroll indicators - const updateIndicators = useScrollIndicators( - tabsListRef as React.RefObject, - setCanScrollLeft, - setCanScrollRight, - ); - - // Function to scroll tabs list left - const scrollTabsLeft = () => { - scrollTabsLeftUtil( - tabsListRef as React.RefObject, - updateIndicators, - ); - }; - - // Function to scroll tabs list right - const scrollTabsRight = () => { - scrollTabsRightUtil( - tabsListRef as React.RefObject, - updateIndicators, - ); - }; - - // Use the scroll to bottom hook - useScrollToBottom(messagesEndRef as React.RefObject, [ - messages, - ]); - - // Function to get a citation source by ID - const getCitationSource = React.useCallback( - (citationId: number, messageIndex?: number): Source | null => { - if (!messages || messages.length === 0) return null; - - // If no specific message index is provided, use the latest assistant message - if (messageIndex === undefined) { - // Find the latest assistant message - const assistantMessages = messages.filter( - (msg) => msg.role === "assistant", - ); - if (assistantMessages.length === 0) return null; - - const latestAssistantMessage = - assistantMessages[assistantMessages.length - 1]; - - // Use our helper function to get sources - const sources = getMessageConnectorSources(latestAssistantMessage); - if (sources.length === 0) return null; - - // Flatten all sources from all connectors - const allSources: Source[] = []; - sources.forEach((connector: ConnectorSource) => { - if (connector.sources && Array.isArray(connector.sources)) { - connector.sources.forEach((source: SourceItem) => { - allSources.push({ - id: source.id, - title: source.title, - description: source.description, - url: source.url, - connectorType: connector.type, - }); - }); - } - }); - - // Find the source with the matching ID - const foundSource = allSources.find( - (source) => source.id === citationId, - ); - - return foundSource || null; - } else { - // Use the specific message by index - const message = messages[messageIndex]; - - // Use our helper function to get sources - const sources = getMessageConnectorSources(message); - if (sources.length === 0) return null; - - // Flatten all sources from all connectors - const allSources: Source[] = []; - sources.forEach((connector: ConnectorSource) => { - if (connector.sources && Array.isArray(connector.sources)) { - connector.sources.forEach((source: SourceItem) => { - allSources.push({ - id: source.id, - title: source.title, - description: source.description, - url: source.url, - connectorType: connector.type, - }); - }); - } - }); - - // Find the source with the matching ID - const foundSource = allSources.find( - (source) => source.id === citationId, - ); - - return foundSource || null; - } - }, - [messages], - ); - - // Pure function for rendering terminal content - no hooks allowed here - const renderTerminalContent = (message: any) => { - if (!message.annotations) return null; - - // Get all TERMINAL_INFO annotations content - const terminalInfoAnnotations = (message.annotations as any[]) - .map((item) => { - if (item.type === "TERMINAL_INFO") { - return item.content.map((a: any) => a.text); - } - }) - .flat() - .filter(Boolean); - - // Render the content of the latest TERMINAL_INFO annotation - return terminalInfoAnnotations.map((item: any, idx: number) => ( -
- - [{String(idx).padStart(2, "0")}: - {String(Math.floor(idx * 2)).padStart(2, "0")}] - - {">"} - - {item} - -
- )); - }; - - return ( - <> -
- {messages.length === 0 && ( -

- -
- Surf{""} -
-
- Sense -
-
-
-

- )} - {messages?.map((message, index) => { - if (message.role === "user") { - return ( -
- -
- - - getCitationSource(id, index)} - className="text-sm" - /> - - -
-
- ); - } - - if (message.role === "assistant") { - return ( -
- - - - Answer - - - - {/* Status Messages Section */} - -
-
-
-
setTerminalExpanded(false)} - >
-
-
setTerminalExpanded(true)} - >
-
- - surfsense-research-terminal - -
-
- -
-
- Last login: {currentDate} {currentTime} -
-
- - researcher@surfsense - - : - ~/research - $ - surfsense-researcher -
- - {renderTerminalContent(message)} - -
- - [00:13] - - - researcher@surfsense - - : - ~/research - $ -
-
- - {/* Terminal scroll button */} -
- -
-
- - - {/* Sources Section with Connector Tabs */} -
-
- - Sources -
- - {(() => { - // Get sources for this specific message - const messageConnectorSources = - getMessageConnectorSources(message); - - if (messageConnectorSources.length === 0) { - return ( -
- -
- ); - } - - // Use these message-specific sources for the Tabs component - return ( - 0 - ? messageConnectorSources[0].type - : undefined - } - className="w-full" - > -
-
- - -
-
- - {messageConnectorSources.map( - (connector) => ( - - {getConnectorIcon(connector.type)} - - {connector.name.split(" ")[0]} - - - {connector.sources?.length || 0} - - - ), - )} - -
-
- - -
-
- - {messageConnectorSources.map((connector) => ( - -
- {connector.sources - ?.slice(0, INITIAL_SOURCES_DISPLAY) - ?.map((source: any, index: number) => ( - -
-
- {getConnectorIcon(connector.type)} -
-
-

- {source.title} -

-

- {source.description} -

-
- -
- - ))} - - {connector.sources?.length > - INITIAL_SOURCES_DISPLAY && ( - - setDialogOpenId( - open ? connector.id : null, - ) - } - > - - - - - - - - )} -
- - ))} - - ); - })()} -
- - {/* Answer Section */} -
- { -
- {message.annotations && - (() => { - // Get all ANSWER annotations - const answerAnnotations = ( - message.annotations as any[] - ).filter((a) => a.type === "ANSWER"); - - // Get the latest ANSWER annotation - const latestAnswer = - answerAnnotations.length > 0 - ? answerAnnotations[ - answerAnnotations.length - 1 - ] - : null; - - // If we have a latest ANSWER annotation with content, render it - if ( - latestAnswer?.content && - latestAnswer.content.length > 0 - ) { - return ( - - getCitationSource(id, index) - } - type="ai" - /> - ); - } - - // Fallback to the message content if no ANSWER annotation is available - return ( - - getCitationSource(id, index) - } - type="ai" - /> - ); - })()} -
- } -
- - {/* Further Questions Section */} - {message.annotations && - (() => { - // Get all FURTHER_QUESTIONS annotations - const furtherQuestionsAnnotations = ( - message.annotations as any[] - ).filter((a) => a.type === "FURTHER_QUESTIONS"); - - // Get the latest FURTHER_QUESTIONS annotation - const latestFurtherQuestions = - furtherQuestionsAnnotations.length > 0 - ? furtherQuestionsAnnotations[ - furtherQuestionsAnnotations.length - 1 - ] - : null; - - // Only render if we have questions - if ( - !latestFurtherQuestions?.content || - latestFurtherQuestions.content.length === 0 - ) { - return null; - } - - const furtherQuestions = latestFurtherQuestions.content; - - return ( -
- {/* Main container with improved styling */} -
- {/* Header with better visual separation */} -
-
-

- - - - Follow-up Questions -

- - {furtherQuestions.length} suggestion - {furtherQuestions.length !== 1 ? "s" : ""} - -
-
- - {/* Questions container with enhanced scrolling */} -
-
- {/* Left fade gradient */} -
- - {/* Right fade gradient */} -
- - {/* Scrollable container */} -
-
- {furtherQuestions.map( - (question: any, qIndex: number) => ( - - ), - )} -
-
-
-
-
-
- ); - })()} - {/* Scroll to bottom button */} -
- -
- - -
- ); - } - - return null; - })} - - {/* New Chat Input Form */} -
-
- - {/* Send button */} - -
-
-
- {/* Enhanced Document Selection Dialog */} - - - {}} - documentsCount={documents?.length || 0} - /> - - - - -
- - Select Documents - - {selectedDocuments.length} selected - -
- - - - Choose documents to include in your research context. Use - filters and search to find specific documents. - - - - {/* Enhanced Search and Filter Controls */} -
-
- {/* Search Input */} -
- - setDocumentFilter(e.target.value)} - /> - {documentFilter && ( - - )} -
- - {/* Document Type Filter */} - { - setDocumentTypeFilter(newType); - setDocumentsPage(1); // Reset to page 1 when filter changes - }} - counts={documentTypeCounts} - /> -
- - {/* Results Summary */} -
- - {isLoadingDocuments - ? "Loading documents..." - : `Showing ${paginatedDocuments.length} of ${filteredDocuments.length} documents`} - - {filteredDocuments.length > 0 && ( - - Page {documentsPage} of {totalPages} - - )} -
-
- - {/* Document List with Proper Scrolling */} -
-
- {isLoadingDocuments ? ( - // Enhanced skeleton loading - Array.from({ length: 6 }, (_, i) => ( - - )) - ) : documentsError ? ( -
-
- -
-

- Error loading documents -

-

- Please try refreshing the page -

-
- ) : filteredDocuments.length === 0 ? ( -
-
- -
-

- No documents found -

-

- {documentFilter || documentTypeFilter !== "ALL" - ? "Try adjusting your search or filters" - : "Upload documents to get started"} -

- {!documentFilter && documentTypeFilter === "ALL" && ( - - )} -
- ) : ( - // Enhanced document list - paginatedDocuments.map((document) => { - const isSelected = selectedDocuments.includes( - document.id, - ); - const typeLabel = document.document_type - .replace(/_/g, " ") - .toLowerCase(); - - return ( -
handleDocumentToggle(document.id)} - > -
-
- {getConnectorIcon(document.document_type)} -
-
-
-
-

- {document.title} -

- {isSelected && ( -
-
- -
-
- )} -
-
- - {typeLabel} - - - {new Date( - document.created_at, - ).toLocaleDateString()} - -
-

- {document.content.substring(0, 200)}... -

-
-
- ); - }) - )} -
-
- - {/* Enhanced Pagination Controls */} - {totalPages > 1 && ( -
-
- -
- {Array.from( - { length: Math.min(5, totalPages) }, - (_, i) => { - const page = - documentsPage <= 3 - ? i + 1 - : documentsPage - 2 + i; - if (page > totalPages) return null; - return ( - - ); - }, - )} - {totalPages > 5 && documentsPage < totalPages - 2 && ( - <> - - ... - - - - )} -
- -
-
- )} - - {/* Enhanced Footer */} - -
- - {selectedDocuments.length} of {filteredDocuments.length}{" "} - document{selectedDocuments.length !== 1 ? "s" : ""}{" "} - selected - -
-
- - - -
- - - - - {/* Connector Selection Dialog */} - - - {}} - /> - - - - Select Connectors - - Choose which data sources to include in your research - - - - {/* Connector selection grid */} -
- {isLoadingConnectors ? ( -
- -
- ) : ( - connectorSourceItems.map((connector) => { - const isSelected = selectedConnectors.includes( - connector.type, - ); - - return ( -
{ - setSelectedConnectors( - isSelected - ? selectedConnectors.filter( - (type) => type !== connector.type, - ) - : [...selectedConnectors, connector.type], - ); - }} - role="checkbox" - aria-checked={isSelected} - tabIndex={0} - > -
- {getConnectorIcon(connector.type)} -
- - {connector.name} - - {isSelected && ( - - )} -
- ); - }) - )} -
- - -
- - -
- - - - - {/* Search Mode Control */} -
- - -
- - {/* Research Mode Control */} -
- -
- - {/* Fast LLM Selector */} -
- -
-
-
-
- - {/* Reference for auto-scrolling */} -
-
- - ); -}; - -export default ChatPage; diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/page.tsx deleted file mode 100644 index 5e18082..0000000 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/page.tsx +++ /dev/null @@ -1,80 +0,0 @@ -"use client"; -import React, { useEffect } from 'react'; -import { useParams, useRouter } from 'next/navigation'; -import { Loader2 } from 'lucide-react'; - -const ResearcherPage = () => { - const router = useRouter(); - const { search_space_id } = useParams(); - const [isCreating, setIsCreating] = React.useState(true); - const [error, setError] = React.useState(null); - - useEffect(() => { - const createChat = async () => { - try { - // Get token from localStorage - const token = localStorage.getItem('surfsense_bearer_token'); - - if (!token) { - setError('Authentication token not found'); - setIsCreating(false); - return; - } - - // Create a new chat - const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` - }, - body: JSON.stringify({ - type: "QNA", - title: "Untitled Chat", // Empty title initially - initial_connectors: [], // No default connectors - messages: [], - search_space_id: Number(search_space_id) - }) - }); - - if (!response.ok) { - throw new Error(`Failed to create chat: ${response.statusText}`); - } - - const data = await response.json(); - - // Redirect to the new chat page - router.push(`/dashboard/${search_space_id}/researcher/${data.id}`); - } catch (err) { - console.error('Error creating chat:', err); - setError(err instanceof Error ? err.message : 'Failed to create chat'); - setIsCreating(false); - } - }; - - createChat(); - }, [search_space_id, router]); - - if (error) { - return ( -
-
Error: {error}
- -
- ); - } - - return ( -
- -

Creating new research chat...

-
- ); -}; - -export default ResearcherPage; \ No newline at end of file diff --git a/surfsense_web/app/globals.css b/surfsense_web/app/globals.css index a744656..88bd7ce 100644 --- a/surfsense_web/app/globals.css +++ b/surfsense_web/app/globals.css @@ -155,4 +155,6 @@ button { cursor: pointer; -} \ No newline at end of file +} + +@source '../node_modules/@llamaindex/chat-ui/**/*.{ts,tsx}' \ No newline at end of file diff --git a/surfsense_web/components/chat/ChatCitation.tsx b/surfsense_web/components/chat/ChatCitation.tsx new file mode 100644 index 0000000..14b61c6 --- /dev/null +++ b/surfsense_web/components/chat/ChatCitation.tsx @@ -0,0 +1,62 @@ +"use client"; + +import React from "react"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { ExternalLink } from "lucide-react"; + +export const CitationDisplay: React.FC<{ index: number; node: any }> = ({ + index, + node, +}) => { + const truncateText = (text: string, maxLength: number = 200) => { + if (text.length <= maxLength) return text; + return text.substring(0, maxLength) + "..."; + }; + + const handleUrlClick = (e: React.MouseEvent, url: string) => { + e.preventDefault(); + e.stopPropagation(); + window.open(url, "_blank", "noopener,noreferrer"); + }; + + return ( + + + + {index + 1} + + + + {/* External Link Button - Top Right */} + {node?.url && ( + + )} + + {/* Heading */} +
+ {node?.metadata?.group_name || "Source"} +
+ + {/* Source */} +
+ {node?.metadata?.title || "Untitled"} +
+ + {/* Body */} +
+ {truncateText(node?.text || "No content available")} +
+ + + ); +}; diff --git a/surfsense_web/components/chat/ChatFurtherQuestions.tsx b/surfsense_web/components/chat/ChatFurtherQuestions.tsx new file mode 100644 index 0000000..580052f --- /dev/null +++ b/surfsense_web/components/chat/ChatFurtherQuestions.tsx @@ -0,0 +1,45 @@ +"use client"; + +import { SuggestedQuestions } from "@llamaindex/chat-ui/widgets"; +import { getAnnotationData, Message, useChatUI } from "@llamaindex/chat-ui"; +import { + Accordion, + AccordionContent, + AccordionItem, + AccordionTrigger, +} from "@/components/ui/accordion"; + +export const ChatFurtherQuestions: React.FC<{ message: Message }> = ({ + message, +}) => { + const annotations: string[][] = getAnnotationData( + message, + "FURTHER_QUESTIONS", + ); + const { append, requestData } = useChatUI(); + + if (annotations.length !== 1 || annotations[0].length === 0) { + return <>; + } + + return ( + + + + Suggested Questions + + + + + + + ); +}; diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx new file mode 100644 index 0000000..142f0bc --- /dev/null +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -0,0 +1,617 @@ +"use client"; + +import { ChatInput } from "@llamaindex/chat-ui"; +import { FolderOpen, Check, Zap, Brain } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogTitle, + DialogTrigger, + DialogFooter, +} from "@/components/ui/dialog"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Badge } from "@/components/ui/badge"; +import { Suspense, useState, useCallback } from "react"; +import { useParams } from "next/navigation"; +import { useDocuments, Document } from "@/hooks/use-documents"; +import { DocumentsDataTable } from "@/components/chat/DocumentsDataTable"; +import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; +import { + getConnectorIcon, + ConnectorButton as ConnectorButtonComponent, +} from "@/components/chat/ConnectorComponents"; +import { ResearchMode } from "@/components/chat"; +import { useLLMConfigs, useLLMPreferences } from "@/hooks/use-llm-configs"; +import React from "react"; + +const DocumentSelector = React.memo( + ({ + onSelectionChange, + selectedDocuments = [], + }: { + onSelectionChange?: (documents: Document[]) => void; + selectedDocuments?: Document[]; + }) => { + const { search_space_id } = useParams(); + const [isOpen, setIsOpen] = useState(false); + + const { documents, loading, isLoaded, fetchDocuments } = useDocuments( + Number(search_space_id), + true, + ); + + const handleOpenChange = useCallback( + (open: boolean) => { + setIsOpen(open); + if (open && !isLoaded) { + fetchDocuments(); + } + }, + [fetchDocuments, isLoaded], + ); + + const handleSelectionChange = useCallback( + (documents: Document[]) => { + onSelectionChange?.(documents); + }, + [onSelectionChange], + ); + + const handleDone = useCallback(() => { + setIsOpen(false); + }, []); + + const selectedCount = React.useMemo( + () => selectedDocuments.length, + [selectedDocuments.length], + ); + + return ( + + + + + + +
+
+ + Select Documents + + + Choose documents to include in your research context + +
+ +
+ {loading ? ( +
+
+
+

+ Loading documents... +

+
+
+ ) : isLoaded ? ( + + ) : null} +
+
+ +
+ ); + }, +); + +DocumentSelector.displayName = "DocumentSelector"; + +const ConnectorSelector = React.memo( + ({ + onSelectionChange, + selectedConnectors = [], + }: { + onSelectionChange?: (connectorTypes: string[]) => void; + selectedConnectors?: string[]; + }) => { + const [isOpen, setIsOpen] = useState(false); + + const { connectorSourceItems, isLoading, isLoaded, fetchConnectors } = + useSearchSourceConnectors(true); + + const handleOpenChange = useCallback( + (open: boolean) => { + setIsOpen(open); + if (open && !isLoaded) { + fetchConnectors(); + } + }, + [fetchConnectors, isLoaded], + ); + + const handleConnectorToggle = useCallback( + (connectorType: string) => { + const isSelected = selectedConnectors.includes(connectorType); + const newSelection = isSelected + ? selectedConnectors.filter((type) => type !== connectorType) + : [...selectedConnectors, connectorType]; + onSelectionChange?.(newSelection); + }, + [selectedConnectors, onSelectionChange], + ); + + const handleSelectAll = useCallback(() => { + onSelectionChange?.(connectorSourceItems.map((c) => c.type)); + }, [connectorSourceItems, onSelectionChange]); + + const handleClearAll = useCallback(() => { + onSelectionChange?.([]); + }, [onSelectionChange]); + + return ( + + + setIsOpen(true)} + connectorSources={connectorSourceItems} + /> + + + + Select Connectors + + Choose which data sources to include in your research + + + {/* Connector selection grid */} +
+ {isLoading ? ( +
+
+
+ ) : ( + connectorSourceItems.map((connector) => { + const isSelected = selectedConnectors.includes(connector.type); + + return ( +
handleConnectorToggle(connector.type)} + role="checkbox" + aria-checked={isSelected} + tabIndex={0} + > +
+ {getConnectorIcon(connector.type)} +
+ + {connector.name} + + {isSelected && } +
+ ); + }) + )} +
+ + +
+ + +
+ + +
+ ); + }, +); + +ConnectorSelector.displayName = "ConnectorSelector"; + +const SearchModeSelector = React.memo( + ({ + searchMode, + onSearchModeChange, + }: { + searchMode?: "DOCUMENTS" | "CHUNKS"; + onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void; + }) => { + const handleDocumentsClick = React.useCallback(() => { + onSearchModeChange?.("DOCUMENTS"); + }, [onSearchModeChange]); + + const handleChunksClick = React.useCallback(() => { + onSearchModeChange?.("CHUNKS"); + }, [onSearchModeChange]); + + return ( +
+ + Scope: + +
+ + +
+
+ ); + }, +); + +SearchModeSelector.displayName = "SearchModeSelector"; + +const ResearchModeSelector = React.memo( + ({ + researchMode, + onResearchModeChange, + }: { + researchMode?: ResearchMode; + onResearchModeChange?: (mode: ResearchMode) => void; + }) => { + const handleValueChange = React.useCallback( + (value: string) => { + onResearchModeChange?.(value as ResearchMode); + }, + [onResearchModeChange], + ); + + // Memoize mode options to prevent recreation + const modeOptions = React.useMemo( + () => [ + { value: "QNA", label: "Q&A", shortLabel: "Q&A" }, + { + value: "REPORT_GENERAL", + label: "General Report", + shortLabel: "General", + }, + { + value: "REPORT_DEEP", + label: "Deep Report", + shortLabel: "Deep", + }, + { + value: "REPORT_DEEPER", + label: "Deeper Report", + shortLabel: "Deeper", + }, + ], + [], + ); + + return ( +
+ + Mode: + + +
+ ); + }, +); + +ResearchModeSelector.displayName = "ResearchModeSelector"; + +const LLMSelector = React.memo(() => { + const { llmConfigs, loading: llmLoading, error } = useLLMConfigs(); + const { + preferences, + updatePreferences, + loading: preferencesLoading, + } = useLLMPreferences(); + + const isLoading = llmLoading || preferencesLoading; + + // Memoize the selected config to avoid repeated lookups + const selectedConfig = React.useMemo(() => { + if (!preferences.fast_llm_id || !llmConfigs.length) return null; + return ( + llmConfigs.find((config) => config.id === preferences.fast_llm_id) || null + ); + }, [preferences.fast_llm_id, llmConfigs]); + + // Memoize the display value for the trigger + const displayValue = React.useMemo(() => { + if (!selectedConfig) return null; + return ( +
+ {selectedConfig.provider} + • + + {selectedConfig.name} + +
+ ); + }, [selectedConfig]); + + const handleValueChange = React.useCallback( + (value: string) => { + const llmId = value ? parseInt(value, 10) : undefined; + updatePreferences({ fast_llm_id: llmId }); + }, + [updatePreferences], + ); + + // Loading skeleton + if (isLoading) { + return ( +
+
+
+
+
+
+ ); + } + + // Error state + if (error) { + return ( +
+ +
+ ); + } + + return ( +
+ +
+ ); +}); + +LLMSelector.displayName = "LLMSelector"; + +const CustomChatInputOptions = React.memo( + ({ + onDocumentSelectionChange, + selectedDocuments, + onConnectorSelectionChange, + selectedConnectors, + searchMode, + onSearchModeChange, + researchMode, + onResearchModeChange, + }: { + onDocumentSelectionChange?: (documents: Document[]) => void; + selectedDocuments?: Document[]; + onConnectorSelectionChange?: (connectorTypes: string[]) => void; + selectedConnectors?: string[]; + searchMode?: "DOCUMENTS" | "CHUNKS"; + onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void; + researchMode?: ResearchMode; + onResearchModeChange?: (mode: ResearchMode) => void; + }) => { + // Memoize the loading fallback to prevent recreation + const loadingFallback = React.useMemo( + () => ( +
+ ), + [], + ); + + return ( +
+ + + + + + + + + +
+ ); + }, +); + +CustomChatInputOptions.displayName = "CustomChatInputOptions"; + +export const ChatInputUI = React.memo( + ({ + onDocumentSelectionChange, + selectedDocuments, + onConnectorSelectionChange, + selectedConnectors, + searchMode, + onSearchModeChange, + researchMode, + onResearchModeChange, + }: { + onDocumentSelectionChange?: (documents: Document[]) => void; + selectedDocuments?: Document[]; + onConnectorSelectionChange?: (connectorTypes: string[]) => void; + selectedConnectors?: string[]; + searchMode?: "DOCUMENTS" | "CHUNKS"; + onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void; + researchMode?: ResearchMode; + onResearchModeChange?: (mode: ResearchMode) => void; + }) => { + return ( + + + + + + + + ); + }, +); + +ChatInputUI.displayName = "ChatInputUI"; diff --git a/surfsense_web/components/chat/ChatInterface.tsx b/surfsense_web/components/chat/ChatInterface.tsx new file mode 100644 index 0000000..684b319 --- /dev/null +++ b/surfsense_web/components/chat/ChatInterface.tsx @@ -0,0 +1,55 @@ +"use client"; + +import React from "react"; +import { + ChatSection as LlamaIndexChatSection, + ChatHandler, +} from "@llamaindex/chat-ui"; +import { Document } from "@/hooks/use-documents"; +import { ChatInputUI } from "@/components/chat/ChatInputGroup"; +import { ResearchMode } from "@/components/chat"; +import { ChatMessagesUI } from "@/components/chat/ChatMessages"; + +interface ChatInterfaceProps { + handler: ChatHandler; + onDocumentSelectionChange?: (documents: Document[]) => void; + selectedDocuments?: Document[]; + onConnectorSelectionChange?: (connectorTypes: string[]) => void; + selectedConnectors?: string[]; + searchMode?: "DOCUMENTS" | "CHUNKS"; + onSearchModeChange?: (mode: "DOCUMENTS" | "CHUNKS") => void; + researchMode?: ResearchMode; + onResearchModeChange?: (mode: ResearchMode) => void; +} + +export default function ChatInterface({ + handler, + onDocumentSelectionChange, + selectedDocuments = [], + onConnectorSelectionChange, + selectedConnectors = [], + searchMode, + onSearchModeChange, + researchMode, + onResearchModeChange, +}: ChatInterfaceProps) { + return ( + +
+ +
+ +
+
+ + ); +} diff --git a/surfsense_web/components/chat/ChatMessages.tsx b/surfsense_web/components/chat/ChatMessages.tsx new file mode 100644 index 0000000..5720cf4 --- /dev/null +++ b/surfsense_web/components/chat/ChatMessages.tsx @@ -0,0 +1,78 @@ +"use client"; + +import React from "react"; +import { + ChatMessage as LlamaIndexChatMessage, + ChatMessages as LlamaIndexChatMessages, + Message, + useChatUI, +} from "@llamaindex/chat-ui"; +import TerminalDisplay from "@/components/chat/ChatTerminal"; +import ChatSourcesDisplay from "@/components/chat/ChatSources"; +import { CitationDisplay } from "@/components/chat/ChatCitation"; +import { ChatFurtherQuestions } from "@/components/chat/ChatFurtherQuestions"; + +export function ChatMessagesUI() { + const { messages } = useChatUI(); + + return ( + + + + {messages.map((message, index) => ( + + ))} + + + + ); +} + +function ChatMessageUI({ + message, + isLast, +}: { + message: Message; + isLast: boolean; +}) { + const bottomRef = React.useRef(null); + + React.useEffect(() => { + if (isLast && bottomRef.current) { + bottomRef.current.scrollIntoView({ behavior: "smooth" }); + } + }, [message]); + + return ( + + {message.role === "assistant" ? ( +
+ + + + + +
+
+ {isLast && } + +
+
+ ) : ( + + + + )} + + ); +} diff --git a/surfsense_web/components/chat/ChatSources.tsx b/surfsense_web/components/chat/ChatSources.tsx new file mode 100644 index 0000000..c334bab --- /dev/null +++ b/surfsense_web/components/chat/ChatSources.tsx @@ -0,0 +1,224 @@ +"use client"; + +import { useState } from "react"; +import { getAnnotationData, Message } from "@llamaindex/chat-ui"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { ExternalLink, FileText, Globe } from "lucide-react"; +import { IconBrandGithub } from "@tabler/icons-react"; + +interface Source { + id: string; + title: string; + description: string; + url: string; +} + +interface SourceGroup { + id: number; + name: string; + type: string; + sources: Source[]; +} + +// New interfaces for the updated data format +interface NodeMetadata { + title: string; + source_type: string; + group_name: string; +} + +interface SourceNode { + id: string; + text: string; + url: string; + metadata: NodeMetadata; +} + +interface NodesResponse { + nodes: SourceNode[]; +} + +function getSourceIcon(type: string) { + switch (type) { + case "USER_SELECTED_GITHUB_CONNECTOR": + case "GITHUB_CONNECTOR": + return ; + case "USER_SELECTED_NOTION_CONNECTOR": + case "NOTION_CONNECTOR": + return ; + case "USER_SELECTED_FILE": + case "FILE": + return ; + default: + return ; + } +} + +function SourceCard({ source }: { source: Source }) { + const hasUrl = source.url && source.url.trim() !== ""; + + return ( + + +
+ + {source.title} + + {hasUrl && ( + + )} +
+ + + + {source.description} + + + + ); +} + +export default function ChatSourcesDisplay({ message }: { message: Message }) { + const [open, setOpen] = useState(false); + const annotations = getAnnotationData(message, "sources"); + + // Transform the new data format to the expected SourceGroup format + const sourceGroups: SourceGroup[] = []; + + if (Array.isArray(annotations) && annotations.length > 0) { + // Extract all nodes from the response + const allNodes: SourceNode[] = []; + + annotations.forEach((item) => { + if ( + item && + typeof item === "object" && + "nodes" in item && + Array.isArray(item.nodes) + ) { + allNodes.push(...item.nodes); + } + }); + + // Group nodes by source_type + const groupedByType = allNodes.reduce( + (acc, node) => { + const sourceType = node.metadata.source_type; + if (!acc[sourceType]) { + acc[sourceType] = []; + } + acc[sourceType].push(node); + return acc; + }, + {} as Record, + ); + + // Convert grouped nodes to SourceGroup format + Object.entries(groupedByType).forEach(([sourceType, nodes], index) => { + if (nodes.length > 0) { + const firstNode = nodes[0]; + sourceGroups.push({ + id: index + 100, // Generate unique ID + name: firstNode.metadata.group_name, + type: sourceType, + sources: nodes.map((node) => ({ + id: node.id, + title: node.metadata.title, + description: node.text, + url: node.url || "", + })), + }); + } + }); + } + + if (sourceGroups.length === 0) { + return null; + } + + const totalSources = sourceGroups.reduce( + (acc, group) => acc + group.sources.length, + 0, + ); + + return ( + + + + + + + Sources + + +
+ + {sourceGroups.map((group) => ( + + {getSourceIcon(group.type)} + + {group.name} + + + {group.sources.length} + + + ))} + +
+ {sourceGroups.map((group) => ( + +
+
+ {group.sources.map((source) => ( + + ))} +
+
+ + ))} + + + + ); +} diff --git a/surfsense_web/components/chat/ChatTerminal.tsx b/surfsense_web/components/chat/ChatTerminal.tsx new file mode 100644 index 0000000..131d58a --- /dev/null +++ b/surfsense_web/components/chat/ChatTerminal.tsx @@ -0,0 +1,115 @@ +"use client"; + +import React from "react"; +import { getAnnotationData, Message } from "@llamaindex/chat-ui"; + +export default function TerminalDisplay({ + message, + open, +}: { + message: Message; + open: boolean; +}) { + const [isCollapsed, setIsCollapsed] = React.useState(!open); + + const bottomRef = React.useRef(null); + + // Get the last assistant message that's not being typed + if (!message) { + return <>; + } + + interface TerminalInfo { + id: number; + text: string; + type: string; + } + + const events = getAnnotationData(message, "TERMINAL_INFO") as TerminalInfo[]; + + if (events.length === 0) { + return <>; + } + + React.useEffect(() => { + if (bottomRef.current) { + bottomRef.current.scrollTo({ + top: bottomRef.current.scrollHeight, + behavior: "smooth", + }); + } + }, [events]); + + return ( +
+ {/* Terminal Header */} +
setIsCollapsed(!isCollapsed)} + > +
+
+
+
+
+
+ Agent Process Terminal ({events.length} events) +
+
+ {isCollapsed ? ( + + + + ) : ( + + + + )} +
+
+ + {/* Terminal Content */} + {!isCollapsed && ( +
+ {events.map((event, index) => ( +
+ $ + [{event.type || ""}] + + {event.text || ""}... + +
+ ))} + {events.length === 0 && ( +
+ No agent events to display... +
+ )} +
+ )} +
+ ); +} diff --git a/surfsense_web/components/chat/DocumentsDataTable.tsx b/surfsense_web/components/chat/DocumentsDataTable.tsx new file mode 100644 index 0000000..7e441bb --- /dev/null +++ b/surfsense_web/components/chat/DocumentsDataTable.tsx @@ -0,0 +1,502 @@ +"use client"; + +import * as React from "react"; +import { + ColumnDef, + ColumnFiltersState, + flexRender, + getCoreRowModel, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + SortingState, + useReactTable, + VisibilityState, +} from "@tanstack/react-table"; +import { ArrowUpDown, Calendar, FileText, Search } from "lucide-react"; + +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from "@/components/ui/select"; +import { Input } from "@/components/ui/input"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { Badge } from "@/components/ui/badge"; +import { Document, DocumentType } from "@/hooks/use-documents"; + +interface DocumentsDataTableProps { + documents: Document[]; + onSelectionChange: (documents: Document[]) => void; + onDone: () => void; + initialSelectedDocuments?: Document[]; +} + +const DOCUMENT_TYPES: (DocumentType | "ALL")[] = [ + "ALL", + "FILE", + "EXTENSION", + "CRAWLED_URL", + "YOUTUBE_VIDEO", + "SLACK_CONNECTOR", + "NOTION_CONNECTOR", + "GITHUB_CONNECTOR", + "LINEAR_CONNECTOR", + "DISCORD_CONNECTOR", + "JIRA_CONNECTOR", +]; + +const getDocumentTypeColor = (type: DocumentType) => { + const colors = { + FILE: "bg-blue-50 text-blue-700 border-blue-200", + EXTENSION: "bg-green-50 text-green-700 border-green-200", + CRAWLED_URL: "bg-purple-50 text-purple-700 border-purple-200", + YOUTUBE_VIDEO: "bg-red-50 text-red-700 border-red-200", + SLACK_CONNECTOR: "bg-yellow-50 text-yellow-700 border-yellow-200", + NOTION_CONNECTOR: "bg-indigo-50 text-indigo-700 border-indigo-200", + GITHUB_CONNECTOR: "bg-gray-50 text-gray-700 border-gray-200", + LINEAR_CONNECTOR: "bg-pink-50 text-pink-700 border-pink-200", + DISCORD_CONNECTOR: "bg-violet-50 text-violet-700 border-violet-200", + JIRA_CONNECTOR: "bg-orange-50 text-orange-700 border-orange-200", + }; + return colors[type] || "bg-gray-50 text-gray-700 border-gray-200"; +}; + +const columns: ColumnDef[] = [ + { + id: "select", + header: ({ table }) => ( + table.toggleAllPageRowsSelected(!!value)} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + row.toggleSelected(!!value)} + aria-label="Select row" + /> + ), + enableSorting: false, + enableHiding: false, + size: 40, + }, + { + accessorKey: "title", + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const title = row.getValue("title") as string; + return ( +
+ {title} +
+ ); + }, + }, + { + accessorKey: "document_type", + header: "Type", + cell: ({ row }) => { + const type = row.getValue("document_type") as DocumentType; + return ( + + {type.replace(/_/g, " ")} + {type.split("_")[0]} + + ); + }, + size: 80, + meta: { + className: "hidden sm:table-cell", + }, + }, + { + accessorKey: "content", + header: "Preview", + cell: ({ row }) => { + const content = row.getValue("content") as string; + return ( +
+ {content.substring(0, 30)}... + + {content.substring(0, 100)}... + +
+ ); + }, + enableSorting: false, + meta: { + className: "hidden md:table-cell", + }, + }, + { + accessorKey: "created_at", + header: ({ column }) => ( + + ), + cell: ({ row }) => { + const date = new Date(row.getValue("created_at")); + return ( +
+ + {date.toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + })} + + + {date.toLocaleDateString("en-US", { + month: "numeric", + day: "numeric", + })} + +
+ ); + }, + size: 80, + }, +]; + +export function DocumentsDataTable({ + documents, + onSelectionChange, + onDone, + initialSelectedDocuments = [], +}: DocumentsDataTableProps) { + const [sorting, setSorting] = React.useState([]); + const [columnFilters, setColumnFilters] = React.useState( + [], + ); + const [columnVisibility, setColumnVisibility] = + React.useState({}); + const [documentTypeFilter, setDocumentTypeFilter] = React.useState< + DocumentType | "ALL" + >("ALL"); + + // Memoize initial row selection to prevent infinite loops + const initialRowSelection = React.useMemo(() => { + if (!documents.length || !initialSelectedDocuments.length) return {}; + + const selection: Record = {}; + initialSelectedDocuments.forEach((selectedDoc) => { + selection[selectedDoc.id] = true; + }); + return selection; + }, [documents, initialSelectedDocuments]); + + const [rowSelection, setRowSelection] = React.useState< + Record + >({}); + + // Only update row selection when initialRowSelection actually changes and is not empty + React.useEffect(() => { + const hasChanges = + JSON.stringify(rowSelection) !== JSON.stringify(initialRowSelection); + if (hasChanges && Object.keys(initialRowSelection).length > 0) { + setRowSelection(initialRowSelection); + } + }, [initialRowSelection]); + + // Initialize row selection on mount + React.useEffect(() => { + if ( + Object.keys(rowSelection).length === 0 && + Object.keys(initialRowSelection).length > 0 + ) { + setRowSelection(initialRowSelection); + } + }, []); + + const filteredDocuments = React.useMemo(() => { + if (documentTypeFilter === "ALL") return documents; + return documents.filter((doc) => doc.document_type === documentTypeFilter); + }, [documents, documentTypeFilter]); + + const table = useReactTable({ + data: filteredDocuments, + columns, + getRowId: (row) => row.id.toString(), + onSortingChange: setSorting, + onColumnFiltersChange: setColumnFilters, + getCoreRowModel: getCoreRowModel(), + getPaginationRowModel: getPaginationRowModel(), + getSortedRowModel: getSortedRowModel(), + getFilteredRowModel: getFilteredRowModel(), + onColumnVisibilityChange: setColumnVisibility, + onRowSelectionChange: setRowSelection, + initialState: { pagination: { pageSize: 10 } }, + state: { sorting, columnFilters, columnVisibility, rowSelection }, + }); + + React.useEffect(() => { + const selectedRows = table.getFilteredSelectedRowModel().rows; + const selectedDocuments = selectedRows.map((row) => row.original); + onSelectionChange(selectedDocuments); + }, [rowSelection, onSelectionChange, table]); + + const handleClearAll = () => setRowSelection({}); + + const handleSelectPage = () => { + const currentPageRows = table.getRowModel().rows; + const newSelection = { ...rowSelection }; + currentPageRows.forEach((row) => { + newSelection[row.id] = true; + }); + setRowSelection(newSelection); + }; + + const handleSelectAllFiltered = () => { + const allFilteredRows = table.getFilteredRowModel().rows; + const newSelection: Record = {}; + allFilteredRows.forEach((row) => { + newSelection[row.id] = true; + }); + setRowSelection(newSelection); + }; + + const selectedCount = table.getFilteredSelectedRowModel().rows.length; + const totalFiltered = table.getFilteredRowModel().rows.length; + + return ( +
+ {/* Header Controls */} +
+ {/* Search and Filter Row */} +
+
+ + + table.getColumn("title")?.setFilterValue(event.target.value) + } + className="pl-10 text-sm" + /> +
+ +
+ + {/* Action Controls Row */} +
+
+ + {selectedCount} of {totalFiltered} selected + +
+
+ + + + +
+
+ +
+
+ + {/* Table Container */} +
+
+ + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder + ? null + : flexRender( + header.column.columnDef.header, + header.getContext(), + )} + + ))} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ))} + + )) + ) : ( + + + No documents found. + + + )} + +
+
+
+ + {/* Footer Pagination */} +
+
+ Showing{" "} + {table.getState().pagination.pageIndex * + table.getState().pagination.pageSize + + 1}{" "} + to{" "} + {Math.min( + (table.getState().pagination.pageIndex + 1) * + table.getState().pagination.pageSize, + table.getFilteredRowModel().rows.length, + )}{" "} + of {table.getFilteredRowModel().rows.length} documents +
+
+ +
+ Page + {table.getState().pagination.pageIndex + 1} + of + {table.getPageCount()} +
+ +
+
+
+ ); +} diff --git a/surfsense_web/components/chat/connector-sources.ts b/surfsense_web/components/chat/connector-sources.ts deleted file mode 100644 index 58a4bd1..0000000 --- a/surfsense_web/components/chat/connector-sources.ts +++ /dev/null @@ -1,23 +0,0 @@ -// Connector sources -export const connectorSourcesMenu = [ - { - id: 1, - name: "Crawled URL", - type: "CRAWLED_URL", - }, - { - id: 2, - name: "File", - type: "FILE", - }, - { - id: 3, - name: "Extension", - type: "EXTENSION", - }, - { - id: 4, - name: "Youtube Video", - type: "YOUTUBE_VIDEO", - } - ]; \ No newline at end of file diff --git a/surfsense_web/components/ui/accordion.tsx b/surfsense_web/components/ui/accordion.tsx index f0b1d32..4a8cca4 100644 --- a/surfsense_web/components/ui/accordion.tsx +++ b/surfsense_web/components/ui/accordion.tsx @@ -2,59 +2,65 @@ import * as React from "react" import * as AccordionPrimitive from "@radix-ui/react-accordion" -import { ChevronDown } from "lucide-react" +import { ChevronDownIcon } from "lucide-react" import { cn } from "@/lib/utils" -const Accordion = AccordionPrimitive.Root +function Accordion({ + ...props +}: React.ComponentProps) { + return +} -const AccordionItem = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef ->(({ className, ...props }, ref) => ( - -)) -AccordionItem.displayName = "AccordionItem" +function AccordionItem({ + className, + ...props +}: React.ComponentProps) { + return ( + + ) +} -const AccordionTrigger = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef ->(({ className, children, ...props }, ref) => ( - - svg]:rotate-180", - className - )} +function AccordionTrigger({ + className, + children, + ...props +}: React.ComponentProps) { + return ( + + svg]:rotate-180", + className + )} + {...props} + > + {children} + + + + ) +} + +function AccordionContent({ + className, + children, + ...props +}: React.ComponentProps) { + return ( + - {children} - - - -)) -AccordionTrigger.displayName = "AccordionTrigger" +
{children}
+ + ) +} -const AccordionContent = React.forwardRef< - React.ElementRef, - React.ComponentPropsWithoutRef ->(({ className, children, ...props }, ref) => ( - -
{children}
- -)) -AccordionContent.displayName = "AccordionContent" - -export { Accordion, AccordionItem, AccordionTrigger, AccordionContent } \ No newline at end of file +export { Accordion, AccordionItem, AccordionTrigger, AccordionContent } diff --git a/surfsense_web/components/ui/table.tsx b/surfsense_web/components/ui/table.tsx index 27697c0..51b74dd 100644 --- a/surfsense_web/components/ui/table.tsx +++ b/surfsense_web/components/ui/table.tsx @@ -1,95 +1,116 @@ -import * as React from "react"; +"use client" -import { cn } from "@/lib/utils"; +import * as React from "react" -const Table = React.forwardRef>( - ({ className, ...props }, ref) => ( -
- +import { cn } from "@/lib/utils" + +function Table({ className, ...props }: React.ComponentProps<"table">) { + return ( +
+
- ), -); -Table.displayName = "Table"; + ) +} -const TableHeader = React.forwardRef< - HTMLTableSectionElement, - React.HTMLAttributes ->(({ className, ...props }, ref) => ); -TableHeader.displayName = "TableHeader"; +function TableHeader({ className, ...props }: React.ComponentProps<"thead">) { + return ( + + ) +} -const TableBody = React.forwardRef< - HTMLTableSectionElement, - React.HTMLAttributes ->(({ className, ...props }, ref) => ( - -)); -TableBody.displayName = "TableBody"; +function TableBody({ className, ...props }: React.ComponentProps<"tbody">) { + return ( + + ) +} -const TableFooter = React.forwardRef< - HTMLTableSectionElement, - React.HTMLAttributes ->(({ className, ...props }, ref) => ( - tr]:last:border-b-0", - className, - )} - {...props} - /> -)); -TableFooter.displayName = "TableFooter"; - -const TableRow = React.forwardRef>( - ({ className, ...props }, ref) => ( - ) { + return ( + tr]:last:border-b-0", + className )} {...props} /> - ), -); -TableRow.displayName = "TableRow"; + ) +} -const TableHead = React.forwardRef< - HTMLTableCellElement, - React.ThHTMLAttributes ->(({ className, ...props }, ref) => ( - + ) +} -const TableCell = React.forwardRef< - HTMLTableCellElement, - React.TdHTMLAttributes ->(({ className, ...props }, ref) => ( -
[role=checkbox]]:translate-y-0.5", - className, - )} - {...props} - /> -)); -TableHead.displayName = "TableHead"; +function TableRow({ className, ...props }: React.ComponentProps<"tr">) { + return ( +
[role=checkbox]]:translate-y-0.5", - className, - )} - {...props} - /> -)); -TableCell.displayName = "TableCell"; +function TableHead({ className, ...props }: React.ComponentProps<"th">) { + return ( + [role=checkbox]]:translate-y-[2px]", + className + )} + {...props} + /> + ) +} -const TableCaption = React.forwardRef< - HTMLTableCaptionElement, - React.HTMLAttributes ->(({ className, ...props }, ref) => ( -
-)); -TableCaption.displayName = "TableCaption"; +function TableCell({ className, ...props }: React.ComponentProps<"td">) { + return ( +
[role=checkbox]]:translate-y-[2px]", + className + )} + {...props} + /> + ) +} -export { Table, TableBody, TableCaption, TableCell, TableFooter, TableHead, TableHeader, TableRow }; +function TableCaption({ + className, + ...props +}: React.ComponentProps<"caption">) { + return ( +
+ ) +} + +export { + Table, + TableHeader, + TableBody, + TableFooter, + TableHead, + TableRow, + TableCell, + TableCaption, +} diff --git a/surfsense_web/components/ui/textarea.tsx b/surfsense_web/components/ui/textarea.tsx new file mode 100644 index 0000000..7f21b5e --- /dev/null +++ b/surfsense_web/components/ui/textarea.tsx @@ -0,0 +1,18 @@ +import * as React from "react" + +import { cn } from "@/lib/utils" + +function Textarea({ className, ...props }: React.ComponentProps<"textarea">) { + return ( +