diff --git a/surfsense_backend/app/agents/researcher/configuration.py b/surfsense_backend/app/agents/researcher/configuration.py index b6d49bd..a45aa9b 100644 --- a/surfsense_backend/app/agents/researcher/configuration.py +++ b/surfsense_backend/app/agents/researcher/configuration.py @@ -33,6 +33,7 @@ class Configuration: search_space_id: int search_mode: SearchMode research_mode: ResearchMode + document_ids_to_add_in_context: List[int] @classmethod def from_runnable_config( diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index e73aefa..165efcb 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -21,6 +21,237 @@ from app.utils.query_service import QueryService from langgraph.types import StreamWriter +# Additional imports for document fetching +from sqlalchemy.future import select +from app.db import Document, SearchSpace + + +async def fetch_documents_by_ids( + document_ids: List[int], + user_id: str, + db_session: AsyncSession +) -> tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + """ + Fetch documents by their IDs with ownership check using DOCUMENTS mode approach. + + This function ensures that only documents belonging to the user are fetched, + providing security by checking ownership through SearchSpace association. + Similar to SearchMode.DOCUMENTS, it fetches full documents and concatenates their chunks. + Also creates source objects for UI display, grouped by document type. + + Args: + document_ids: List of document IDs to fetch + user_id: The user ID to check ownership + db_session: The database session + + Returns: + Tuple of (source_objects, document_chunks) - similar to ConnectorService pattern + """ + if not document_ids: + return [], [] + + try: + # Query documents with ownership check + result = await db_session.execute( + select(Document) + .join(SearchSpace) + .filter( + Document.id.in_(document_ids), + SearchSpace.user_id == user_id + ) + ) + documents = result.scalars().all() + + # Group documents by type for source object creation + documents_by_type = {} + formatted_documents = [] + + for doc in documents: + # Fetch associated chunks for this document (similar to DocumentHybridSearchRetriever) + from app.db import Chunk + chunks_query = select(Chunk).where(Chunk.document_id == doc.id).order_by(Chunk.id) + chunks_result = await db_session.execute(chunks_query) + chunks = chunks_result.scalars().all() + + # Concatenate chunks content (similar to SearchMode.DOCUMENTS approach) + concatenated_chunks_content = " ".join([chunk.content for chunk in chunks]) if chunks else doc.content + + # Format to match connector service return format + formatted_doc = { + "chunk_id": f"user_doc_{doc.id}", + "content": concatenated_chunks_content, # Use concatenated content like DOCUMENTS mode + "score": 0.5, # High score since user explicitly selected these + "document": { + "id": doc.id, + "title": doc.title, + "document_type": doc.document_type.value if doc.document_type else "UNKNOWN", + "metadata": doc.document_metadata or {}, + }, + "source": doc.document_type.value if doc.document_type else "UNKNOWN" + } + formatted_documents.append(formatted_doc) + + # Group by document type for source objects + doc_type = doc.document_type.value if doc.document_type else "UNKNOWN" + if doc_type not in documents_by_type: + documents_by_type[doc_type] = [] + documents_by_type[doc_type].append(doc) + + # Create source objects for each document type (similar to ConnectorService) + source_objects = [] + connector_id_counter = 100 # Start from 100 to avoid conflicts with regular connectors + + for doc_type, docs in documents_by_type.items(): + sources_list = [] + + for doc in docs: + metadata = doc.document_metadata or {} + + # Create type-specific source formatting (similar to ConnectorService) + if doc_type == "LINEAR_CONNECTOR": + # Extract Linear-specific metadata + issue_identifier = metadata.get('issue_identifier', '') + issue_title = metadata.get('issue_title', doc.title) + issue_state = metadata.get('state', '') + comment_count = metadata.get('comment_count', 0) + + # Create a more descriptive title for Linear issues + title = f"Linear: {issue_identifier} - {issue_title}" if issue_identifier else f"Linear: {issue_title}" + if issue_state: + title += f" ({issue_state})" + + # Create description + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + if comment_count: + description += f" | Comments: {comment_count}" + + # Create URL + url = f"https://linear.app/issue/{issue_identifier}" if issue_identifier else "" + + elif doc_type == "SLACK_CONNECTOR": + # Extract Slack-specific metadata + channel_name = metadata.get('channel_name', 'Unknown Channel') + channel_id = metadata.get('channel_id', '') + message_date = metadata.get('start_date', '') + + title = f"Slack: {channel_name}" + if message_date: + title += f" ({message_date})" + + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + url = f"https://slack.com/app_redirect?channel={channel_id}" if channel_id else "" + + elif doc_type == "NOTION_CONNECTOR": + # Extract Notion-specific metadata + page_title = metadata.get('page_title', doc.title) + page_id = metadata.get('page_id', '') + + title = f"Notion: {page_title}" + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + url = f"https://notion.so/{page_id.replace('-', '')}" if page_id else "" + + elif doc_type == "GITHUB_CONNECTOR": + title = f"GitHub: {doc.title}" + description = metadata.get('description', doc.content[:100] + "..." if len(doc.content) > 100 else doc.content) + url = metadata.get('url', '') + + elif doc_type == "YOUTUBE_VIDEO": + # Extract YouTube-specific metadata + video_title = metadata.get('video_title', doc.title) + video_id = metadata.get('video_id', '') + channel_name = metadata.get('channel_name', '') + + title = video_title + if channel_name: + title += f" - {channel_name}" + + description = metadata.get('description', doc.content[:100] + "..." if len(doc.content) > 100 else doc.content) + url = f"https://www.youtube.com/watch?v={video_id}" if video_id else "" + + elif doc_type == "DISCORD_CONNECTOR": + # Extract Discord-specific metadata + channel_name = metadata.get('channel_name', 'Unknown Channel') + channel_id = metadata.get('channel_id', '') + guild_id = metadata.get('guild_id', '') + message_date = metadata.get('start_date', '') + + title = f"Discord: {channel_name}" + if message_date: + title += f" ({message_date})" + + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + + if guild_id and channel_id: + url = f"https://discord.com/channels/{guild_id}/{channel_id}" + elif channel_id: + url = f"https://discord.com/channels/@me/{channel_id}" + else: + url = "" + + elif doc_type == "EXTENSION": + # Extract Extension-specific metadata + webpage_title = metadata.get('VisitedWebPageTitle', doc.title) + webpage_url = metadata.get('VisitedWebPageURL', '') + visit_date = metadata.get('VisitedWebPageDateWithTimeInISOString', '') + + title = webpage_title + if visit_date: + formatted_date = visit_date.split('T')[0] if 'T' in visit_date else visit_date + title += f" (visited: {formatted_date})" + + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + url = webpage_url + + elif doc_type == "CRAWLED_URL": + title = doc.title + description = metadata.get('og:description', metadata.get('ogDescription', doc.content[:100] + "..." if len(doc.content) > 100 else doc.content)) + url = metadata.get('url', '') + + else: # FILE and other types + title = doc.title + description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content + url = metadata.get('url', '') + + # Create source entry + source = { + "id": doc.id, + "title": title, + "description": description, + "url": url + } + sources_list.append(source) + + # Create source object for this document type + friendly_type_names = { + "LINEAR_CONNECTOR": "Linear Issues (Selected)", + "SLACK_CONNECTOR": "Slack (Selected)", + "NOTION_CONNECTOR": "Notion (Selected)", + "GITHUB_CONNECTOR": "GitHub (Selected)", + "YOUTUBE_VIDEO": "YouTube Videos (Selected)", + "DISCORD_CONNECTOR": "Discord (Selected)", + "EXTENSION": "Browser Extension (Selected)", + "CRAWLED_URL": "Web Pages (Selected)", + "FILE": "Files (Selected)" + } + + source_object = { + "id": connector_id_counter, + "name": friendly_type_names.get(doc_type, f"{doc_type} (Selected)"), + "type": f"USER_SELECTED_{doc_type}", + "sources": sources_list, + } + source_objects.append(source_object) + connector_id_counter += 1 + + print(f"Fetched {len(formatted_documents)} user-selected documents (with concatenated chunks) from {len(document_ids)} requested IDs") + print(f"Created {len(source_objects)} source objects for UI display") + + return source_objects, formatted_documents + + except Exception as e: + print(f"Error fetching documents by IDs: {str(e)}") + return [], [] + class Section(BaseModel): """A section in the answer outline.""" @@ -150,7 +381,8 @@ async def fetch_relevant_documents( state: State = None, top_k: int = 10, connector_service: ConnectorService = None, - search_mode: SearchMode = SearchMode.CHUNKS + search_mode: SearchMode = SearchMode.CHUNKS, + user_selected_sources: List[Dict[str, Any]] = None ) -> List[Dict[str, Any]]: """ Fetch relevant documents for research questions using the provided connectors. @@ -436,6 +668,21 @@ async def fetch_relevant_documents( deduplicated_sources = [] seen_source_keys = set() + # First add user-selected sources (if any) + if user_selected_sources: + for source_obj in user_selected_sources: + source_id = source_obj.get('id') + source_type = source_obj.get('type') + + if source_id and source_type: + source_key = f"{source_type}_{source_id}" + if source_key not in seen_source_keys: + seen_source_keys.add(source_key) + deduplicated_sources.append(source_obj) + else: + deduplicated_sources.append(source_obj) + + # Then add connector sources for source_obj in all_sources: # Use combination of source ID and type as a unique identifier # This ensures we don't accidentally deduplicate sources from different connectors @@ -453,7 +700,9 @@ async def fetch_relevant_documents( # Stream info about deduplicated sources if streaming_service and writer: - streaming_service.only_update_terminal(f"📚 Collected {len(deduplicated_sources)} unique sources across all connectors") + user_source_count = len(user_selected_sources) if user_selected_sources else 0 + connector_source_count = len(deduplicated_sources) - user_source_count + streaming_service.only_update_terminal(f"📚 Collected {len(deduplicated_sources)} total sources ({user_source_count} user-selected + {connector_source_count} from connectors)") writer({"yeild_value": streaming_service._format_annotations()}) # After all sources are collected and deduplicated, stream them @@ -576,8 +825,26 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW TOP_K = 10 relevant_documents = [] + user_selected_documents = [] + user_selected_sources = [] + async with async_session_maker() as db_session: try: + # First, fetch user-selected documents if any + if configuration.document_ids_to_add_in_context: + streaming_service.only_update_terminal(f"📋 Including {len(configuration.document_ids_to_add_in_context)} user-selected documents...") + writer({"yeild_value": streaming_service._format_annotations()}) + + user_selected_sources, user_selected_documents = await fetch_documents_by_ids( + document_ids=configuration.document_ids_to_add_in_context, + user_id=configuration.user_id, + db_session=db_session + ) + + if user_selected_documents: + streaming_service.only_update_terminal(f"✅ Successfully added {len(user_selected_documents)} user-selected documents to context") + writer({"yeild_value": streaming_service._format_annotations()}) + # Create connector service inside the db_session scope connector_service = ConnectorService(db_session, user_id=configuration.user_id) await connector_service.initialize_counter() @@ -592,7 +859,8 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW state=state, top_k=TOP_K, connector_service=connector_service, - search_mode=configuration.search_mode + search_mode=configuration.search_mode, + user_selected_sources=user_selected_sources ) except Exception as e: error_message = f"Error fetching relevant documents: {str(e)}" @@ -603,8 +871,14 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW # This allows the process to continue, but the report might lack information relevant_documents = [] + # Combine user-selected documents with connector-fetched documents + all_documents = user_selected_documents + relevant_documents + print(f"Fetched {len(relevant_documents)} relevant documents for all sections") - streaming_service.only_update_terminal(f"✨ Starting to draft {len(answer_outline.answer_outline)} sections using {len(relevant_documents)} relevant document chunks") + print(f"Added {len(user_selected_documents)} user-selected documents for all sections") + print(f"Total documents for sections: {len(all_documents)}") + + streaming_service.only_update_terminal(f"✨ Starting to draft {len(answer_outline.answer_outline)} sections using {len(all_documents)} total document chunks ({len(user_selected_documents)} user-selected + {len(relevant_documents)} connector-found)") writer({"yeild_value": streaming_service._format_annotations()}) # Create tasks to process each section in parallel with the same document set @@ -635,7 +909,7 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW user_query=configuration.user_query, user_id=configuration.user_id, search_space_id=configuration.search_space_id, - relevant_documents=relevant_documents, + relevant_documents=all_documents, # Use combined documents state=state, writer=writer, sub_section_type=sub_section_type, @@ -875,8 +1149,26 @@ async def handle_qna_workflow(state: State, config: RunnableConfig, writer: Stre TOP_K = 15 relevant_documents = [] + user_selected_documents = [] + user_selected_sources = [] + async with async_session_maker() as db_session: try: + # First, fetch user-selected documents if any + if configuration.document_ids_to_add_in_context: + streaming_service.only_update_terminal(f"📋 Including {len(configuration.document_ids_to_add_in_context)} user-selected documents...") + writer({"yeild_value": streaming_service._format_annotations()}) + + user_selected_sources, user_selected_documents = await fetch_documents_by_ids( + document_ids=configuration.document_ids_to_add_in_context, + user_id=configuration.user_id, + db_session=db_session + ) + + if user_selected_documents: + streaming_service.only_update_terminal(f"✅ Successfully added {len(user_selected_documents)} user-selected documents to context") + writer({"yeild_value": streaming_service._format_annotations()}) + # Create connector service inside the db_session scope connector_service = ConnectorService(db_session, user_id=configuration.user_id) await connector_service.initialize_counter() @@ -894,7 +1186,8 @@ async def handle_qna_workflow(state: State, config: RunnableConfig, writer: Stre state=state, top_k=TOP_K, connector_service=connector_service, - search_mode=configuration.search_mode + search_mode=configuration.search_mode, + user_selected_sources=user_selected_sources ) except Exception as e: error_message = f"Error fetching relevant documents for QNA: {str(e)}" @@ -904,15 +1197,21 @@ async def handle_qna_workflow(state: State, config: RunnableConfig, writer: Stre # Continue with empty documents - the QNA agent will handle this gracefully relevant_documents = [] + # Combine user-selected documents with connector-fetched documents + all_documents = user_selected_documents + relevant_documents + print(f"Fetched {len(relevant_documents)} relevant documents for QNA") - streaming_service.only_update_terminal(f"🧠 Generating comprehensive answer using {len(relevant_documents)} relevant sources...") + print(f"Added {len(user_selected_documents)} user-selected documents for QNA") + print(f"Total documents for QNA: {len(all_documents)}") + + streaming_service.only_update_terminal(f"🧠 Generating comprehensive answer using {len(all_documents)} total sources ({len(user_selected_documents)} user-selected + {len(relevant_documents)} connector-found)...") writer({"yeild_value": streaming_service._format_annotations()}) # Prepare configuration for the QNA agent qna_config = { "configurable": { "user_query": reformulated_query, # Use the reformulated query - "relevant_documents": relevant_documents, + "relevant_documents": all_documents, # Use combined documents "user_id": configuration.user_id, "search_space_id": configuration.search_space_id } diff --git a/surfsense_backend/app/routes/chats_routes.py b/surfsense_backend/app/routes/chats_routes.py index 9a2aa79..ff80b7a 100644 --- a/surfsense_backend/app/routes/chats_routes.py +++ b/surfsense_backend/app/routes/chats_routes.py @@ -30,6 +30,7 @@ async def handle_chat_data( search_space_id = request.data.get('search_space_id') research_mode: str = request.data.get('research_mode') selected_connectors: List[str] = request.data.get('selected_connectors') + document_ids_to_add_in_context: List[int] = request.data.get('document_ids_to_add_in_context') search_mode_str = request.data.get('search_mode', "CHUNKS") @@ -71,7 +72,8 @@ async def handle_chat_data( research_mode, selected_connectors, langchain_chat_history, - search_mode_str + search_mode_str, + document_ids_to_add_in_context )) response.headers['x-vercel-ai-data-stream'] = 'v1' return response diff --git a/surfsense_backend/app/tasks/stream_connector_search_results.py b/surfsense_backend/app/tasks/stream_connector_search_results.py index 3a182ae..441ce1e 100644 --- a/surfsense_backend/app/tasks/stream_connector_search_results.py +++ b/surfsense_backend/app/tasks/stream_connector_search_results.py @@ -17,7 +17,8 @@ async def stream_connector_search_results( research_mode: str, selected_connectors: List[str], langchain_chat_history: List[Any], - search_mode_str: str + search_mode_str: str, + document_ids_to_add_in_context: List[int] ) -> AsyncGenerator[str, None]: """ Stream connector search results to the client @@ -62,7 +63,8 @@ async def stream_connector_search_results( "user_id": user_id_str, "search_space_id": search_space_id, "search_mode": search_mode, - "research_mode": research_mode + "research_mode": research_mode, + "document_ids_to_add_in_context": document_ids_to_add_in_context } } # Initialize state with database session and streaming service diff --git a/surfsense_backend/app/utils/connector_service.py b/surfsense_backend/app/utils/connector_service.py index 1052939..f53fd4d 100644 --- a/surfsense_backend/app/utils/connector_service.py +++ b/surfsense_backend/app/utils/connector_service.py @@ -4,7 +4,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever -from app.db import SearchSourceConnector, SearchSourceConnectorType, Chunk, Document +from app.db import SearchSourceConnector, SearchSourceConnectorType, Chunk, Document, SearchSpace from tavily import TavilyClient from linkup import LinkupClient from sqlalchemy import func @@ -33,7 +33,8 @@ class ConnectorService: result = await self.session.execute( select(func.count(Chunk.id)) .join(Document) - .filter(Document.user_id == self.user_id) + .join(SearchSpace) + .filter(SearchSpace.user_id == self.user_id) ) chunk_count = result.scalar() or 0 self.source_id_counter = chunk_count + 1 diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index fb3ecb8..8986444 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -72,7 +72,7 @@ export default function EditConnectorPage() { const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); - console.log("connector", connector); + // console.log("connector", connector); // Initialize the form const form = useForm({ resolver: zodResolver(apiConnectorFormSchema), diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 82ac0ed..7d9aa3c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -283,8 +283,8 @@ export default function DocumentsTable() { const searchSpaceId = Number(params.search_space_id); const { documents, loading, error, refreshDocuments, deleteDocument } = useDocuments(searchSpaceId); - console.log("Search Space ID:", searchSpaceId); - console.log("Documents loaded:", documents?.length); + // console.log("Search Space ID:", searchSpaceId); + // console.log("Documents loaded:", documents?.length); useEffect(() => { console.log("Delete document function available:", !!deleteDocument); @@ -315,7 +315,7 @@ export default function DocumentsTable() { const handleDeleteRows = async () => { const selectedRows = table.getSelectedRowModel().rows; - console.log("Deleting selected rows:", selectedRows.length); + // console.log("Deleting selected rows:", selectedRows.length); if (selectedRows.length === 0) { toast.error("No rows selected"); @@ -324,14 +324,14 @@ export default function DocumentsTable() { // Create an array of promises for each delete operation const deletePromises = selectedRows.map(row => { - console.log("Deleting row with ID:", row.original.id); + // console.log("Deleting row with ID:", row.original.id); return deleteDocument(row.original.id); }); try { // Execute all delete operations const results = await Promise.all(deletePromises); - console.log("Delete results:", results); + // console.log("Delete results:", results); // Check if all deletions were successful const allSuccessful = results.every(result => result === true); diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx index 6a547a6..f6d0993 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx @@ -15,8 +15,14 @@ import { Database, SendHorizontal, FileText, - Grid3x3 + Grid3x3, + File, + Globe, + Webhook, + FolderOpen, + Upload } from 'lucide-react'; +import { IconBrandDiscord, IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconLayoutKanban } from "@tabler/icons-react"; import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card'; import { Button } from '@/components/ui/button'; import { Input } from '@/components/ui/input'; @@ -47,6 +53,7 @@ import { import { MarkdownViewer } from '@/components/markdown-viewer'; import { Logo } from '@/components/Logo'; import { useSearchSourceConnectors } from '@/hooks'; +import { useDocuments } from '@/hooks/use-documents'; interface SourceItem { id: number; @@ -63,6 +70,31 @@ interface ConnectorSource { sources: SourceItem[]; } +type DocumentType = "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO" | "GITHUB_CONNECTOR" | "LINEAR_CONNECTOR" | "DISCORD_CONNECTOR"; + +interface Document { + id: number; + title: string; + document_type: DocumentType; + document_metadata: any; + content: string; + created_at: string; + search_space_id: number; +} + +// Document type icons mapping +const documentTypeIcons = { + EXTENSION: Webhook, + CRAWLED_URL: Globe, + SLACK_CONNECTOR: IconBrandSlack, + NOTION_CONNECTOR: IconBrandNotion, + FILE: File, + YOUTUBE_VIDEO: IconBrandYoutube, + GITHUB_CONNECTOR: IconBrandGithub, + LINEAR_CONNECTOR: IconLayoutKanban, + DISCORD_CONNECTOR: IconBrandDiscord, +} as const; + /** * Button that displays selected connectors and opens connector selection dialog */ @@ -78,6 +110,41 @@ const ConnectorButton = ({ selectedConnectors, onClick }: { selectedConnectors: ); }; +/** + * Button that displays selected documents count and opens document selection dialog + */ +const DocumentSelectorButton = ({ + selectedDocuments, + onClick, + documentsCount +}: { + selectedDocuments: number[], + onClick: () => void, + documentsCount: number +}) => { + return ( +
+ + {selectedDocuments.length > 0 && ( + + {selectedDocuments.length > 99 ? '99+' : selectedDocuments.length} + + )} + {selectedDocuments.length === 0 && ( + + 0 + + )} +
+ ); +}; + // Create a wrapper component for the sources dialog content const SourcesDialogContent = ({ connector, @@ -245,7 +312,7 @@ const ChatPage = () => { const [sourceFilter, setSourceFilter] = useState(""); const tabsListRef = useRef(null); const [terminalExpanded, setTerminalExpanded] = useState(false); - const [selectedConnectors, setSelectedConnectors] = useState(["CRAWLED_URL"]); + const [selectedConnectors, setSelectedConnectors] = useState([]); const [searchMode, setSearchMode] = useState<'DOCUMENTS' | 'CHUNKS'>('DOCUMENTS'); const [researchMode, setResearchMode] = useState("QNA"); const [currentTime, setCurrentTime] = useState(''); @@ -256,6 +323,11 @@ const ChatPage = () => { const INITIAL_SOURCES_DISPLAY = 3; const { search_space_id, chat_id } = useParams(); + + // Document selection state + const [selectedDocuments, setSelectedDocuments] = useState([]); + const [documentFilter, setDocumentFilter] = useState(""); + const { documents, loading: isLoadingDocuments, error: documentsError } = useDocuments(Number(search_space_id)); // Function to scroll terminal to bottom const scrollTerminalToBottom = () => { @@ -342,6 +414,13 @@ const ChatPage = () => { background-color: rgba(155, 155, 155, 0.5); border-radius: 20px; } + /* Line clamp utility */ + .line-clamp-2 { + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; + } `; document.head.appendChild(style); @@ -362,7 +441,8 @@ const ChatPage = () => { search_space_id: search_space_id, selected_connectors: selectedConnectors, research_mode: researchMode, - search_mode: searchMode + search_mode: searchMode, + document_ids_to_add_in_context: selectedDocuments } }, onError: (error) => { @@ -377,7 +457,7 @@ const ChatPage = () => { try { if (!token) return; // Wait for token to be set - console.log('Fetching chat details for chat ID:', chat_id); + // console.log('Fetching chat details for chat ID:', chat_id); const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/${Number(chat_id)}`, { method: 'GET', @@ -392,7 +472,7 @@ const ChatPage = () => { } const chatData = await response.json(); - console.log('Chat details fetched:', chatData); + // console.log('Chat details fetched:', chatData); // Set research mode from chat data if (chatData.type) { @@ -442,7 +522,7 @@ const ChatPage = () => { const title = userMessages[0].content; - console.log('Updating chat with title:', title); + // console.log('Updating chat with title:', title); // Update the chat const response = await fetch(`${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/chats/${Number(chat_id)}`, { @@ -464,7 +544,7 @@ const ChatPage = () => { throw new Error(`Failed to update chat: ${response.statusText}`); } - console.log('Chat updated successfully'); + // console.log('Chat updated successfully'); } } catch (err) { console.error('Error updating chat:', err); @@ -519,10 +599,9 @@ const ChatPage = () => { if (!input.trim() || status !== 'ready') return; - // You can add additional logic here if needed - // For example, validation for selected connectors - if (selectedConnectors.length === 0) { - alert("Please select at least one connector"); + // Validation: require at least one connector OR at least one document + if (selectedConnectors.length === 0 && selectedDocuments.length === 0) { + alert("Please select at least one connector or document"); return; } @@ -988,17 +1067,162 @@ const ChatPage = () => {
-
- {/* Connector Selection Dialog */} - - -
- { }} - /> +
+ {/* Document Selection Dialog */} + + + { }} + documentsCount={documents?.length || 0} + /> + + + + + Select Documents + + + + Choose documents to include in your research context + + + + {/* Document Search */} +
+ + setDocumentFilter(e.target.value)} + /> + {documentFilter && ( + + )} +
+ + {/* Document List */} +
+ {isLoadingDocuments ? ( +
+ +
+ ) : documentsError ? ( +
+

Error loading documents

+
+ ) : ( + (() => { + const filteredDocuments = documents?.filter(doc => + doc.title.toLowerCase().includes(documentFilter.toLowerCase()) + ) || []; + + if (filteredDocuments.length === 0) { + return ( +
+ +

{documentFilter ? `No documents found matching "${documentFilter}"` : 'No documents available'}

+
+ ); + } + + return filteredDocuments.map((document) => { + const Icon = documentTypeIcons[document.document_type]; + const isSelected = selectedDocuments.includes(document.id); + + return ( +
{ + setSelectedDocuments(prev => + isSelected + ? prev.filter(id => id !== document.id) + : [...prev, document.id] + ); + }} + > +
+ +
+
+

{document.title}

+

+ {document.document_type.replace(/_/g, ' ').toLowerCase()} + {' • '} + {new Date(document.created_at).toLocaleDateString()} +

+

+ {document.content.substring(0, 150)}... +

+
+ {isSelected && ( +
+ +
+ )} +
+ ); + }); + })() + )} +
+ + +
+ {selectedDocuments.length} document{selectedDocuments.length !== 1 ? 's' : ''} selected
- +
+ + +
+ + + + + {/* Connector Selection Dialog */} + + + { }} + /> + Select Connectors @@ -1065,33 +1289,31 @@ const ChatPage = () => { {/* Search Mode Control */} -
- - + + + Chunks +
{/* Research Mode Control */} -
+
(`api/v1/chats/?limit=5&skip=0&search_space_id=${searchSpaceId}`); + // Sort chats by created_at in descending order (newest first) + const sortedChats = chats.sort((a, b) => + new Date(b.created_at).getTime() - new Date(a.created_at).getTime() + ); + // console.log("sortedChats", sortedChats); // Transform API response to the format expected by AppSidebar - const formattedChats = chats.map(chat => ({ + const formattedChats = sortedChats.map(chat => ({ name: chat.title || `Chat ${chat.id}`, // Fallback if title is empty url: `/dashboard/${chat.search_space_id}/researcher/${chat.id}`, icon: 'MessageCircleMore', diff --git a/surfsense_web/components/sidebar/app-sidebar.tsx b/surfsense_web/components/sidebar/app-sidebar.tsx index 9206d63..62f8cf6 100644 --- a/surfsense_web/components/sidebar/app-sidebar.tsx +++ b/surfsense_web/components/sidebar/app-sidebar.tsx @@ -227,7 +227,7 @@ export function AppSidebar({ - {processedRecentChats.length > 0 && } + {processedRecentChats.length > 0 && } diff --git a/surfsense_web/components/sidebar/nav-projects.tsx b/surfsense_web/components/sidebar/nav-projects.tsx index a4cef64..756ca80 100644 --- a/surfsense_web/components/sidebar/nav-projects.tsx +++ b/surfsense_web/components/sidebar/nav-projects.tsx @@ -43,9 +43,9 @@ interface ChatAction { } export function NavProjects({ - projects, + chats, }: { - projects: { + chats: { name: string url: string icon: LucideIcon @@ -57,13 +57,13 @@ export function NavProjects({ const { isMobile } = useSidebar() const router = useRouter() - const searchSpaceId = projects[0]?.search_space_id || "" + const searchSpaceId = chats[0]?.search_space_id || "" return ( Recent Chats - {projects.map((item, index) => ( + {chats.map((item, index) => (