diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 4c3bc72..902dbe3 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -14,6 +14,8 @@ from .configuration import Configuration from .prompts import get_answer_outline_system_prompt from .state import State from .sub_section_writer.graph import graph as sub_section_writer_graph +from .sub_section_writer.configuration import SubSectionType + from langgraph.types import StreamWriter @@ -41,14 +43,14 @@ async def write_answer_outline(state: State, config: RunnableConfig, writer: Str """ streaming_service = state.streaming_service - streaming_service.only_update_terminal("Generating answer outline...") + streaming_service.only_update_terminal("๐Ÿ” Generating answer outline...") writer({"yeild_value": streaming_service._format_annotations()}) # Get configuration from runnable config configuration = Configuration.from_runnable_config(config) user_query = configuration.user_query num_sections = configuration.num_sections - streaming_service.only_update_terminal(f"Planning research approach for query: {user_query[:100]}...") + streaming_service.only_update_terminal(f"๐Ÿค” Planning research approach for: \"{user_query[:100]}...\"") writer({"yeild_value": streaming_service._format_annotations()}) # Initialize LLM @@ -78,7 +80,7 @@ async def write_answer_outline(state: State, config: RunnableConfig, writer: Str Your output MUST be valid JSON in exactly this format. Do not include any other text or explanation. """ - streaming_service.only_update_terminal("Designing structured outline with AI...") + streaming_service.only_update_terminal("๐Ÿ“ Designing structured outline with AI...") writer({"yeild_value": streaming_service._format_annotations()}) # Create messages for the LLM @@ -88,7 +90,7 @@ async def write_answer_outline(state: State, config: RunnableConfig, writer: Str ] # Call the LLM directly without using structured output - streaming_service.only_update_terminal("Processing answer structure...") + streaming_service.only_update_terminal("โš™๏ธ Processing answer structure...") writer({"yeild_value": streaming_service._format_annotations()}) response = await llm.ainvoke(messages) @@ -111,7 +113,7 @@ async def write_answer_outline(state: State, config: RunnableConfig, writer: Str answer_outline = AnswerOutline(**parsed_data) total_questions = sum(len(section.questions) for section in answer_outline.answer_outline) - streaming_service.only_update_terminal(f"Successfully generated outline with {len(answer_outline.answer_outline)} sections and {total_questions} research questions") + streaming_service.only_update_terminal(f"โœ… Successfully generated outline with {len(answer_outline.answer_outline)} sections and {total_questions} research questions!") writer({"yeild_value": streaming_service._format_annotations()}) print(f"Successfully generated answer outline with {len(answer_outline.answer_outline)} sections") @@ -121,14 +123,14 @@ async def write_answer_outline(state: State, config: RunnableConfig, writer: Str else: # If JSON structure not found, raise a clear error error_message = f"Could not find valid JSON in LLM response. Raw response: {content}" - streaming_service.only_update_terminal(error_message, "error") + streaming_service.only_update_terminal(f"โŒ {error_message}", "error") writer({"yeild_value": streaming_service._format_annotations()}) raise ValueError(error_message) except (json.JSONDecodeError, ValueError) as e: # Log the error and re-raise it error_message = f"Error parsing LLM response: {str(e)}" - streaming_service.only_update_terminal(error_message, "error") + streaming_service.only_update_terminal(f"โŒ {error_message}", "error") writer({"yeild_value": streaming_service._format_annotations()}) print(f"Error parsing LLM response: {str(e)}") @@ -149,6 +151,11 @@ async def fetch_relevant_documents( """ Fetch relevant documents for research questions using the provided connectors. + This function searches across multiple data sources for information related to the + research questions. It provides user-friendly feedback during the search process by + displaying connector names (like "Web Search" instead of "TAVILY_API") and adding + relevant emojis to indicate the type of source being searched. + Args: research_questions: List of research questions to find documents for user_id: The user ID @@ -158,6 +165,7 @@ async def fetch_relevant_documents( writer: StreamWriter for sending progress updates state: The current state containing the streaming service top_k: Number of top results to retrieve per connector per question + connector_service: An initialized connector service to use for searching Returns: List of relevant documents @@ -170,7 +178,9 @@ async def fetch_relevant_documents( # Stream initial status update if streaming_service and writer: - streaming_service.only_update_terminal(f"Starting research on {len(research_questions)} questions using {len(connectors_to_search)} connectors...") + connector_names = [get_connector_friendly_name(connector) for connector in connectors_to_search] + connector_names_str = ", ".join(connector_names) + streaming_service.only_update_terminal(f"๐Ÿ”Ž Starting research on {len(research_questions)} questions using {connector_names_str} data sources") writer({"yeild_value": streaming_service._format_annotations()}) all_raw_documents = [] # Store all raw documents @@ -179,7 +189,7 @@ async def fetch_relevant_documents( for i, user_query in enumerate(research_questions): # Stream question being researched if streaming_service and writer: - streaming_service.only_update_terminal(f"Researching question {i+1}/{len(research_questions)}: {user_query[:100]}...") + streaming_service.only_update_terminal(f"๐Ÿง  Researching question {i+1}/{len(research_questions)}: \"{user_query[:100]}...\"") writer({"yeild_value": streaming_service._format_annotations()}) # Use original research question as the query @@ -189,7 +199,9 @@ async def fetch_relevant_documents( for connector in connectors_to_search: # Stream connector being searched if streaming_service and writer: - streaming_service.only_update_terminal(f"Searching {connector} for relevant information...") + connector_emoji = get_connector_emoji(connector) + friendly_name = get_connector_friendly_name(connector) + streaming_service.only_update_terminal(f"{connector_emoji} Searching {friendly_name} for relevant information...") writer({"yeild_value": streaming_service._format_annotations()}) try: @@ -208,7 +220,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(youtube_chunks)} YouTube chunks relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ“น Found {len(youtube_chunks)} YouTube chunks related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "EXTENSION": @@ -226,7 +238,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(extension_chunks)} extension chunks relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿงฉ Found {len(extension_chunks)} Browser Extension chunks related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "CRAWLED_URL": @@ -244,7 +256,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(crawled_urls_chunks)} crawled URL chunks relevant to the query") + streaming_service.only_update_terminal(f"๐ŸŒ Found {len(crawled_urls_chunks)} Web Pages chunks related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "FILE": @@ -262,7 +274,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(files_chunks)} file chunks relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ“„ Found {len(files_chunks)} Files chunks related to your query") writer({"yeild_value": streaming_service._format_annotations()}) @@ -281,7 +293,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(slack_chunks)} Slack messages relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ’ฌ Found {len(slack_chunks)} Slack messages related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "NOTION_CONNECTOR": @@ -299,7 +311,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(notion_chunks)} Notion pages/blocks relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ“˜ Found {len(notion_chunks)} Notion pages/blocks related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "GITHUB_CONNECTOR": @@ -317,7 +329,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(github_chunks)} GitHub files/issues relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ™ Found {len(github_chunks)} GitHub files/issues related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "LINEAR_CONNECTOR": @@ -335,7 +347,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(linear_chunks)} Linear issues relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ“Š Found {len(linear_chunks)} Linear issues related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "TAVILY_API": @@ -352,7 +364,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(tavily_chunks)} web search results relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ” Found {len(tavily_chunks)} Web Search results related to your query") writer({"yeild_value": streaming_service._format_annotations()}) elif connector == "LINKUP_API": @@ -374,7 +386,7 @@ async def fetch_relevant_documents( # Stream found document count if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(linkup_chunks)} Linkup chunks relevant to the query") + streaming_service.only_update_terminal(f"๐Ÿ”— Found {len(linkup_chunks)} Linkup results related to your query") writer({"yeild_value": streaming_service._format_annotations()}) @@ -384,7 +396,8 @@ async def fetch_relevant_documents( # Stream error message if streaming_service and writer: - streaming_service.only_update_terminal(error_message, "error") + friendly_name = get_connector_friendly_name(connector) + streaming_service.only_update_terminal(f"โš ๏ธ Error searching {friendly_name}: {str(e)}", "error") writer({"yeild_value": streaming_service._format_annotations()}) # Continue with other connectors on error @@ -411,7 +424,7 @@ async def fetch_relevant_documents( # Stream info about deduplicated sources if streaming_service and writer: - streaming_service.only_update_terminal(f"Collected {len(deduplicated_sources)} unique sources across all connectors") + streaming_service.only_update_terminal(f"๐Ÿ“š Collected {len(deduplicated_sources)} unique sources across all connectors") writer({"yeild_value": streaming_service._format_annotations()}) # After all sources are collected and deduplicated, stream them @@ -441,12 +454,44 @@ async def fetch_relevant_documents( # Stream info about deduplicated documents if streaming_service and writer: - streaming_service.only_update_terminal(f"Found {len(deduplicated_docs)} unique document chunks after deduplication") + streaming_service.only_update_terminal(f"๐Ÿงน Found {len(deduplicated_docs)} unique document chunks after removing duplicates") writer({"yeild_value": streaming_service._format_annotations()}) # Return deduplicated documents return deduplicated_docs +def get_connector_emoji(connector_name: str) -> str: + """Get an appropriate emoji for a connector type.""" + connector_emojis = { + "YOUTUBE_VIDEO": "๐Ÿ“น", + "EXTENSION": "๐Ÿงฉ", + "CRAWLED_URL": "๐ŸŒ", + "FILE": "๐Ÿ“„", + "SLACK_CONNECTOR": "๐Ÿ’ฌ", + "NOTION_CONNECTOR": "๐Ÿ“˜", + "GITHUB_CONNECTOR": "๐Ÿ™", + "LINEAR_CONNECTOR": "๐Ÿ“Š", + "TAVILY_API": "๐Ÿ”", + "LINKUP_API": "๐Ÿ”—" + } + return connector_emojis.get(connector_name, "๐Ÿ”Ž") + +def get_connector_friendly_name(connector_name: str) -> str: + """Convert technical connector IDs to user-friendly names.""" + connector_friendly_names = { + "YOUTUBE_VIDEO": "YouTube", + "EXTENSION": "Browser Extension", + "CRAWLED_URL": "Web Pages", + "FILE": "Files", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + "LINEAR_CONNECTOR": "Linear", + "TAVILY_API": "Tavily Search", + "LINKUP_API": "Linkup Search" + } + return connector_friendly_names.get(connector_name, connector_name) + async def process_sections(state: State, config: RunnableConfig, writer: StreamWriter) -> Dict[str, Any]: """ Process all sections in parallel and combine the results. @@ -463,13 +508,13 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW answer_outline = state.answer_outline streaming_service = state.streaming_service - streaming_service.only_update_terminal(f"Starting to process research sections...") + streaming_service.only_update_terminal(f"๐Ÿš€ Starting to process research sections...") writer({"yeild_value": streaming_service._format_annotations()}) print(f"Processing sections from outline: {answer_outline is not None}") if not answer_outline: - streaming_service.only_update_terminal("Error: No answer outline was provided. Cannot generate report.", "error") + streaming_service.only_update_terminal("โŒ Error: No answer outline was provided. Cannot generate report.", "error") writer({"yeild_value": streaming_service._format_annotations()}) return { "final_written_report": "No answer outline was provided. Cannot generate final report." @@ -481,11 +526,11 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW all_questions.extend(section.questions) print(f"Collected {len(all_questions)} questions from all sections") - streaming_service.only_update_terminal(f"Found {len(all_questions)} research questions across {len(answer_outline.answer_outline)} sections") + streaming_service.only_update_terminal(f"๐Ÿงฉ Found {len(all_questions)} research questions across {len(answer_outline.answer_outline)} sections") writer({"yeild_value": streaming_service._format_annotations()}) # Fetch relevant documents once for all questions - streaming_service.only_update_terminal("Searching for relevant information across all connectors...") + streaming_service.only_update_terminal("๐Ÿ” Searching for relevant information across all connectors...") writer({"yeild_value": streaming_service._format_annotations()}) if configuration.num_sections == 1: @@ -515,7 +560,7 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW except Exception as e: error_message = f"Error fetching relevant documents: {str(e)}" print(error_message) - streaming_service.only_update_terminal(error_message, "error") + streaming_service.only_update_terminal(f"โŒ {error_message}", "error") writer({"yeild_value": streaming_service._format_annotations()}) # Log the error and continue with an empty list of documents # This allows the process to continue, but the report might lack information @@ -523,15 +568,22 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW # Consider adding more robust error handling or reporting if needed print(f"Fetched {len(relevant_documents)} relevant documents for all sections") - streaming_service.only_update_terminal(f"Starting to draft {len(answer_outline.answer_outline)} sections using {len(relevant_documents)} relevant document chunks") + streaming_service.only_update_terminal(f"โœจ Starting to draft {len(answer_outline.answer_outline)} sections using {len(relevant_documents)} relevant document chunks") writer({"yeild_value": streaming_service._format_annotations()}) # Create tasks to process each section in parallel with the same document set section_tasks = [] - streaming_service.only_update_terminal("Creating processing tasks for each section...") + streaming_service.only_update_terminal("โš™๏ธ Creating processing tasks for each section...") writer({"yeild_value": streaming_service._format_annotations()}) - for section in answer_outline.answer_outline: + for i, section in enumerate(answer_outline.answer_outline): + if i == 0: + sub_section_type = SubSectionType.START + elif i == len(answer_outline.answer_outline) - 1: + sub_section_type = SubSectionType.END + else: + sub_section_type = SubSectionType.MIDDLE + section_tasks.append( process_section_with_documents( section_title=section.section_title, @@ -541,19 +593,20 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW search_space_id=configuration.search_space_id, relevant_documents=relevant_documents, state=state, - writer=writer + writer=writer, + sub_section_type=sub_section_type ) ) # Run all section processing tasks in parallel print(f"Running {len(section_tasks)} section processing tasks in parallel") - streaming_service.only_update_terminal(f"Processing {len(section_tasks)} sections simultaneously...") + streaming_service.only_update_terminal(f"โณ Processing {len(section_tasks)} sections simultaneously...") writer({"yeild_value": streaming_service._format_annotations()}) section_results = await asyncio.gather(*section_tasks, return_exceptions=True) # Handle any exceptions in the results - streaming_service.only_update_terminal("Combining section results into final report...") + streaming_service.only_update_terminal("๐Ÿงต Combining section results into final report...") writer({"yeild_value": streaming_service._format_annotations()}) processed_results = [] @@ -562,7 +615,7 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW section_title = answer_outline.answer_outline[i].section_title error_message = f"Error processing section '{section_title}': {str(result)}" print(error_message) - streaming_service.only_update_terminal(error_message, "error") + streaming_service.only_update_terminal(f"โš ๏ธ {error_message}", "error") writer({"yeild_value": streaming_service._format_annotations()}) processed_results.append(error_message) else: @@ -580,7 +633,7 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW final_written_report = "\n".join(final_report) print(f"Generated final report with {len(final_report)} parts") - streaming_service.only_update_terminal("Final research report generated successfully!") + streaming_service.only_update_terminal("๐ŸŽ‰ Final research report generated successfully!") writer({"yeild_value": streaming_service._format_annotations()}) if hasattr(state, 'streaming_service') and state.streaming_service: @@ -612,7 +665,8 @@ async def process_section_with_documents( relevant_documents: List[Dict[str, Any]], user_query: str, state: State = None, - writer: StreamWriter = None + writer: StreamWriter = None, + sub_section_type: SubSectionType = SubSectionType.MIDDLE ) -> str: """ Process a single section using pre-fetched documents. @@ -635,14 +689,14 @@ async def process_section_with_documents( # Send status update via streaming if available if state and state.streaming_service and writer: - state.streaming_service.only_update_terminal(f"Writing section: {section_title} with {len(section_questions)} research questions") + state.streaming_service.only_update_terminal(f"๐Ÿ“ Writing section: \"{section_title}\" with {len(section_questions)} research questions") writer({"yeild_value": state.streaming_service._format_annotations()}) # Fallback if no documents found if not documents_to_use: print(f"No relevant documents found for section: {section_title}") if state and state.streaming_service and writer: - state.streaming_service.only_update_terminal(f"Warning: No relevant documents found for section: {section_title}", "warning") + state.streaming_service.only_update_terminal(f"โš ๏ธ Warning: No relevant documents found for section: \"{section_title}\"", "warning") writer({"yeild_value": state.streaming_service._format_annotations()}) documents_to_use = [ @@ -657,6 +711,7 @@ async def process_section_with_documents( "configurable": { "sub_section_title": section_title, "sub_section_questions": section_questions, + "sub_section_type": sub_section_type, "user_query": user_query, "relevant_documents": documents_to_use, "user_id": user_id, @@ -670,7 +725,7 @@ async def process_section_with_documents( # Invoke the sub-section writer graph print(f"Invoking sub_section_writer for: {section_title}") if state and state.streaming_service and writer: - state.streaming_service.only_update_terminal(f"Analyzing information and drafting content for section: {section_title}") + state.streaming_service.only_update_terminal(f"๐Ÿง  Analyzing information and drafting content for section: \"{section_title}\"") writer({"yeild_value": state.streaming_service._format_annotations()}) result = await sub_section_writer_graph.ainvoke(sub_state, config) @@ -680,7 +735,7 @@ async def process_section_with_documents( # Send section content update via streaming if available if state and state.streaming_service and writer: - state.streaming_service.only_update_terminal(f"Completed writing section: {section_title}") + state.streaming_service.only_update_terminal(f"โœ… Completed writing section: \"{section_title}\"") writer({"yeild_value": state.streaming_service._format_annotations()}) return final_answer @@ -689,7 +744,7 @@ async def process_section_with_documents( # Send error update via streaming if available if state and state.streaming_service and writer: - state.streaming_service.only_update_terminal(f"Error processing section '{section_title}': {str(e)}", "error") + state.streaming_service.only_update_terminal(f"โŒ Error processing section \"{section_title}\": {str(e)}", "error") writer({"yeild_value": state.streaming_service._format_annotations()}) return f"Error processing section: {section_title}. Details: {str(e)}" diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py index 9e1ca32..b7acf8b 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/configuration.py @@ -3,11 +3,19 @@ from __future__ import annotations from dataclasses import dataclass, fields +from enum import Enum from typing import Optional, List, Any from langchain_core.runnables import RunnableConfig +class SubSectionType(Enum): + """Enum defining the type of sub-section.""" + START = "START" + MIDDLE = "MIDDLE" + END = "END" + + @dataclass(kw_only=True) class Configuration: """The configuration for the agent.""" @@ -15,6 +23,7 @@ class Configuration: # Input parameters provided at invocation sub_section_title: str sub_section_questions: List[str] + sub_section_type: SubSectionType user_query: str relevant_documents: List[Any] # Documents provided directly to the agent user_id: str diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index f1d50ae..3cd699c 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -5,6 +5,7 @@ from typing import Any, Dict from app.config import config as app_config from .prompts import get_citation_system_prompt from langchain_core.messages import HumanMessage, SystemMessage +from .configuration import SubSectionType async def rerank_documents(state: State, config: RunnableConfig) -> Dict[str, Any]: """ @@ -122,10 +123,20 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A sub_section_questions = configuration.sub_section_questions user_query = configuration.user_query # Get the original user query documents_text = "\n".join(formatted_documents) - + sub_section_type = configuration.sub_section_type + # Format the questions as bullet points for clarity questions_text = "\n".join([f"- {question}" for question in sub_section_questions]) + # Provide more context based on the subsection type + section_position_context = "" + if sub_section_type == SubSectionType.START: + section_position_context = "This is the INTRODUCTION section. Focus on providing an overview of the topic, setting the context, and introducing key concepts that will be discussed in later sections. Do not provide any conclusions in this section, as conclusions should only appear in the final section." + elif sub_section_type == SubSectionType.MIDDLE: + section_position_context = "This is a MIDDLE section. Ensure this content flows naturally from previous sections and into subsequent ones. This could be any middle section in the document, so maintain coherence with the overall structure while addressing the specific topic of this section. Do not provide any conclusions in this section, as conclusions should only appear in the final section." + elif sub_section_type == SubSectionType.END: + section_position_context = "This is the CONCLUSION section. Focus on summarizing key points, providing closure, and possibly suggesting implications or future directions related to the topic." + # Construct a clear, structured query for the LLM human_message_content = f""" Now user's query is: @@ -137,6 +148,14 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A {section_title} + + + {section_position_context} + + + + {questions_text} + Use the provided documents as your source material and cite them properly using the IEEE citation format [X] where X is the source_id. diff --git a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx index d371e9e..bc58e8c 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx @@ -239,7 +239,6 @@ const SourcesDialogContent = ({ const ChatPage = () => { const [token, setToken] = React.useState(null); - const [activeTab, setActiveTab] = useState(""); const [dialogOpenId, setDialogOpenId] = useState(null); const [sourcesPage, setSourcesPage] = useState(1); const [expandedSources, setExpandedSources] = useState(false); @@ -252,7 +251,6 @@ const ChatPage = () => { const [researchMode, setResearchMode] = useState("GENERAL"); const [currentTime, setCurrentTime] = useState(''); const [currentDate, setCurrentDate] = useState(''); - const [connectorSources, setConnectorSources] = useState([]); const terminalMessagesRef = useRef(null); const { connectorSourceItems, isLoading: isLoadingConnectors } = useSearchSourceConnectors(); @@ -476,43 +474,10 @@ const ChatPage = () => { updateChat(); }, [messages, status, chat_id, researchMode, selectedConnectors, search_space_id]); - // Memoize connector sources to prevent excessive re-renders - const processedConnectorSources = React.useMemo(() => { - if (messages.length === 0) return connectorSources; - - // Only process when we have a complete message (not streaming) - if (status !== 'ready') return connectorSources; - - // Find the latest assistant message - const assistantMessages = messages.filter(msg => msg.role === 'assistant'); - if (assistantMessages.length === 0) return connectorSources; - - const latestAssistantMessage = assistantMessages[assistantMessages.length - 1]; - if (!latestAssistantMessage?.annotations) return connectorSources; - - // Find the latest SOURCES annotation - const annotations = latestAssistantMessage.annotations as any[]; - const sourcesAnnotations = annotations.filter(a => a.type === 'SOURCES'); - - if (sourcesAnnotations.length === 0) return connectorSources; - - const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1]; - if (!latestSourcesAnnotation.content) return connectorSources; - - // Use this content if it differs from current - return latestSourcesAnnotation.content; - }, [messages, status, connectorSources]); - - // Update connector sources when processed value changes - useEffect(() => { - if (processedConnectorSources !== connectorSources) { - setConnectorSources(processedConnectorSources); - } - }, [processedConnectorSources, connectorSources]); - // Check and scroll terminal when terminal info is available useEffect(() => { - if (messages.length === 0 || status !== 'ready') return; + // Modified to trigger during streaming as well (removed status check) + if (messages.length === 0) return; // Find the latest assistant message const assistantMessages = messages.filter(msg => msg.role === 'assistant'); @@ -526,10 +491,27 @@ const ChatPage = () => { const terminalInfoAnnotations = annotations.filter(a => a.type === 'TERMINAL_INFO'); if (terminalInfoAnnotations.length > 0) { - // Schedule scrolling after the DOM has been updated - setTimeout(scrollTerminalToBottom, 100); + // Always scroll to bottom when terminal info is updated, even during streaming + scrollTerminalToBottom(); } - }, [messages, status]); + }, [messages]); // Removed status from dependencies to ensure it triggers during streaming + + // Pure function to get connector sources for a specific message + const getMessageConnectorSources = (message: any): any[] => { + if (!message || message.role !== 'assistant' || !message.annotations) return []; + + // Find all SOURCES annotations + const annotations = message.annotations as any[]; + const sourcesAnnotations = annotations.filter(a => a.type === 'SOURCES'); + + // Get the latest SOURCES annotation + if (sourcesAnnotations.length === 0) return []; + const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1]; + + if (!latestSourcesAnnotation.content) return []; + + return latestSourcesAnnotation.content; + }; // Custom handleSubmit function to include selected connectors and answer type const handleSubmit = (e: React.FormEvent) => { @@ -561,17 +543,12 @@ const ChatPage = () => { scrollToBottom(); }, [messages]); - // Set activeTab when connectorSources change using a memoized value - const activeTabValue = React.useMemo(() => { - return connectorSources.length > 0 ? connectorSources[0].type : ""; - }, [connectorSources]); - - // Update activeTab when the memoized value changes + // Reset sources page when new messages arrive useEffect(() => { - if (activeTabValue && activeTabValue !== activeTab) { - setActiveTab(activeTabValue); - } - }, [activeTabValue, activeTab]); + // Reset pagination when we get new messages + setSourcesPage(1); + setExpandedSources(false); + }, [messages]); // Scroll terminal to bottom when expanded useEffect(() => { @@ -582,7 +559,7 @@ const ChatPage = () => { // Get total sources count for a connector type const getSourcesCount = (connectorType: string) => { - return getSourcesCountUtil(connectorSources, connectorType); + return getSourcesCountUtil(getMessageConnectorSources(messages[messages.length - 1]), connectorType); }; // Function to check scroll position and update indicators @@ -638,23 +615,14 @@ const ChatPage = () => { if (assistantMessages.length === 0) return null; const latestAssistantMessage = assistantMessages[assistantMessages.length - 1]; - if (!latestAssistantMessage?.annotations) return null; - - // Find all SOURCES annotations - const annotations = latestAssistantMessage.annotations as any[]; - const sourcesAnnotations = annotations.filter( - (annotation) => annotation.type === 'SOURCES' - ); - - // Get the latest SOURCES annotation - if (sourcesAnnotations.length === 0) return null; - const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1]; - - if (!latestSourcesAnnotation.content) return null; + + // Use our helper function to get sources + const sources = getMessageConnectorSources(latestAssistantMessage); + if (sources.length === 0) return null; // Flatten all sources from all connectors const allSources: Source[] = []; - latestSourcesAnnotation.content.forEach((connector: ConnectorSource) => { + sources.forEach((connector: ConnectorSource) => { if (connector.sources && Array.isArray(connector.sources)) { connector.sources.forEach((source: SourceItem) => { allSources.push({ @@ -675,23 +643,14 @@ const ChatPage = () => { } else { // Use the specific message by index const message = messages[messageIndex]; - if (!message || message.role !== 'assistant' || !message.annotations) return null; - - // Find all SOURCES annotations - const annotations = message.annotations as any[]; - const sourcesAnnotations = annotations.filter( - (annotation) => annotation.type === 'SOURCES' - ); - - // Get the latest SOURCES annotation - if (sourcesAnnotations.length === 0) return null; - const latestSourcesAnnotation = sourcesAnnotations[sourcesAnnotations.length - 1]; - - if (!latestSourcesAnnotation.content) return null; + + // Use our helper function to get sources + const sources = getMessageConnectorSources(message); + if (sources.length === 0) return null; // Flatten all sources from all connectors const allSources: Source[] = []; - latestSourcesAnnotation.content.forEach((connector: ConnectorSource) => { + sources.forEach((connector: ConnectorSource) => { if (connector.sources && Array.isArray(connector.sources)) { connector.sources.forEach((source: SourceItem) => { allSources.push({ @@ -712,6 +671,34 @@ const ChatPage = () => { } }, [messages]); + // Pure function for rendering terminal content - no hooks allowed here + const renderTerminalContent = (message: any) => { + if (!message.annotations) return null; + + // Get all TERMINAL_INFO annotations + const terminalInfoAnnotations = (message.annotations as any[]) + .filter(a => a.type === 'TERMINAL_INFO'); + + // Get the latest TERMINAL_INFO annotation + const latestTerminalInfo = terminalInfoAnnotations.length > 0 + ? terminalInfoAnnotations[terminalInfoAnnotations.length - 1] + : null; + + // Render the content of the latest TERMINAL_INFO annotation + return latestTerminalInfo?.content.map((item: any, idx: number) => ( +
+ [{String(idx).padStart(2, '0')}:{String(Math.floor(idx * 2)).padStart(2, '0')}] + {'>'} + {item.text} +
+ )); + }; + return ( <>
@@ -781,30 +768,9 @@ const ChatPage = () => { $ surfsense-researcher
- {message.annotations && (() => { - // Get all TERMINAL_INFO annotations - const terminalInfoAnnotations = (message.annotations as any[]) - .filter(a => a.type === 'TERMINAL_INFO'); - - // Get the latest TERMINAL_INFO annotation - const latestTerminalInfo = terminalInfoAnnotations.length > 0 - ? terminalInfoAnnotations[terminalInfoAnnotations.length - 1] - : null; - - // Render the content of the latest TERMINAL_INFO annotation - return latestTerminalInfo?.content.map((item: any, idx: number) => ( -
- [{String(idx).padStart(2, '0')}:{String(Math.floor(idx * 2)).padStart(2, '0')}] - {'>'} - {item.text} -
- )); - })()} + + {renderTerminalContent(message)} +
[00:13] researcher@surfsense @@ -836,105 +802,120 @@ const ChatPage = () => { Sources
- 0 ? connectorSources[0].type : "CRAWLED_URL"} - className="w-full" - onValueChange={setActiveTab} - > -
-
- + {(() => { + // Get sources for this specific message + const messageConnectorSources = getMessageConnectorSources(message); + + if (messageConnectorSources.length === 0) { + return ( +
+ +
+ ); + } + + // Use these message-specific sources for the Tabs component + return ( + 0 ? messageConnectorSources[0].type : "CRAWLED_URL"} + className="w-full" + > +
+
+ -
-
- - {connectorSources.map((connector) => ( - - {getConnectorIcon(connector.type)} - {connector.name.split(' ')[0]} - - {getSourcesCount(connector.type)} - - - ))} - +
+
+ + {messageConnectorSources.map((connector) => ( + + {getConnectorIcon(connector.type)} + {connector.name.split(' ')[0]} + + {connector.sources?.length || 0} + + + ))} + +
+
+ +
- -
-
+ {messageConnectorSources.map(connector => ( + +
+ {connector.sources?.slice(0, INITIAL_SOURCES_DISPLAY)?.map((source: any) => ( + +
+
+ {getConnectorIcon(connector.type)} +
+
+

{source.title}

+

{source.description}

+
+ +
+
+ ))} - {connectorSources.map(connector => ( - -
- {getMainViewSources(connector)?.map((source: any) => ( - -
-
- {getConnectorIcon(connector.type)} -
-
-

{source.title}

-

{source.description}

-
- -
-
- ))} - - {connector.sources.length > INITIAL_SOURCES_DISPLAY && ( - setDialogOpenId(open ? connector.id : null)}> - - - - - - - - )} -
-
- ))} - + {connector.sources?.length > INITIAL_SOURCES_DISPLAY && ( + setDialogOpenId(open ? connector.id : null)}> + + + + + + + + )} +
+
+ ))} +
+ ); + })()}
{/* Answer Section */}