feat: Added Q/A Mode in Research Agent

.vscode/launch.json

@@ -0,0 +1,37 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: UV Run with Reload",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "app.app:app",
+                "--reload",
+                "--host",
+                "0.0.0.0",
+                "--log-level",
+                "info",
+                "--reload-dir",
+                "app"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/surfsense_backend",
+            "python": "${workspaceFolder}/surfsense_backend/.venv/Scripts/python.exe"
+        },
+        {
+            "name": "Python Debugger: main.py (direct)",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/surfsense_backend/main.py",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "cwd": "${workspaceFolder}/surfsense_backend"
+        }
+    ]
+}

surfsense_backend/.vscode/launch.json

@@ -1,16 +0,0 @@
-{
-    // Use IntelliSense to learn about possible attributes.
-    // Hover to view descriptions of existing attributes.
-    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-    "version": "0.2.0",
-    "configurations": [
-        {
-            "name": "Python Debugger: main.py",
-            "type": "debugpy",
-            "request": "launch",
-            "program": "${workspaceFolder}/main.py",
-            "console": "integratedTerminal",
-            "justMyCode": false
-        }
-    ]
-}

surfsense_backend/alembic/versions/10_update_chattype_enum_to_qna_report_structure.py

@@ -0,0 +1,108 @@
+"""Update ChatType enum from GENERAL/DEEP/DEEPER/DEEPEST to QNA/REPORT_* structure
+
+Revision ID: 10
+Revises: 9
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "10"
+down_revision: Union[str, None] = "9"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+# Define the ENUM type name
+CHAT_TYPE_ENUM = "chattype"
+
+
+def upgrade() -> None:
+    """Upgrade schema - replace ChatType enum values with new QNA/REPORT structure."""
+    
+    # Old enum name for temporary storage
+    old_enum_name = f"{CHAT_TYPE_ENUM}_old"
+    
+    # New enum values
+    new_values = (
+        "QNA",
+        "REPORT_GENERAL", 
+        "REPORT_DEEP",
+        "REPORT_DEEPER"
+    )
+    new_values_sql = ", ".join([f"'{v}'" for v in new_values])
+    
+    # Table and column info
+    table_name = "chats"
+    column_name = "type"
+    
+    # Step 1: Rename the current enum type
+    op.execute(f"ALTER TYPE {CHAT_TYPE_ENUM} RENAME TO {old_enum_name}")
+    
+    # Step 2: Create the new enum type with new values
+    op.execute(f"CREATE TYPE {CHAT_TYPE_ENUM} AS ENUM({new_values_sql})")
+    
+    # Step 3: Add a temporary column with the new type
+    op.execute(f"ALTER TABLE {table_name} ADD COLUMN {column_name}_new {CHAT_TYPE_ENUM}")
+    
+    # Step 4: Update the temporary column with mapped values
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'QNA' WHERE {column_name}::text = 'GENERAL'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'REPORT_DEEP' WHERE {column_name}::text = 'DEEP'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'REPORT_DEEPER' WHERE {column_name}::text = 'DEEPER'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'REPORT_DEEPER' WHERE {column_name}::text = 'DEEPEST'")
+    
+    # Step 5: Drop the old column
+    op.execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name}")
+    
+    # Step 6: Rename the new column to the original name
+    op.execute(f"ALTER TABLE {table_name} RENAME COLUMN {column_name}_new TO {column_name}")
+    
+    # Step 7: Drop the old enum type
+    op.execute(f"DROP TYPE {old_enum_name}")
+
+
+def downgrade() -> None:
+    """Downgrade schema - revert ChatType enum to old GENERAL/DEEP/DEEPER/DEEPEST structure."""
+    
+    # Old enum name for temporary storage
+    old_enum_name = f"{CHAT_TYPE_ENUM}_old"
+    
+    # Original enum values
+    original_values = (
+        "GENERAL",
+        "DEEP", 
+        "DEEPER",
+        "DEEPEST"
+    )
+    original_values_sql = ", ".join([f"'{v}'" for v in original_values])
+    
+    # Table and column info
+    table_name = "chats"
+    column_name = "type"
+    
+    # Step 1: Rename the current enum type
+    op.execute(f"ALTER TYPE {CHAT_TYPE_ENUM} RENAME TO {old_enum_name}")
+    
+    # Step 2: Create the new enum type with original values
+    op.execute(f"CREATE TYPE {CHAT_TYPE_ENUM} AS ENUM({original_values_sql})")
+    
+    # Step 3: Add a temporary column with the original type
+    op.execute(f"ALTER TABLE {table_name} ADD COLUMN {column_name}_new {CHAT_TYPE_ENUM}")
+    
+    # Step 4: Update the temporary column with mapped values back to old values
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'GENERAL' WHERE {column_name}::text = 'QNA'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'GENERAL' WHERE {column_name}::text = 'REPORT_GENERAL'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'DEEP' WHERE {column_name}::text = 'REPORT_DEEP'")
+    op.execute(f"UPDATE {table_name} SET {column_name}_new = 'DEEPER' WHERE {column_name}::text = 'REPORT_DEEPER'")
+    
+    # Step 5: Drop the old column
+    op.execute(f"ALTER TABLE {table_name} DROP COLUMN {column_name}")
+    
+    # Step 6: Rename the new column to the original name
+    op.execute(f"ALTER TABLE {table_name} RENAME COLUMN {column_name}_new TO {column_name}")
+    
+    # Step 7: Drop the old enum type
+    op.execute(f"DROP TYPE {old_enum_name}") 

surfsense_backend/app/agents/researcher/configuration.py

@@ -13,6 +13,13 @@ class SearchMode(Enum):
     CHUNKS = "CHUNKS"
     DOCUMENTS = "DOCUMENTS"
 
+class ResearchMode(Enum):
+    """Enum defining the type of research mode."""
+    QNA = "QNA"
+    REPORT_GENERAL = "REPORT_GENERAL"
+    REPORT_DEEP = "REPORT_DEEP"
+    REPORT_DEEPER = "REPORT_DEEPER"
+
 
 @dataclass(kw_only=True)
 class Configuration:
@@ -25,7 +32,7 @@ class Configuration:
     user_id: str
     search_space_id: int
     search_mode: SearchMode
-
+    research_mode: ResearchMode
 
     @classmethod
     def from_runnable_config(

surfsense_backend/app/agents/researcher/graph.py

@@ -1,7 +1,7 @@
 from langgraph.graph import StateGraph
 from .state import State
-from .nodes import reformulate_user_query, write_answer_outline, process_sections
+from .nodes import reformulate_user_query, write_answer_outline, process_sections, handle_qna_workflow
-from .configuration import Configuration
+from .configuration import Configuration, ResearchMode
 from typing import TypedDict, List, Dict, Any, Optional
 
 # Define what keys are in our state dict
@@ -11,12 +11,27 @@ class GraphState(TypedDict):
     # Final output
     final_written_report: Optional[str]
 
+def route_based_on_research_mode(state: State) -> str:
+    """
+    Route to different workflows based on research_mode.
+    
+    Args:
+        state: The current state containing the configuration
+    
+    Returns:
+        "qna_workflow" for QNA mode, "report_workflow" for report modes
+    """
+    # The configuration should be available in the graph context
+    # We'll handle this by checking the research_mode during execution
+    return "route_research_mode"
+
 def build_graph():
     """
     Build and return the LangGraph workflow.
     
-    This function constructs the researcher agent graph with proper state management
+    This function constructs the researcher agent graph with conditional routing
-    and node connections following LangGraph best practices.
+    based on research_mode - QNA mode uses a direct Q&A workflow while other modes
+    use the full report generation pipeline.
     
     Returns:
         A compiled LangGraph workflow
@@ -26,12 +41,36 @@ def build_graph():
     
     # Add nodes to the graph
     workflow.add_node("reformulate_user_query", reformulate_user_query)
+    workflow.add_node("handle_qna_workflow", handle_qna_workflow)
     workflow.add_node("write_answer_outline", write_answer_outline)
     workflow.add_node("process_sections", process_sections)
 
-    # Define the edges - create a linear flow
+    # Define the edges
     workflow.add_edge("__start__", "reformulate_user_query")
-    workflow.add_edge("reformulate_user_query", "write_answer_outline")
+    
+    # Add conditional edges from reformulate_user_query based on research mode
+    def route_after_reformulate(state: State, config) -> str:
+        """Route based on research_mode after reformulating the query."""
+        configuration = Configuration.from_runnable_config(config)
+        
+        if configuration.research_mode == ResearchMode.QNA.value:
+            return "handle_qna_workflow"
+        else:
+            return "write_answer_outline"
+    
+    workflow.add_conditional_edges(
+        "reformulate_user_query",
+        route_after_reformulate,
+        {
+            "handle_qna_workflow": "handle_qna_workflow",
+            "write_answer_outline": "write_answer_outline"
+        }
+    )
+    
+    # QNA workflow path
+    workflow.add_edge("handle_qna_workflow", "__end__")
+    
+    # Report generation workflow path
     workflow.add_edge("write_answer_outline", "process_sections")
     workflow.add_edge("process_sections", "__end__")
 

surfsense_backend/app/agents/researcher/nodes.py

@@ -15,10 +15,10 @@ from .prompts import get_answer_outline_system_prompt
 from .state import State
 from .sub_section_writer.graph import graph as sub_section_writer_graph
 from .sub_section_writer.configuration import SubSectionType
+from .qna_agent.graph import graph as qna_agent_graph
 
 from app.utils.query_service import QueryService
 
-
 from langgraph.types import StreamWriter
 
 
@@ -842,4 +842,131 @@ async def reformulate_user_query(state: State, config: RunnableConfig, writer: S
         "reformulated_query": reformulated_query
     }
 
+async def handle_qna_workflow(state: State, config: RunnableConfig, writer: StreamWriter) -> Dict[str, Any]:
+    """
+    Handle the QNA research workflow.
+    
+    This node fetches relevant documents for the user query and then uses the QNA agent
+    to generate a comprehensive answer with proper citations.
+    
+    Returns:
+        Dict containing the final answer in the "final_written_report" key for consistency.
+    """
+    streaming_service = state.streaming_service
+    configuration = Configuration.from_runnable_config(config)
+    
+    reformulated_query = state.reformulated_query
+    user_query = configuration.user_query
+    
+    streaming_service.only_update_terminal("🤔 Starting Q&A research workflow...")
+    writer({"yeild_value": streaming_service._format_annotations()})
+    
+    streaming_service.only_update_terminal(f"🔍 Researching: \"{user_query[:100]}...\"")
+    writer({"yeild_value": streaming_service._format_annotations()})
+    
+    # Fetch relevant documents for the QNA query
+    streaming_service.only_update_terminal("🔍 Searching for relevant information across all connectors...")
+    writer({"yeild_value": streaming_service._format_annotations()})
+    
+    # Use a reasonable top_k for QNA - not too many documents to avoid overwhelming the LLM
+    TOP_K = 15
+    
+    relevant_documents = []
+    async with async_session_maker() as db_session:
+        try:
+            # Create connector service inside the db_session scope
+            connector_service = ConnectorService(db_session)
+            
+            # Use the reformulated query as a single research question
+            research_questions = [reformulated_query]
+            
+            relevant_documents = await fetch_relevant_documents(
+                research_questions=research_questions,
+                user_id=configuration.user_id,
+                search_space_id=configuration.search_space_id,
+                db_session=db_session,
+                connectors_to_search=configuration.connectors_to_search,
+                writer=writer,
+                state=state,
+                top_k=TOP_K,
+                connector_service=connector_service,
+                search_mode=configuration.search_mode
+            )
+        except Exception as e:
+            error_message = f"Error fetching relevant documents for QNA: {str(e)}"
+            print(error_message)
+            streaming_service.only_update_terminal(f"❌ {error_message}", "error")
+            writer({"yeild_value": streaming_service._format_annotations()})
+            # Continue with empty documents - the QNA agent will handle this gracefully
+            relevant_documents = []
+    
+    print(f"Fetched {len(relevant_documents)} relevant documents for QNA")
+    streaming_service.only_update_terminal(f"🧠 Generating comprehensive answer using {len(relevant_documents)} relevant sources...")
+    writer({"yeild_value": streaming_service._format_annotations()})
+    
+    # Prepare configuration for the QNA agent
+    qna_config = {
+        "configurable": {
+            "user_query": reformulated_query,  # Use the reformulated query
+            "relevant_documents": relevant_documents,
+            "user_id": configuration.user_id,
+            "search_space_id": configuration.search_space_id
+        }
+    }
+    
+    # Create the state for the QNA agent (it has a different state structure)
+    qna_state = {
+        "db_session": state.db_session,
+        "chat_history": state.chat_history
+    }
+    
+    try:
+        streaming_service.only_update_terminal("✍️ Writing comprehensive answer with citations...")
+        writer({"yeild_value": streaming_service._format_annotations()})
+        
+        # Track streaming content for real-time updates
+        complete_content = ""
+        
+        # Call the QNA agent with streaming
+        async for chunk_type, chunk in qna_agent_graph.astream(qna_state, qna_config, stream_mode=["values"]):
+            if "final_answer" in chunk:
+                new_content = chunk["final_answer"]
+                if new_content and new_content != complete_content:
+                    # Extract only the new content (delta)
+                    delta = new_content[len(complete_content):]
+                    complete_content = new_content
+                    
+                    # Stream the real-time answer if there's new content
+                    if delta:
+                        # Update terminal with progress
+                        word_count = len(complete_content.split())
+                        streaming_service.only_update_terminal(f"✍️ Writing answer... ({word_count} words)")
+                        
+                        # Update the answer in real-time
+                        answer_lines = complete_content.split("\n")
+                        streaming_service.only_update_answer(answer_lines)
+                        writer({"yeild_value": streaming_service._format_annotations()})
+        
+        # Set default if no content was received
+        if not complete_content:
+            complete_content = "I couldn't find relevant information in your knowledge base to answer this question."
+        
+        streaming_service.only_update_terminal("🎉 Q&A answer generated successfully!")
+        writer({"yeild_value": streaming_service._format_annotations()})
+        
+        # Return the final answer in the expected state field
+        return {
+            "final_written_report": complete_content
+        }
+        
+    except Exception as e:
+        error_message = f"Error generating QNA answer: {str(e)}"
+        print(error_message)
+        streaming_service.only_update_terminal(f"❌ {error_message}", "error")
+        writer({"yeild_value": streaming_service._format_annotations()})
+        
+        return {
+            "final_written_report": f"Error generating answer: {str(e)}"
+        }
+
 

surfsense_backend/app/agents/researcher/qna_agent/__init__.py

@@ -0,0 +1,6 @@
+"""QnA Agent.
+"""
+
+from .graph import graph
+
+__all__ = ["graph"]

surfsense_backend/app/agents/researcher/qna_agent/configuration.py

@@ -0,0 +1,28 @@
+"""Define the configurable parameters for the agent."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, fields
+from typing import Optional, List, Any
+
+from langchain_core.runnables import RunnableConfig
+
+
+@dataclass(kw_only=True)
+class Configuration:
+    """The configuration for the Q&A agent."""
+
+    # Configuration parameters for the Q&A agent
+    user_query: str  # The user's question to answer
+    relevant_documents: List[Any]  # Documents provided directly to the agent for answering
+    user_id: str  # User identifier
+    search_space_id: int  # Search space identifier
+
+    @classmethod
+    def from_runnable_config(
+        cls, config: Optional[RunnableConfig] = None
+    ) -> Configuration:
+        """Create a Configuration instance from a RunnableConfig object."""
+        configurable = (config.get("configurable") or {}) if config else {}
+        _fields = {f.name for f in fields(cls) if f.init}
+        return cls(**{k: v for k, v in configurable.items() if k in _fields})

surfsense_backend/app/agents/researcher/qna_agent/graph.py

@@ -0,0 +1,20 @@
+from langgraph.graph import StateGraph
+from .state import State
+from .nodes import rerank_documents, answer_question
+from .configuration import Configuration
+
+# Define a new graph
+workflow = StateGraph(State, config_schema=Configuration)
+
+# Add the nodes to the graph
+workflow.add_node("rerank_documents", rerank_documents)
+workflow.add_node("answer_question", answer_question)
+
+# Connect the nodes
+workflow.add_edge("__start__", "rerank_documents")
+workflow.add_edge("rerank_documents", "answer_question")
+workflow.add_edge("answer_question", "__end__")
+
+# Compile the workflow into an executable graph
+graph = workflow.compile()
+graph.name = "SurfSense QnA Agent"  # This defines the custom name in LangSmith

surfsense_backend/app/agents/researcher/qna_agent/nodes.py

@@ -0,0 +1,148 @@
+from .configuration import Configuration
+from langchain_core.runnables import RunnableConfig
+from .state import State
+from typing import Any, Dict
+from app.config import config as app_config
+from .prompts import get_qna_citation_system_prompt
+from langchain_core.messages import HumanMessage, SystemMessage
+
+async def rerank_documents(state: State, config: RunnableConfig) -> Dict[str, Any]:
+    """
+    Rerank the documents based on relevance to the user's question.
+    
+    This node takes the relevant documents provided in the configuration,
+    reranks them using the reranker service based on the user's query,
+    and updates the state with the reranked documents.
+    
+    Returns:
+        Dict containing the reranked documents.
+    """
+    # Get configuration and relevant documents
+    configuration = Configuration.from_runnable_config(config)
+    documents = configuration.relevant_documents
+    user_query = configuration.user_query
+
+    # If no documents were provided, return empty list
+    if not documents or len(documents) == 0:
+        return {
+            "reranked_documents": []
+        }
+    
+    # Get reranker service from app config
+    reranker_service = getattr(app_config, "reranker_service", None)
+    
+    # Use documents as is if no reranker service is available
+    reranked_docs = documents
+    
+    if reranker_service:
+        try:
+            # Convert documents to format expected by reranker if needed
+            reranker_input_docs = [
+                {
+                    "chunk_id": doc.get("chunk_id", f"chunk_{i}"),
+                    "content": doc.get("content", ""),
+                    "score": doc.get("score", 0.0),
+                    "document": {
+                        "id": doc.get("document", {}).get("id", ""),
+                        "title": doc.get("document", {}).get("title", ""),
+                        "document_type": doc.get("document", {}).get("document_type", ""),
+                        "metadata": doc.get("document", {}).get("metadata", {})
+                    }
+                } for i, doc in enumerate(documents)
+            ]
+            
+            # Rerank documents using the user's query
+            reranked_docs = reranker_service.rerank_documents(user_query, reranker_input_docs)
+            
+            # Sort by score in descending order
+            reranked_docs.sort(key=lambda x: x.get("score", 0), reverse=True)
+            
+            print(f"Reranked {len(reranked_docs)} documents for Q&A query: {user_query}")
+        except Exception as e:
+            print(f"Error during reranking: {str(e)}")
+            # Use original docs if reranking fails
+    
+    return {
+        "reranked_documents": reranked_docs
+    }
+
+async def answer_question(state: State, config: RunnableConfig) -> Dict[str, Any]:
+    """
+    Answer the user's question using the provided documents.
+    
+    This node takes the relevant documents provided in the configuration and uses
+    an LLM to generate a comprehensive answer to the user's question with
+    proper citations. The citations follow IEEE format using source IDs from the
+    documents.
+    
+    Returns:
+        Dict containing the final answer in the "final_answer" key.
+    """
+    
+    # Get configuration and relevant documents from configuration
+    configuration = Configuration.from_runnable_config(config)
+    documents = configuration.relevant_documents
+    user_query = configuration.user_query
+    
+    # Initialize LLM
+    llm = app_config.fast_llm_instance
+    
+    # If no documents were provided, return a message indicating this
+    if not documents or len(documents) == 0:
+        return {
+            "final_answer": "I don't have any relevant documents in your personal knowledge base to answer this question. Please try asking about topics covered in your saved content, or add more documents to your knowledge base."
+        }
+    
+    # Prepare documents for citation formatting
+    formatted_documents = []
+    for i, doc in enumerate(documents):
+        # Extract content and metadata
+        content = doc.get("content", "")
+        doc_info = doc.get("document", {})
+        document_id = doc_info.get("id")  # Use document ID
+        
+        # Format document according to the citation system prompt's expected format
+        formatted_doc = f"""
+        <document>
+            <metadata>
+                <source_id>{document_id}</source_id>
+                <source_type>{doc_info.get("document_type", "CRAWLED_URL")}</source_type>
+            </metadata>
+            <content>
+                {content}
+            </content>
+        </document>
+        """
+        formatted_documents.append(formatted_doc)
+    
+    # Create the formatted documents text
+    documents_text = "\n".join(formatted_documents)
+    
+    # Construct a clear, structured query for the LLM
+    human_message_content = f"""
+    Source material from your personal knowledge base:
+    <documents>
+        {documents_text}
+    </documents>
+    
+    User's question:
+    <user_query>
+        {user_query}
+    </user_query>
+    
+    Please provide a detailed, comprehensive answer to the user's question using the information from their personal knowledge sources. Make sure to cite all information appropriately and engage in a conversational manner.
+    """
+    
+    # Create messages for the LLM, including chat history for context
+    messages_with_chat_history = state.chat_history + [
+        SystemMessage(content=get_qna_citation_system_prompt()),
+        HumanMessage(content=human_message_content)
+    ]
+    
+    # Call the LLM and get the response
+    response = await llm.ainvoke(messages_with_chat_history)
+    final_answer = response.content
+    
+    return {
+        "final_answer": final_answer
+    }

surfsense_backend/app/agents/researcher/qna_agent/prompts.py

@@ -0,0 +1,120 @@
+import datetime
+
+
+def get_qna_citation_system_prompt():
+    return f"""
+Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")}
+You are SurfSense, an advanced AI research assistant that provides detailed, well-researched answers to user questions by synthesizing information from multiple personal knowledge sources.
+
+<knowledge_sources>
+- EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history)
+- CRAWLED_URL: "Webpages indexed by SurfSense web crawler" (personally selected websites)
+- FILE: "User-uploaded documents (PDFs, Word, etc.)" (personal files)
+- SLACK_CONNECTOR: "Slack conversations and shared content" (personal workspace communications)
+- NOTION_CONNECTOR: "Notion workspace pages and databases" (personal knowledge management)
+- YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
+- GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
+- LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
+- DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions)
+- TAVILY_API: "Tavily search API results" (personalized search results)
+- LINKUP_API: "Linkup search API results" (personalized search results)
+</knowledge_sources>
+
+<instructions>
+1. Carefully analyze all provided documents in the <document> sections.
+2. Extract relevant information that directly addresses the user's question.
+3. Provide a comprehensive, detailed answer using information from the user's personal knowledge sources.
+4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata.
+5. Make sure ALL factual statements from the documents have proper citations.
+6. If multiple documents support the same point, include all relevant citations [X], [Y].
+7. Structure your answer logically and conversationally, as if having a detailed discussion with the user.
+8. Use your own words to synthesize and connect ideas, but cite ALL information from the documents.
+9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations.
+10. If the user's question cannot be fully answered with the provided documents, clearly state what information is missing.
+11. Provide actionable insights and practical information when relevant to the user's question.
+12. CRITICAL: You MUST use the exact source_id value from each document's metadata for citations. Do not create your own citation numbers.
+13. CRITICAL: Every citation MUST be in the IEEE format [X] where X is the exact source_id value.
+14. CRITICAL: Never renumber or reorder citations - always use the original source_id values.
+15. CRITICAL: Do not return citations as clickable links.
+16. CRITICAL: Never format citations as markdown links like "([1](https://example.com))". Always use plain square brackets only.
+17. CRITICAL: Citations must ONLY appear as [X] or [X], [Y], [Z] format - never with parentheses, hyperlinks, or other formatting.
+18. CRITICAL: Never make up citation numbers. Only use source_id values that are explicitly provided in the document metadata.
+19. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up.
+20. CRITICAL: Remember that all knowledge sources contain personal information - provide answers that reflect this personal context.
+21. CRITICAL: Be conversational and engaging while maintaining accuracy and proper citations.
+</instructions>
+
+<format>
+- Write in a clear, conversational tone suitable for detailed Q&A discussions
+- Provide comprehensive answers that thoroughly address the user's question
+- Use appropriate paragraphs and structure for readability
+- Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata
+- Citations should appear at the end of the sentence containing the information they support
+- Multiple citations should be separated by commas: [X], [Y], [Z]
+- No need to return references section. Just citation numbers in answer.
+- NEVER create your own citation numbering system - use the exact source_id values from the documents
+- NEVER format citations as clickable links or as markdown links like "([1](https://example.com))". Always use plain square brackets only
+- NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess
+- ALWAYS provide personalized answers that reflect the user's own knowledge and context
+- Be thorough and detailed in your explanations while remaining focused on the user's specific question
+- If asking follow-up questions would be helpful, suggest them at the end of your response
+</format>
+
+<input_example>
+<documents>
+    <document>
+        <metadata>
+            <source_id>5</source_id>
+            <source_type>GITHUB_CONNECTOR</source_type>
+        </metadata>
+        <content>
+            Python's asyncio library provides tools for writing concurrent code using the async/await syntax. It's particularly useful for I/O-bound and high-level structured network code.
+        </content>
+    </document>
+    
+    <document>
+        <metadata>
+            <source_id>12</source_id>
+            <source_type>YOUTUBE_VIDEO</source_type>
+        </metadata>
+        <content>
+            Asyncio can improve performance by allowing other code to run while waiting for I/O operations to complete. However, it's not suitable for CPU-bound tasks as it runs on a single thread.
+        </content>
+    </document>
+</documents>
+
+User Question: "How does Python asyncio work and when should I use it?"
+</input_example>
+
+<output_example>
+Based on your GitHub repositories and video content, Python's asyncio library provides tools for writing concurrent code using the async/await syntax [5]. It's particularly useful for I/O-bound and high-level structured network code [5].
+
+The key advantage of asyncio is that it can improve performance by allowing other code to run while waiting for I/O operations to complete [12]. This makes it excellent for scenarios like web scraping, API calls, database operations, or any situation where your program spends time waiting for external resources.
+
+However, from your video learning, it's important to note that asyncio is not suitable for CPU-bound tasks as it runs on a single thread [12]. For computationally intensive work, you'd want to use multiprocessing instead.
+
+Would you like me to explain more about specific asyncio patterns or help you determine if asyncio is right for a particular project you're working on?
+</output_example>
+
+<incorrect_citation_formats>
+DO NOT use any of these incorrect citation formats:
+- Using parentheses and markdown links: ([1](https://github.com/MODSetter/SurfSense))
+- Using parentheses around brackets: ([1])
+- Using hyperlinked text: [link to source 1](https://example.com)
+- Using footnote style: ... library¹
+- Making up citation numbers when source_id is unknown
+
+ONLY use plain square brackets [1] or multiple citations [1], [2], [3]
+</incorrect_citation_formats>
+
+<user_query_instructions>
+When you see a user query, focus exclusively on providing a detailed, comprehensive answer using information from the provided documents, which contain the user's personal knowledge and data.
+
+Make sure your response:
+1. Directly and thoroughly answers the user's question with personalized information from their own knowledge sources
+2. Uses proper citations for all information from documents
+3. Is conversational, engaging, and detailed
+4. Acknowledges the personal nature of the information being provided
+5. Offers follow-up suggestions when appropriate
+</user_query_instructions>
+"""

surfsense_backend/app/agents/researcher/qna_agent/state.py

@@ -0,0 +1,25 @@
+"""Define the state structures for the agent."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Any
+from sqlalchemy.ext.asyncio import AsyncSession
+
+@dataclass
+class State:
+    """Defines the dynamic state for the Q&A agent during execution.
+
+    This state tracks the database session, chat history, and the outputs 
+    generated by the agent's nodes during question answering.
+    See: https://langchain-ai.github.io/langgraph/concepts/low_level/#state
+    for more information.
+    """
+
+    # Runtime context
+    db_session: AsyncSession
+    
+    chat_history: Optional[List[Any]] = field(default_factory=list)
+    # OUTPUT: Populated by agent nodes
+    reranked_documents: Optional[List[Any]] = None
+    final_answer: Optional[str] = None

surfsense_backend/app/db.py

@@ -63,10 +63,10 @@ class SearchSourceConnectorType(str, Enum):
     DISCORD_CONNECTOR = "DISCORD_CONNECTOR"
     
 class ChatType(str, Enum):
-    GENERAL = "GENERAL"
+    QNA = "QNA"
-    DEEP = "DEEP"
+    REPORT_GENERAL = "REPORT_GENERAL"
-    DEEPER = "DEEPER"
+    REPORT_DEEP = "REPORT_DEEP"
-    DEEPEST = "DEEPEST"
+    REPORT_DEEPER = "REPORT_DEEPER"
     
 class Base(DeclarativeBase):
     pass

surfsense_backend/app/tasks/stream_connector_search_results.py

@@ -34,12 +34,16 @@ async def stream_connector_search_results(
         str: Formatted response strings
     """
     streaming_service = StreamingService()
-    if research_mode == "GENERAL":
+    
+    if research_mode == "REPORT_GENERAL":
         NUM_SECTIONS = 1
-    elif research_mode == "DEEP":
+    elif research_mode == "REPORT_DEEP":
         NUM_SECTIONS = 3
-    elif research_mode == "DEEPER":
+    elif research_mode == "REPORT_DEEPER":
         NUM_SECTIONS = 6
+    else:
+        # Default fallback
+        NUM_SECTIONS = 1
     
     # Convert UUID to string if needed
     user_id_str = str(user_id) if isinstance(user_id, UUID) else user_id
@@ -57,7 +61,8 @@ async def stream_connector_search_results(
             "connectors_to_search": selected_connectors,
             "user_id": user_id_str,
             "search_space_id": search_space_id,
-            "search_mode": search_mode
+            "search_mode": search_mode,
+            "research_mode": research_mode
         }
     }
     # Initialize state with database session and streaming service

surfsense_web/app/dashboard/[search_space_id]/researcher/[chat_id]/page.tsx

@@ -45,7 +45,7 @@ import {
   scrollTabsRight as scrollTabsRightUtil,
   Source,
   ResearchMode,
-  researcherOptions
+  ResearchModeControl
 } from '@/components/chat';
 import { MarkdownViewer } from '@/components/markdown-viewer';
 import { Logo } from '@/components/Logo';
@@ -250,7 +250,7 @@ const ChatPage = () => {
   const [terminalExpanded, setTerminalExpanded] = useState(false);
   const [selectedConnectors, setSelectedConnectors] = useState<string[]>(["CRAWLED_URL"]);
   const [searchMode, setSearchMode] = useState<'DOCUMENTS' | 'CHUNKS'>('DOCUMENTS');
-  const [researchMode, setResearchMode] = useState<ResearchMode>("GENERAL");
+  const [researchMode, setResearchMode] = useState<ResearchMode>("QNA");
   const [currentTime, setCurrentTime] = useState<string>('');
   const [currentDate, setCurrentDate] = useState<string>('');
   const terminalMessagesRef = useRef<HTMLDivElement>(null);
@@ -1079,12 +1079,11 @@ const ChatPage = () => {
                 </button>
               </div>
 
-              {/* Research Mode Segmented Control */}
+              {/* Research Mode Control */}
               <div className="h-8">
-                <SegmentedControl<ResearchMode>
+                <ResearchModeControl
                   value={researchMode}
                   onChange={setResearchMode}
-                  options={researcherOptions}
                 />
               </div>
             </div>

surfsense_web/app/dashboard/[search_space_id]/researcher/page.tsx

@@ -29,7 +29,7 @@ const ResearcherPage = () => {
             'Authorization': `Bearer ${token}`
           },
           body: JSON.stringify({
-            type: "GENERAL",
+            type: "QNA",
             title: "Untitled Chat", // Empty title initially
             initial_connectors: ["CRAWLED_URL"], // Default connector
             messages: [],

surfsense_web/components/chat/ConnectorComponents.tsx

@@ -10,6 +10,8 @@ import {
   File,
   Link,
   Webhook,
+  MessageCircle,
+  FileText,
 } from 'lucide-react';
 import { IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconBrandGithub, IconLayoutKanban, IconLinkPlus, IconBrandDiscord } from "@tabler/icons-react";
 import { Button } from '@/components/ui/button';
@@ -56,17 +58,22 @@ export const getConnectorIcon = (connectorType: string) => {
 
 export const researcherOptions: { value: ResearchMode; label: string; icon: React.ReactNode }[] = [
   {
-    value: 'GENERAL',
+    value: 'QNA',
+    label: 'Q/A',
+    icon: getConnectorIcon('GENERAL')
+  },
+  {
+    value: 'REPORT_GENERAL',
     label: 'General',
     icon: getConnectorIcon('GENERAL')
   },
   {
-    value: 'DEEP',
+    value: 'REPORT_DEEP',
     label: 'Deep',
     icon: getConnectorIcon('DEEP')
   },
   {
-    value: 'DEEPER',
+    value: 'REPORT_DEEPER',
     label: 'Deeper',
     icon: getConnectorIcon('DEEPER')
   },
@@ -169,3 +176,93 @@ export const ConnectorButton = ({ selectedConnectors, onClick, connectorSources
     </Button>
   );
 };
+
+// New component for Research Mode Control with Q/A and Report toggle
+type ResearchModeControlProps = {
+  value: ResearchMode;
+  onChange: (value: ResearchMode) => void;
+};
+
+export const ResearchModeControl = ({ value, onChange }: ResearchModeControlProps) => {
+  // Determine if we're in Q/A mode or Report mode
+  const isQnaMode = value === 'QNA';
+  const isReportMode = value.startsWith('REPORT_');
+  
+  // Get the current report sub-mode
+  const getCurrentReportMode = () => {
+    if (!isReportMode) return 'GENERAL';
+    return value.replace('REPORT_', '') as 'GENERAL' | 'DEEP' | 'DEEPER';
+  };
+
+  const reportSubOptions = [
+    { value: 'GENERAL', label: 'General', icon: getConnectorIcon('GENERAL') },
+    { value: 'DEEP', label: 'Deep', icon: getConnectorIcon('DEEP') },
+    { value: 'DEEPER', label: 'Deeper', icon: getConnectorIcon('DEEPER') },
+  ];
+
+  const handleModeToggle = (mode: 'QNA' | 'REPORT') => {
+    if (mode === 'QNA') {
+      onChange('QNA');
+    } else {
+      // Default to GENERAL for Report mode
+      onChange('REPORT_GENERAL');
+    }
+  };
+
+  const handleReportSubModeChange = (subMode: string) => {
+    onChange(`REPORT_${subMode}` as ResearchMode);
+  };
+
+  return (
+    <div className="flex items-center gap-2">
+      {/* Main Q/A vs Report Toggle */}
+      <div className="flex h-8 rounded-md border border-border overflow-hidden">
+        <button
+          className={`flex h-full items-center gap-1 px-3 text-xs font-medium transition-colors whitespace-nowrap ${
+            isQnaMode 
+              ? 'bg-primary text-primary-foreground' 
+              : 'hover:bg-muted text-muted-foreground hover:text-foreground'
+          }`}
+          onClick={() => handleModeToggle('QNA')}
+          aria-pressed={isQnaMode}
+        >
+          <MessageCircle className="h-3 w-3" />
+          <span>Q/A</span>
+        </button>
+        <button
+          className={`flex h-full items-center gap-1 px-3 text-xs font-medium transition-colors whitespace-nowrap ${
+            isReportMode 
+              ? 'bg-primary text-primary-foreground' 
+              : 'hover:bg-muted text-muted-foreground hover:text-foreground'
+          }`}
+          onClick={() => handleModeToggle('REPORT')}
+          aria-pressed={isReportMode}
+        >
+          <FileText className="h-3 w-3" />
+          <span>Report</span>
+        </button>
+      </div>
+
+      {/* Report Sub-options (only show when in Report mode) */}
+      {isReportMode && (
+        <div className="flex h-8 rounded-md border border-border overflow-hidden">
+          {reportSubOptions.map((option) => (
+            <button
+              key={option.value}
+              className={`flex h-full items-center gap-1 px-2 text-xs font-medium transition-colors whitespace-nowrap ${
+                getCurrentReportMode() === option.value 
+                  ? 'bg-primary text-primary-foreground' 
+                  : 'hover:bg-muted text-muted-foreground hover:text-foreground'
+              }`}
+              onClick={() => handleReportSubModeChange(option.value)}
+              aria-pressed={getCurrentReportMode() === option.value}
+            >
+              {option.icon}
+              <span>{option.label}</span>
+            </button>
+          ))}
+        </div>
+      )}
+    </div>
+  );
+}; 

surfsense_web/components/chat/types.ts

@@ -48,4 +48,4 @@ export interface ToolInvocationUIPart {
 }
 
 
-export type ResearchMode = 'GENERAL' | 'DEEP' | 'DEEPER' | 'DEEPEST'; 
+export type ResearchMode = 'QNA' | 'REPORT_GENERAL' | 'REPORT_DEEP' | 'REPORT_DEEPER';