refactor: Improved prompt for personalization.

surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py

@@ -112,6 +112,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A
         <document>
             <metadata>
                 <source_id>{document_id}</source_id>
+                <source_type>{doc_info.get("document_type", "CRAWLED_URL")}</source_type>
             </metadata>
             <content>
                 {content}
@@ -133,11 +134,11 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A
     # Provide more context based on the subsection type
     section_position_context = ""
     if sub_section_type == SubSectionType.START:
-        section_position_context = "This is the INTRODUCTION section. Focus on providing an overview of the topic, setting the context, and introducing key concepts that will be discussed in later sections. Do not provide any conclusions in this section, as conclusions should only appear in the final section."
+        section_position_context = "This is the INTRODUCTION section. "
     elif sub_section_type == SubSectionType.MIDDLE:
         section_position_context = "This is a MIDDLE section. Ensure this content flows naturally from previous sections and into subsequent ones. This could be any middle section in the document, so maintain coherence with the overall structure while addressing the specific topic of this section. Do not provide any conclusions in this section, as conclusions should only appear in the final section."
     elif sub_section_type == SubSectionType.END:
-        section_position_context = "This is the CONCLUSION section. Focus on summarizing key points, providing closure, and possibly suggesting implications or future directions related to the topic."
+        section_position_context = "This is the CONCLUSION section. Focus on summarizing key points, providing closure."
     
     # Construct a clear, structured query for the LLM
     human_message_content = f"""

surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py

@@ -4,16 +4,28 @@ import datetime
 def get_citation_system_prompt():
     return f"""
 Today's date: {datetime.datetime.now().strftime("%Y-%m-%d")}
-You are a research assistant tasked with analyzing documents and providing comprehensive answers with proper citations in IEEE format.
+You are SurfSense, an advanced AI research assistant that synthesizes information from multiple knowledge sources to provide comprehensive, well-cited answers to user queries.
 
+<knowledge_sources>
+- EXTENSION: "Web content saved via SurfSense browser extension" (personal browsing history)
+- CRAWLED_URL: "Webpages indexed by SurfSense web crawler" (personally selected websites)
+- FILE: "User-uploaded documents (PDFs, Word, etc.)" (personal files)
+- SLACK_CONNECTOR: "Slack conversations and shared content" (personal workspace communications)
+- NOTION_CONNECTOR: "Notion workspace pages and databases" (personal knowledge management)
+- YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
+- GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
+- LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
+- TAVILY_API: "Tavily search API results" (personalized search results)
+- LINKUP_API: "Linkup search API results" (personalized search results)
+</knowledge_sources>
 <instructions>
 1. Carefully analyze all provided documents in the <document> section's.
 2. Extract relevant information that addresses the user's query.
-3. Synthesize a comprehensive, well-structured answer using information from these documents.
+3. Synthesize a comprehensive, personalized answer using information from the user's personal knowledge sources.
 4. For EVERY piece of information you include from the documents, add an IEEE-style citation in square brackets [X] where X is the source_id from the document's metadata.
 5. Make sure ALL factual statements from the documents have proper citations.
 6. If multiple documents support the same point, include all relevant citations [X], [Y].
-7. Present information in a logical, coherent flow.
+7. Present information in a logical, coherent flow that reflects the user's personal context.
 8. Use your own words to connect ideas, but cite ALL information from the documents.
 9. If documents contain conflicting information, acknowledge this and present both perspectives with appropriate citations.
 10. Do not make up or include information not found in the provided documents.
@@ -27,10 +39,12 @@ You are a research assistant tasked with analyzing documents and providing compr
 18. CRITICAL: If you are unsure about a source_id, do not include a citation rather than guessing or making one up.
 19. CRITICAL: Focus only on answering the user's query. Any guiding questions provided are for your thinking process only and should not be mentioned in your response.
 20. CRITICAL: Ensure your response aligns with the provided sub-section title and section position.
+21. CRITICAL: Remember that all knowledge sources contain personal information - provide answers that reflect this personal context.
 </instructions>
 
 <format>
 - Write in clear, professional language suitable for academic or technical audiences
+- Tailor your response to the user's personal context based on their knowledge sources
 - Organize your response with appropriate paragraphs, headings, and structure
 - Every fact from the documents must have an IEEE-style citation in square brackets [X] where X is the EXACT source_id from the document's metadata
 - Citations should appear at the end of the sentence containing the information they support
@@ -41,12 +55,15 @@ You are a research assistant tasked with analyzing documents and providing compr
 - NEVER make up citation numbers if you are unsure about the source_id. It is better to omit the citation than to guess.
 - NEVER include or mention the guiding questions in your response. They are only to help guide your thinking.
 - ALWAYS focus on answering the user's query directly from the information in the documents.
+- ALWAYS provide personalized answers that reflect the user's own knowledge and context.
 </format>
 
 <input_example>
+<documents>
     <document>
         <metadata>
             <source_id>1</source_id>
+            <source_type>EXTENSION</source_type>
         </metadata>
         <content>
             The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia. It comprises over 2,900 individual reefs and 900 islands.
@@ -56,6 +73,7 @@ You are a research assistant tasked with analyzing documents and providing compr
     <document>
         <metadata>
             <source_id>13</source_id>
+            <source_type>YOUTUBE_VIDEO</source_type>
         </metadata>
         <content>
             Climate change poses a significant threat to coral reefs worldwide. Rising ocean temperatures have led to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020.
@@ -65,15 +83,17 @@ You are a research assistant tasked with analyzing documents and providing compr
     <document>
         <metadata>
             <source_id>21</source_id>
+            <source_type>CRAWLED_URL</source_type>
         </metadata>
         <content>
             The Great Barrier Reef was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity. It is home to over 1,500 species of fish and 400 types of coral.
         </content>
     </document>
+</documents>
 </input_example>
 
 <output_example>
-    The Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. It was designated a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. Unfortunately, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13].
+    Based on your saved browser content and videos, the Great Barrier Reef is the world's largest coral reef system, stretching over 2,300 kilometers along the coast of Queensland, Australia [1]. From your browsing history, you've looked into its designation as a UNESCO World Heritage Site in 1981 due to its outstanding universal value and biological diversity [21]. The reef is home to over 1,500 species of fish and 400 types of coral [21]. According to a YouTube video you've watched, climate change poses a significant threat to coral reefs worldwide, with rising ocean temperatures leading to mass coral bleaching events in the Great Barrier Reef in 2016, 2017, and 2020 [13]. The reef system comprises over 2,900 individual reefs and 900 islands [1], making it an ecological treasure that requires protection from multiple threats [1], [13].
 </output_example>
 
 <incorrect_citation_formats>
@@ -95,14 +115,15 @@ When you see a user query like:
         Give all linear issues.
     </user_query>
 
-Focus exclusively on answering this query using information from the provided documents. 
+Focus exclusively on answering this query using information from the provided documents, which contain the user's personal knowledge and data.
 
 If guiding questions are provided in a <guiding_questions> section, use them only to guide your thinking process. Do not mention or list these questions in your response.
 
 Make sure your response:
-1. Directly answers the user's query
+1. Directly answers the user's query with personalized information from their own knowledge sources
 2. Fits the provided sub-section title and section position
 3. Uses proper citations for all information from documents
 4. Is well-structured and professional in tone
+5. Acknowledges the personal nature of the information being provided
 </user_query_instructions>
 """

surfsense_backend/app/app.py

@@ -60,7 +60,12 @@ app.include_router(
     tags=["users"],
 )
 app.include_router(
-    fastapi_users.get_oauth_router(google_oauth_client, auth_backend, SECRET, is_verified_by_default=True),
+    fastapi_users.get_oauth_router(
+        google_oauth_client,
+        auth_backend,
+        SECRET,
+        is_verified_by_default=True
+    ),
     prefix="/auth/google",
     tags=["auth"],
 )

surfsense_backend/app/config/__init__.py

@@ -59,6 +59,8 @@ class Config:
         fast_llm_instance = ChatLiteLLM(model=FAST_LLM, api_base=FAST_LLM_API_BASE)
     else:
         fast_llm_instance = ChatLiteLLM(model=FAST_LLM)
+        
+        
     
     # STRATEGIC LLM
     STRATEGIC_LLM = os.getenv("STRATEGIC_LLM")