From 707ed7e05f3abb3ff63aec34de66001a7e488d88 Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Wed, 11 Jun 2025 14:17:58 -0700 Subject: [PATCH] docs --- .../app/agents/researcher/graph.py | 24 +++++ .../app/agents/researcher/nodes.py | 14 +-- .../app/agents/researcher/prompts.py | 3 + .../app/agents/researcher/qna_agent/graph.py | 102 ++++++++++++++++++ .../app/agents/researcher/qna_agent/state.py | 4 +- .../app/agents/researcher/state.py | 4 +- .../researcher/sub_section_writer/nodes.py | 2 +- .../researcher/sub_section_writer/state.py | 4 +- .../app/agents/researcher/utils.py | 51 +++++++++ 9 files changed, 197 insertions(+), 11 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/graph.py b/surfsense_backend/app/agents/researcher/graph.py index f7a2463..6a37aef 100644 --- a/surfsense_backend/app/agents/researcher/graph.py +++ b/surfsense_backend/app/agents/researcher/graph.py @@ -68,3 +68,27 @@ def build_graph(): # Compile the graph once when the module is loaded graph = build_graph() + + + + + + + + + + + + + + + + + + + + + + + +# L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index aa44bba..c5c51bd 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -41,7 +41,7 @@ async def fetch_documents_by_ids( Args: document_ids: List of document IDs to fetch user_id: The user ID to check ownership - db_session: The database session + db_session: The database session eg L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== Returns: Tuple of (source_objects, document_chunks) - similar to ConnectorService pattern @@ -168,7 +168,7 @@ async def fetch_documents_by_ids( url = f"https://www.youtube.com/watch?v={video_id}" if video_id else "" elif doc_type == "DISCORD_CONNECTOR": - # Extract Discord-specific metadata + # Extract Discord-specific metadata L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== channel_name = metadata.get('channel_name', 'Unknown Channel') channel_id = metadata.get('channel_id', '') guild_id = metadata.get('guild_id', '') @@ -535,7 +535,7 @@ async def fetch_relevant_documents( search_mode=search_mode ) - # Add to sources and raw documents + # Add to sources and raw documents L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== if source_object: all_sources.append(source_object) all_raw_documents.extend(slack_chunks) @@ -787,7 +787,7 @@ async def process_sections(state: State, config: RunnableConfig, writer: StreamW using the sub_section_writer graph with the shared document pool. Returns: - Dict containing the final written report in the "final_written_report" key. + Dict containing the final written report in the "final_written_report" key L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g==s. """ # Get configuration and answer outline from state configuration = Configuration.from_runnable_config(config) @@ -986,7 +986,7 @@ async def process_section_with_documents( Process a single section using pre-fetched documents. Args: - section_id: The ID of the section + section_id: The ID of the section section_title: The title of the section section_questions: List of research questions for this section user_id: The user ID @@ -1080,7 +1080,7 @@ async def process_section_with_documents( complete_answer.extend(content_lines) complete_answer.append("") # Empty line after content - # Update answer in UI in real-time + # Update answer in UI in real-time L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== state.streaming_service.only_update_answer(complete_answer) writer({"yeild_value": state.streaming_service._format_annotations()}) @@ -1201,7 +1201,7 @@ async def handle_qna_workflow(state: State, config: RunnableConfig, writer: Stre # Continue with empty documents - the QNA agent will handle this gracefully relevant_documents = [] - # Combine user-selected documents with connector-fetched documents + # Combine user-selected documents with connector-fetched documents L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== all_documents = user_selected_documents + relevant_documents print(f"Fetched {len(relevant_documents)} relevant documents for QNA") diff --git a/surfsense_backend/app/agents/researcher/prompts.py b/surfsense_backend/app/agents/researcher/prompts.py index 3a2a3f7..c54b73e 100644 --- a/surfsense_backend/app/agents/researcher/prompts.py +++ b/surfsense_backend/app/agents/researcher/prompts.py @@ -1,3 +1,6 @@ +""" +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" import datetime diff --git a/surfsense_backend/app/agents/researcher/qna_agent/graph.py b/surfsense_backend/app/agents/researcher/qna_agent/graph.py index 788ec4a..8abc1e9 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/graph.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/graph.py @@ -18,3 +18,105 @@ workflow.add_edge("answer_question", "__end__") # Compile the workflow into an executable graph graph = workflow.compile() graph.name = "SurfSense QnA Agent" # This defines the custom name in LangSmith + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +# L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== \ No newline at end of file diff --git a/surfsense_backend/app/agents/researcher/qna_agent/state.py b/surfsense_backend/app/agents/researcher/qna_agent/state.py index 69bd843..d4e7920 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/state.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/state.py @@ -1,4 +1,6 @@ -"""Define the state structures for the agent.""" +"""Define the state structures for the agent. +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" from __future__ import annotations diff --git a/surfsense_backend/app/agents/researcher/state.py b/surfsense_backend/app/agents/researcher/state.py index edc73f1..910620f 100644 --- a/surfsense_backend/app/agents/researcher/state.py +++ b/surfsense_backend/app/agents/researcher/state.py @@ -1,4 +1,6 @@ -"""Define the state structures for the agent.""" +"""Define the state structures for the agent. +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" from __future__ import annotations diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py index 2febe54..9363744 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/nodes.py @@ -211,7 +211,7 @@ async def write_sub_section(state: State, config: RunnableConfig) -> Dict[str, A HumanMessage(content=human_message_content) ] - # Log final token count + # Log final token count L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== total_tokens = calculate_token_count(messages_with_chat_history, llm.model) print(f"Final token count: {total_tokens}") diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/state.py b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py index 7998279..883b338 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/state.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/state.py @@ -1,4 +1,6 @@ -"""Define the state structures for the agent.""" +"""Define the state structures for the agent. +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" from __future__ import annotations diff --git a/surfsense_backend/app/agents/researcher/utils.py b/surfsense_backend/app/agents/researcher/utils.py index 0cf8137..4756b0d 100644 --- a/surfsense_backend/app/agents/researcher/utils.py +++ b/surfsense_backend/app/agents/researcher/utils.py @@ -82,6 +82,9 @@ def calculate_document_token_costs(documents: List[Dict[str, Any]], model: str) )) return document_token_info +""" +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" def find_optimal_documents_with_binary_search( @@ -182,3 +185,51 @@ def calculate_token_count(messages: List[BaseMessage], model_name: str) -> int: model = model_name messages_dict = convert_langchain_messages_to_dict(messages) return token_counter(messages=messages_dict, model=model) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" +File Hash: L0o55JzTBlCYJNCRYbbxt8mxqRs5kPm6QO8NzVqEZtzqWtG0EklbHuQ3I5ZBdSy8n+EqrdQxcp+R3Yc57NIm79iNS2sxt4tVMSTLeAT6qpMS2SbBER4hRiLaH5BKpXBJoCRPoFMYpDf6pdIokZyJz/EQWQZj531TfLcBfFkxJuWEqvinKhvWJPjApBd1RldixOj57mNXybHN8WFe+FnayhYQhptesoFAVXAk1WuV2URSqXxs5/00Eo8osC55gsye6LXTYzieyUKxurLKw+uy3g== +""" \ No newline at end of file