update connector indexing / update connector service

surfsense_backend/app/agents/researcher/nodes.py

@@ -172,20 +172,41 @@ async def fetch_documents_by_ids(
                     channel_id = metadata.get('channel_id', '')
                     guild_id = metadata.get('guild_id', '')
                     message_date = metadata.get('start_date', '')
-                    
+
                     title = f"Discord: {channel_name}"
                     if message_date:
                         title += f" ({message_date})"
-                    
+
                     description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content
-                    
+
                     if guild_id and channel_id:
                         url = f"https://discord.com/channels/{guild_id}/{channel_id}"
                     elif channel_id:
                         url = f"https://discord.com/channels/@me/{channel_id}"
                     else:
                         url = ""
-                        
+
+                elif doc_type == "JIRA_CONNECTOR":
+                    # Extract Jira-specific metadata
+                    issue_key = metadata.get('issue_key', 'Unknown Issue')
+                    issue_title = metadata.get('issue_title', 'Untitled Issue')
+                    status = metadata.get('status', '')
+                    priority = metadata.get('priority', '')
+                    issue_type = metadata.get('issue_type', '')
+
+                    title = f"Jira: {issue_key} - {issue_title}"
+                    if status:
+                        title += f" ({status})"
+
+                    description = doc.content[:100] + "..." if len(doc.content) > 100 else doc.content
+
+                    # Construct Jira URL if we have the base URL
+                    base_url = metadata.get('base_url', '')
+                    if base_url and issue_key:
+                        url = f"{base_url}/browse/{issue_key}"
+                    else:
+                        url = ""
+
                 elif doc_type == "EXTENSION":
                     # Extract Extension-specific metadata
                     webpage_title = metadata.get('VisitedWebPageTitle', doc.title)
@@ -227,6 +248,7 @@ async def fetch_documents_by_ids(
                 "GITHUB_CONNECTOR": "GitHub (Selected)",
                 "YOUTUBE_VIDEO": "YouTube Videos (Selected)",
                 "DISCORD_CONNECTOR": "Discord (Selected)",
+                "JIRA_CONNECTOR": "Jira Issues (Selected)",
                 "EXTENSION": "Browser Extension (Selected)",
                 "CRAWLED_URL": "Web Pages (Selected)",
                 "FILE": "Files (Selected)"
@@ -741,6 +763,30 @@ async def fetch_relevant_documents(
                             }
                         )
 
+                elif connector == "JIRA_CONNECTOR":
+                    source_object, jira_chunks = await connector_service.search_jira(
+                        user_query=reformulated_query,
+                        user_id=user_id,
+                        search_space_id=search_space_id,
+                        top_k=top_k,
+                        search_mode=search_mode
+                    )
+
+                    # Add to sources and raw documents
+                    if source_object:
+                        all_sources.append(source_object)
+                    all_raw_documents.extend(jira_chunks)
+
+                    # Stream found document count
+                    if streaming_service and writer:
+                        writer(
+                            {
+                                "yield_value": streaming_service.format_terminal_info_delta(
+                                    f"🎫 Found {len(jira_chunks)} Jira issues related to your query"
+                                )
+                            }
+                        )
+
             except Exception as e:
                 error_message = f"Error searching connector {connector}: {str(e)}"
                 print(error_message)

surfsense_backend/app/agents/researcher/qna_agent/prompts.py

@@ -15,6 +15,8 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
 - YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
 - GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
 - LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
+- JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking)
+- DISCORD_CONNECTOR: "Discord server conversations and shared content" (personal community communications)
 - DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions)
 - TAVILY_API: "Tavily search API results" (personalized search results)
 - LINKUP_API: "Linkup search API results" (personalized search results)

surfsense_backend/app/agents/researcher/utils.py

@@ -33,6 +33,8 @@ def get_connector_emoji(connector_name: str) -> str:
         "NOTION_CONNECTOR": "📘",
         "GITHUB_CONNECTOR": "🐙",
         "LINEAR_CONNECTOR": "📊",
+        "JIRA_CONNECTOR": "🎫",
+        "DISCORD_CONNECTOR": "🗨️",
         "TAVILY_API": "🔍",
         "LINKUP_API": "🔗"
     }
@@ -50,6 +52,8 @@ def get_connector_friendly_name(connector_name: str) -> str:
         "NOTION_CONNECTOR": "Notion",
         "GITHUB_CONNECTOR": "GitHub",
         "LINEAR_CONNECTOR": "Linear",
+        "JIRA_CONNECTOR": "Jira",
+        "DISCORD_CONNECTOR": "Discord",
         "TAVILY_API": "Tavily Search",
         "LINKUP_API": "Linkup Search"
     }

surfsense_backend/app/connectors/test_jira_connector.py

@@ -0,0 +1,218 @@
+import unittest
+from unittest.mock import patch, Mock
+from datetime import datetime
+
+# Import the JiraConnector
+from .jira_connector import JiraConnector
+
+
+class TestJiraConnector(unittest.TestCase):
+
+    def setUp(self):
+        """Set up test fixtures."""
+        self.base_url = "https://test.atlassian.net"
+        self.token = "test_token"
+        self.connector = JiraConnector(base_url=self.base_url, personal_access_token=self.token)
+
+    def test_init(self):
+        """Test JiraConnector initialization."""
+        self.assertEqual(self.connector.base_url, self.base_url)
+        self.assertEqual(self.connector.personal_access_token, self.token)
+        self.assertEqual(self.connector.api_version, "3")
+
+    def test_init_with_trailing_slash(self):
+        """Test JiraConnector initialization with trailing slash in URL."""
+        connector = JiraConnector(base_url="https://test.atlassian.net/", personal_access_token=self.token)
+        self.assertEqual(connector.base_url, "https://test.atlassian.net")
+
+    def test_set_credentials(self):
+        """Test setting credentials."""
+        new_url = "https://newtest.atlassian.net/"
+        new_token = "new_token"
+        
+        self.connector.set_credentials(new_url, new_token)
+        
+        self.assertEqual(self.connector.base_url, "https://newtest.atlassian.net")
+        self.assertEqual(self.connector.personal_access_token, new_token)
+
+    def test_get_headers(self):
+        """Test header generation."""
+        headers = self.connector.get_headers()
+        
+        self.assertIn('Content-Type', headers)
+        self.assertIn('Authorization', headers)
+        self.assertIn('Accept', headers)
+        self.assertEqual(headers['Content-Type'], 'application/json')
+        self.assertEqual(headers['Accept'], 'application/json')
+        self.assertTrue(headers['Authorization'].startswith('Bearer '))
+
+    def test_get_headers_no_credentials(self):
+        """Test header generation without credentials."""
+        connector = JiraConnector()
+        
+        with self.assertRaises(ValueError) as context:
+            connector.get_headers()
+        
+        self.assertIn("Jira credentials not initialized", str(context.exception))
+
+    @patch('requests.get')
+    def test_make_api_request_success(self, mock_get):
+        """Test successful API request."""
+        mock_response = Mock()
+        mock_response.status_code = 200
+        mock_response.json.return_value = {"test": "data"}
+        mock_get.return_value = mock_response
+        
+        result = self.connector.make_api_request("test/endpoint")
+        
+        self.assertEqual(result, {"test": "data"})
+        mock_get.assert_called_once()
+
+    @patch('requests.get')
+    def test_make_api_request_failure(self, mock_get):
+        """Test failed API request."""
+        mock_response = Mock()
+        mock_response.status_code = 401
+        mock_response.text = "Unauthorized"
+        mock_get.return_value = mock_response
+        
+        with self.assertRaises(Exception) as context:
+            self.connector.make_api_request("test/endpoint")
+        
+        self.assertIn("API request failed with status code 401", str(context.exception))
+
+    @patch.object(JiraConnector, 'make_api_request')
+    def test_get_all_projects(self, mock_api_request):
+        """Test getting all projects."""
+        mock_api_request.return_value = {
+            "values": [
+                {"id": "1", "key": "TEST", "name": "Test Project"},
+                {"id": "2", "key": "DEMO", "name": "Demo Project"}
+            ]
+        }
+        
+        projects = self.connector.get_all_projects()
+        
+        self.assertEqual(len(projects), 2)
+        self.assertEqual(projects[0]["key"], "TEST")
+        self.assertEqual(projects[1]["key"], "DEMO")
+        mock_api_request.assert_called_once_with("project")
+
+    @patch.object(JiraConnector, 'make_api_request')
+    def test_get_all_issues(self, mock_api_request):
+        """Test getting all issues."""
+        mock_api_request.return_value = {
+            "issues": [
+                {
+                    "id": "1",
+                    "key": "TEST-1",
+                    "fields": {
+                        "summary": "Test Issue",
+                        "description": "Test Description",
+                        "status": {"name": "Open"},
+                        "priority": {"name": "High"},
+                        "issuetype": {"name": "Bug"},
+                        "project": {"key": "TEST"},
+                        "created": "2023-01-01T10:00:00.000+0000",
+                        "updated": "2023-01-01T12:00:00.000+0000"
+                    }
+                }
+            ],
+            "total": 1
+        }
+        
+        issues = self.connector.get_all_issues()
+        
+        self.assertEqual(len(issues), 1)
+        self.assertEqual(issues[0]["key"], "TEST-1")
+        self.assertEqual(issues[0]["fields"]["summary"], "Test Issue")
+
+    def test_format_issue(self):
+        """Test issue formatting."""
+        raw_issue = {
+            "id": "1",
+            "key": "TEST-1",
+            "fields": {
+                "summary": "Test Issue",
+                "description": "Test Description",
+                "status": {"name": "Open", "statusCategory": {"name": "To Do"}},
+                "priority": {"name": "High"},
+                "issuetype": {"name": "Bug"},
+                "project": {"key": "TEST"},
+                "created": "2023-01-01T10:00:00.000+0000",
+                "updated": "2023-01-01T12:00:00.000+0000",
+                "reporter": {
+                    "accountId": "123",
+                    "displayName": "John Doe",
+                    "emailAddress": "john@example.com"
+                },
+                "assignee": {
+                    "accountId": "456",
+                    "displayName": "Jane Smith",
+                    "emailAddress": "jane@example.com"
+                }
+            }
+        }
+        
+        formatted = self.connector.format_issue(raw_issue)
+        
+        self.assertEqual(formatted["id"], "1")
+        self.assertEqual(formatted["key"], "TEST-1")
+        self.assertEqual(formatted["title"], "Test Issue")
+        self.assertEqual(formatted["status"], "Open")
+        self.assertEqual(formatted["priority"], "High")
+        self.assertEqual(formatted["issue_type"], "Bug")
+        self.assertEqual(formatted["project"], "TEST")
+        self.assertEqual(formatted["reporter"]["display_name"], "John Doe")
+        self.assertEqual(formatted["assignee"]["display_name"], "Jane Smith")
+
+    def test_format_date(self):
+        """Test date formatting."""
+        iso_date = "2023-01-01T10:30:00.000+0000"
+        formatted_date = JiraConnector.format_date(iso_date)
+        
+        self.assertEqual(formatted_date, "2023-01-01 10:30:00")
+
+    def test_format_date_invalid(self):
+        """Test date formatting with invalid input."""
+        formatted_date = JiraConnector.format_date("invalid-date")
+        self.assertEqual(formatted_date, "invalid-date")
+        
+        formatted_date = JiraConnector.format_date("")
+        self.assertEqual(formatted_date, "Unknown date")
+        
+        formatted_date = JiraConnector.format_date(None)
+        self.assertEqual(formatted_date, "Unknown date")
+
+    def test_format_issue_to_markdown(self):
+        """Test issue to markdown conversion."""
+        formatted_issue = {
+            "key": "TEST-1",
+            "title": "Test Issue",
+            "status": "Open",
+            "priority": "High",
+            "issue_type": "Bug",
+            "project": "TEST",
+            "assignee": {"display_name": "Jane Smith"},
+            "reporter": {"display_name": "John Doe"},
+            "created_at": "2023-01-01T10:00:00.000+0000",
+            "updated_at": "2023-01-01T12:00:00.000+0000",
+            "description": "Test Description",
+            "comments": []
+        }
+        
+        markdown = self.connector.format_issue_to_markdown(formatted_issue)
+        
+        self.assertIn("# TEST-1: Test Issue", markdown)
+        self.assertIn("**Status:** Open", markdown)
+        self.assertIn("**Priority:** High", markdown)
+        self.assertIn("**Type:** Bug", markdown)
+        self.assertIn("**Project:** TEST", markdown)
+        self.assertIn("**Assignee:** Jane Smith", markdown)
+        self.assertIn("**Reporter:** John Doe", markdown)
+        self.assertIn("## Description", markdown)
+        self.assertIn("Test Description", markdown)
+
+
+if __name__ == '__main__':
+    unittest.main()

surfsense_backend/app/routes/search_source_connectors_routes.py

@@ -19,7 +19,7 @@ from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate
 from app.users import current_active_user
 from app.utils.check_ownership import check_ownership
 from pydantic import BaseModel, Field, ValidationError
-from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues, index_discord_messages
+from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues, index_discord_messages, index_jira_issues
 from app.connectors.github_connector import GitHubConnector
 from datetime import datetime, timedelta
 import logging
@@ -284,6 +284,7 @@ async def index_connector_content(
     - NOTION_CONNECTOR: Indexes pages from all accessible Notion pages
     - GITHUB_CONNECTOR: Indexes code and documentation from GitHub repositories
     - LINEAR_CONNECTOR: Indexes issues and comments from Linear
+    - JIRA_CONNECTOR: Indexes issues and comments from Jira
     - DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
     
     Args:
@@ -349,6 +350,12 @@ async def index_connector_content(
             background_tasks.add_task(run_linear_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
             response_message = "Linear indexing started in the background."
 
+        elif connector.connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
+            # Run indexing in background
+            logger.info(f"Triggering Jira indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}")
+            background_tasks.add_task(run_jira_indexing_with_new_session, connector_id, search_space_id, str(user.id), indexing_from, indexing_to)
+            response_message = "Jira indexing started in the background."
+
         elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
             # Run indexing in background
             logger.info(
@@ -647,4 +654,45 @@ async def run_discord_indexing(
         else:
             logger.error(f"Discord indexing failed or no documents processed: {error_or_warning}")
     except Exception as e:
-        logger.error(f"Error in background Discord indexing task: {str(e)}")
+        logger.error(f"Error in background Discord indexing task: {str(e)}")
+
+
+# Add new helper functions for Jira indexing
+async def run_jira_indexing_with_new_session(
+    connector_id: int,
+    search_space_id: int,
+    user_id: str,
+    start_date: str,
+    end_date: str
+):
+    """Wrapper to run Jira indexing with its own database session."""
+    logger.info(f"Background task started: Indexing Jira connector {connector_id} into space {search_space_id} from {start_date} to {end_date}")
+    async with async_session_maker() as session:
+        await run_jira_indexing(session, connector_id, search_space_id, user_id, start_date, end_date)
+    logger.info(f"Background task finished: Indexing Jira connector {connector_id}")
+
+async def run_jira_indexing(
+    session: AsyncSession,
+    connector_id: int,
+    search_space_id: int,
+    user_id: str,
+    start_date: str,
+    end_date: str
+):
+    """Runs the Jira indexing task and updates the timestamp."""
+    try:
+        indexed_count, error_message = await index_jira_issues(
+            session, connector_id, search_space_id, user_id, start_date, end_date, update_last_indexed=False
+        )
+        if error_message:
+            logger.error(f"Jira indexing failed for connector {connector_id}: {error_message}")
+            # Optionally update status in DB to indicate failure
+        else:
+            logger.info(f"Jira indexing successful for connector {connector_id}. Indexed {indexed_count} documents.")
+            # Update the last indexed timestamp only on success
+            await update_connector_last_indexed(session, connector_id)
+            await session.commit() # Commit timestamp update
+    except Exception as e:
+        await session.rollback()
+        logger.error(f"Critical error in run_jira_indexing for connector {connector_id}: {e}", exc_info=True)
+        # Optionally update status in DB to indicate failure

surfsense_backend/app/schemas/search_source_connector.py

@@ -101,6 +101,19 @@ class SearchSourceConnectorBase(BaseModel):
             # Ensure the bot token is not empty
             if not config.get("DISCORD_BOT_TOKEN"):
                 raise ValueError("DISCORD_BOT_TOKEN cannot be empty")
+        elif connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
+            # For JIRA_CONNECTOR, allow JIRA_PERSONAL_ACCESS_TOKEN and JIRA_BASE_URL
+            allowed_keys = ["JIRA_PERSONAL_ACCESS_TOKEN", "JIRA_BASE_URL"]
+            if set(config.keys()) != set(allowed_keys):
+                raise ValueError(f"For JIRA_CONNECTOR connector type, config must only contain these keys: {allowed_keys}")
+
+            # Ensure the token is not empty
+            if not config.get("JIRA_PERSONAL_ACCESS_TOKEN"):
+                raise ValueError("JIRA_PERSONAL_ACCESS_TOKEN cannot be empty")
+
+            # Ensure the base URL is not empty
+            if not config.get("JIRA_BASE_URL"):
+                raise ValueError("JIRA_BASE_URL cannot be empty")
 
         return config