diff --git a/README.md b/README.md
index 3b4ed36..916864b 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@
# SurfSense
-While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily, LinkUp), Slack, Linear, Notion, YouTube, GitHub, Discord and more to come.
+While tools like NotebookLM and Perplexity are impressive and highly effective for conducting research on any topic/query, SurfSense elevates this capability by integrating with your personal knowledge base. It is a highly customizable AI research agent, connected to external sources such as search engines (Tavily, LinkUp), Slack, Linear, Jira, Notion, YouTube, GitHub, Discord and more to come.
@@ -63,6 +63,7 @@ Open source and easy to deploy locally.
- Search Engines (Tavily, LinkUp)
- Slack
- Linear
+- Jira
- Notion
- Youtube Videos
- GitHub
diff --git a/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json b/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json
new file mode 100644
index 0000000..e744e3a
--- /dev/null
+++ b/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json
@@ -0,0 +1 @@
+{"2d0ec64d93969318101ee479b664221b32241665":{"files":{"surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx":["EHKKvlOK0vfy0GgHwlG/J2Bx5rw=",true]},"modified":1753426633288}}
\ No newline at end of file
diff --git a/surfsense_backend/alembic/versions/11_add_llm_config_table_and_relationships.py b/surfsense_backend/alembic/versions/11_add_llm_config_table_and_relationships.py
index f807f8b..0a6a107 100644
--- a/surfsense_backend/alembic/versions/11_add_llm_config_table_and_relationships.py
+++ b/surfsense_backend/alembic/versions/11_add_llm_config_table_and_relationships.py
@@ -20,70 +20,119 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema - add LiteLLMProvider enum, LLMConfig table and user LLM preferences."""
- # Check if enum type exists and create if it doesn't
- op.execute("""
+ # Create enum only if not exists
+ op.execute(
+ """
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'litellmprovider') THEN
- CREATE TYPE litellmprovider AS ENUM ('OPENAI', 'ANTHROPIC', 'GROQ', 'COHERE', 'HUGGINGFACE', 'AZURE_OPENAI', 'GOOGLE', 'AWS_BEDROCK', 'OLLAMA', 'MISTRAL', 'TOGETHER_AI', 'REPLICATE', 'PALM', 'VERTEX_AI', 'ANYSCALE', 'PERPLEXITY', 'DEEPINFRA', 'AI21', 'NLPCLOUD', 'ALEPH_ALPHA', 'PETALS', 'CUSTOM');
+ CREATE TYPE litellmprovider AS ENUM (
+ 'OPENAI', 'ANTHROPIC', 'GROQ', 'COHERE', 'HUGGINGFACE',
+ 'AZURE_OPENAI', 'GOOGLE', 'AWS_BEDROCK', 'OLLAMA', 'MISTRAL',
+ 'TOGETHER_AI', 'REPLICATE', 'PALM', 'VERTEX_AI', 'ANYSCALE',
+ 'PERPLEXITY', 'DEEPINFRA', 'AI21', 'NLPCLOUD', 'ALEPH_ALPHA',
+ 'PETALS', 'CUSTOM'
+ );
END IF;
END$$;
- """)
-
- # Create llm_configs table using raw SQL to avoid enum creation conflicts
- op.execute("""
- CREATE TABLE llm_configs (
- id SERIAL PRIMARY KEY,
- created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
- name VARCHAR(100) NOT NULL,
- provider litellmprovider NOT NULL,
- custom_provider VARCHAR(100),
- model_name VARCHAR(100) NOT NULL,
- api_key TEXT NOT NULL,
- api_base VARCHAR(500),
- litellm_params JSONB,
- user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE
- )
- """)
-
- # Create indexes
- op.create_index(op.f("ix_llm_configs_id"), "llm_configs", ["id"], unique=False)
- op.create_index(
- op.f("ix_llm_configs_created_at"), "llm_configs", ["created_at"], unique=False
+ """
)
- op.create_index(op.f("ix_llm_configs_name"), "llm_configs", ["name"], unique=False)
- # Add LLM preference columns to user table
- op.add_column("user", sa.Column("long_context_llm_id", sa.Integer(), nullable=True))
- op.add_column("user", sa.Column("fast_llm_id", sa.Integer(), nullable=True))
- op.add_column("user", sa.Column("strategic_llm_id", sa.Integer(), nullable=True))
+ # Create llm_configs table only if it doesn't already exist
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT FROM information_schema.tables
+ WHERE table_name = 'llm_configs'
+ ) THEN
+ CREATE TABLE llm_configs (
+ id SERIAL PRIMARY KEY,
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+ name VARCHAR(100) NOT NULL,
+ provider litellmprovider NOT NULL,
+ custom_provider VARCHAR(100),
+ model_name VARCHAR(100) NOT NULL,
+ api_key TEXT NOT NULL,
+ api_base VARCHAR(500),
+ litellm_params JSONB,
+ user_id UUID NOT NULL REFERENCES "user"(id) ON DELETE CASCADE
+ );
+ END IF;
+ END$$;
+ """
+ )
- # Create foreign key constraints for LLM preferences
- op.create_foreign_key(
- op.f("fk_user_long_context_llm_id_llm_configs"),
- "user",
- "llm_configs",
- ["long_context_llm_id"],
- ["id"],
- ondelete="SET NULL",
- )
- op.create_foreign_key(
- op.f("fk_user_fast_llm_id_llm_configs"),
- "user",
- "llm_configs",
- ["fast_llm_id"],
- ["id"],
- ondelete="SET NULL",
- )
- op.create_foreign_key(
- op.f("fk_user_strategic_llm_id_llm_configs"),
- "user",
- "llm_configs",
- ["strategic_llm_id"],
- ["id"],
- ondelete="SET NULL",
+ # Create indexes if they don't exist
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_indexes
+ WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_id'
+ ) THEN
+ CREATE INDEX ix_llm_configs_id ON llm_configs(id);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_indexes
+ WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_created_at'
+ ) THEN
+ CREATE INDEX ix_llm_configs_created_at ON llm_configs(created_at);
+ END IF;
+
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_indexes
+ WHERE tablename = 'llm_configs' AND indexname = 'ix_llm_configs_name'
+ ) THEN
+ CREATE INDEX ix_llm_configs_name ON llm_configs(name);
+ END IF;
+ END$$;
+ """
)
+ # Safely add columns to user table
+ bind = op.get_bind()
+ inspector = sa.inspect(bind)
+ existing_columns = [col["name"] for col in inspector.get_columns("user")]
+
+ with op.batch_alter_table("user") as batch_op:
+ if "long_context_llm_id" not in existing_columns:
+ batch_op.add_column(
+ sa.Column("long_context_llm_id", sa.Integer(), nullable=True)
+ )
+ batch_op.create_foreign_key(
+ op.f("fk_user_long_context_llm_id_llm_configs"),
+ "llm_configs",
+ ["long_context_llm_id"],
+ ["id"],
+ ondelete="SET NULL",
+ )
+
+ if "fast_llm_id" not in existing_columns:
+ batch_op.add_column(sa.Column("fast_llm_id", sa.Integer(), nullable=True))
+ batch_op.create_foreign_key(
+ op.f("fk_user_fast_llm_id_llm_configs"),
+ "llm_configs",
+ ["fast_llm_id"],
+ ["id"],
+ ondelete="SET NULL",
+ )
+
+ if "strategic_llm_id" not in existing_columns:
+ batch_op.add_column(
+ sa.Column("strategic_llm_id", sa.Integer(), nullable=True)
+ )
+ batch_op.create_foreign_key(
+ op.f("fk_user_strategic_llm_id_llm_configs"),
+ "llm_configs",
+ ["strategic_llm_id"],
+ ["id"],
+ ondelete="SET NULL",
+ )
+
def downgrade() -> None:
"""Downgrade schema - remove LLMConfig table and user LLM preferences."""
diff --git a/surfsense_backend/alembic/versions/12_add_logs_table.py b/surfsense_backend/alembic/versions/12_add_logs_table.py
index 9e12fe6..947c77c 100644
--- a/surfsense_backend/alembic/versions/12_add_logs_table.py
+++ b/surfsense_backend/alembic/versions/12_add_logs_table.py
@@ -6,6 +6,8 @@ Revises: 11
from collections.abc import Sequence
+from sqlalchemy import inspect
+
from alembic import op
# revision identifiers, used by Alembic.
@@ -18,47 +20,73 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
"""Upgrade schema - add LogLevel and LogStatus enums and logs table."""
- # Create LogLevel enum
- op.execute("""
- CREATE TYPE loglevel AS ENUM ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL')
- """)
+ # Create LogLevel enum if it doesn't exist
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'loglevel') THEN
+ CREATE TYPE loglevel AS ENUM ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL');
+ END IF;
+ END$$;
+ """
+ )
- # Create LogStatus enum
- op.execute("""
- CREATE TYPE logstatus AS ENUM ('IN_PROGRESS', 'SUCCESS', 'FAILED')
- """)
+ # Create LogStatus enum if it doesn't exist
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (SELECT 1 FROM pg_type WHERE typname = 'logstatus') THEN
+ CREATE TYPE logstatus AS ENUM ('IN_PROGRESS', 'SUCCESS', 'FAILED');
+ END IF;
+ END$$;
+ """
+ )
- # Create logs table
- op.execute("""
- CREATE TABLE logs (
+ # Create logs table if it doesn't exist
+ op.execute(
+ """
+ CREATE TABLE IF NOT EXISTS logs (
id SERIAL PRIMARY KEY,
- created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
level loglevel NOT NULL,
status logstatus NOT NULL,
message TEXT NOT NULL,
source VARCHAR(200),
log_metadata JSONB DEFAULT '{}',
search_space_id INTEGER NOT NULL REFERENCES searchspaces(id) ON DELETE CASCADE
- )
- """)
+ );
+ """
+ )
- # Create indexes
- op.create_index(op.f("ix_logs_id"), "logs", ["id"], unique=False)
- op.create_index(op.f("ix_logs_created_at"), "logs", ["created_at"], unique=False)
- op.create_index(op.f("ix_logs_level"), "logs", ["level"], unique=False)
- op.create_index(op.f("ix_logs_status"), "logs", ["status"], unique=False)
- op.create_index(op.f("ix_logs_source"), "logs", ["source"], unique=False)
+ # Get existing indexes
+ conn = op.get_bind()
+ inspector = inspect(conn)
+ existing_indexes = [idx["name"] for idx in inspector.get_indexes("logs")]
+
+ # Create indexes only if they don't already exist
+ if "ix_logs_id" not in existing_indexes:
+ op.create_index("ix_logs_id", "logs", ["id"])
+ if "ix_logs_created_at" not in existing_indexes:
+ op.create_index("ix_logs_created_at", "logs", ["created_at"])
+ if "ix_logs_level" not in existing_indexes:
+ op.create_index("ix_logs_level", "logs", ["level"])
+ if "ix_logs_status" not in existing_indexes:
+ op.create_index("ix_logs_status", "logs", ["status"])
+ if "ix_logs_source" not in existing_indexes:
+ op.create_index("ix_logs_source", "logs", ["source"])
def downgrade() -> None:
"""Downgrade schema - remove logs table and enums."""
# Drop indexes
- op.drop_index(op.f("ix_logs_source"), table_name="logs")
- op.drop_index(op.f("ix_logs_status"), table_name="logs")
- op.drop_index(op.f("ix_logs_level"), table_name="logs")
- op.drop_index(op.f("ix_logs_created_at"), table_name="logs")
- op.drop_index(op.f("ix_logs_id"), table_name="logs")
+ op.drop_index("ix_logs_source", table_name="logs")
+ op.drop_index("ix_logs_status", table_name="logs")
+ op.drop_index("ix_logs_level", table_name="logs")
+ op.drop_index("ix_logs_created_at", table_name="logs")
+ op.drop_index("ix_logs_id", table_name="logs")
# Drop logs table
op.drop_table("logs")
diff --git a/surfsense_backend/alembic/versions/13_add_jira_connector_enums.py b/surfsense_backend/alembic/versions/13_add_jira_connector_enums.py
new file mode 100644
index 0000000..18930b4
--- /dev/null
+++ b/surfsense_backend/alembic/versions/13_add_jira_connector_enums.py
@@ -0,0 +1,61 @@
+"""Add JIRA_CONNECTOR to enums
+
+Revision ID: 13
+Revises: 12
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "13"
+down_revision: str | None = "12"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+def upgrade() -> None:
+ """Safely add 'JIRA_CONNECTOR' to enum types if missing."""
+
+ # Add to searchsourceconnectortype enum
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_type t
+ JOIN pg_enum e ON t.oid = e.enumtypid
+ WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'JIRA_CONNECTOR'
+ ) THEN
+ ALTER TYPE searchsourceconnectortype ADD VALUE 'JIRA_CONNECTOR';
+ END IF;
+ END
+ $$;
+ """
+ )
+
+ # Add to documenttype enum
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_type t
+ JOIN pg_enum e ON t.oid = e.enumtypid
+ WHERE t.typname = 'documenttype' AND e.enumlabel = 'JIRA_CONNECTOR'
+ ) THEN
+ ALTER TYPE documenttype ADD VALUE 'JIRA_CONNECTOR';
+ END IF;
+ END
+ $$;
+ """
+ )
+
+
+def downgrade() -> None:
+ """
+ Downgrade logic not implemented since PostgreSQL
+ does not support removing enum values.
+ """
+ pass
diff --git a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
index e54418c..235908b 100644
--- a/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
+++ b/surfsense_backend/alembic/versions/1_add_github_connector_enum.py
@@ -25,7 +25,23 @@ def upgrade() -> None:
# Manually add the command to add the enum value
# Note: It's generally better to let autogenerate handle this, but we're bypassing it
- op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'GITHUB_CONNECTOR'")
+ op.execute(
+ """
+DO $$
+BEGIN
+ IF NOT EXISTS (
+ SELECT 1
+ FROM pg_enum
+ WHERE enumlabel = 'GITHUB_CONNECTOR'
+ AND enumtypid = (
+ SELECT oid FROM pg_type WHERE typname = 'searchsourceconnectortype'
+ )
+ ) THEN
+ ALTER TYPE searchsourceconnectortype ADD VALUE 'GITHUB_CONNECTOR';
+ END IF;
+END$$;
+"""
+ )
# Pass for the rest, as autogenerate didn't run to add other schema details
pass
diff --git a/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py b/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py
index ffe6293..b044556 100644
--- a/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py
+++ b/surfsense_backend/alembic/versions/2_add_linear_connector_enum.py
@@ -17,14 +17,25 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
- # ### commands auto generated by Alembic - please adjust! ###
+ op.execute(
+ """
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_enum
+ WHERE enumlabel = 'LINEAR_CONNECTOR'
+ AND enumtypid = (
+ SELECT oid FROM pg_type WHERE typname = 'searchsourceconnectortype'
+ )
+ ) THEN
+ ALTER TYPE searchsourceconnectortype ADD VALUE 'LINEAR_CONNECTOR';
+ END IF;
+ END$$;
+ """
+ )
- # Manually add the command to add the enum value
- op.execute("ALTER TYPE searchsourceconnectortype ADD VALUE 'LINEAR_CONNECTOR'")
- # Pass for the rest, as autogenerate didn't run to add other schema details
- pass
- # ### end Alembic commands ###
+#
def downgrade() -> None:
diff --git a/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py b/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py
index 8c4625b..001cac3 100644
--- a/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py
+++ b/surfsense_backend/alembic/versions/3_add_linear_connector_to_documenttype_.py
@@ -22,7 +22,22 @@ NEW_VALUE = "LINEAR_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema."""
- op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'")
+ op.execute(
+ f"""
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_enum
+ WHERE enumlabel = '{NEW_VALUE}'
+ AND enumtypid = (
+ SELECT oid FROM pg_type WHERE typname = '{ENUM_NAME}'
+ )
+ ) THEN
+ ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
+ END IF;
+ END$$;
+ """
+ )
# Warning: This will delete all rows with the new value
diff --git a/surfsense_backend/alembic/versions/6_change_podcast_content_to_transcript.py b/surfsense_backend/alembic/versions/6_change_podcast_content_to_transcript.py
index 8a23e86..54c265d 100644
--- a/surfsense_backend/alembic/versions/6_change_podcast_content_to_transcript.py
+++ b/surfsense_backend/alembic/versions/6_change_podcast_content_to_transcript.py
@@ -8,6 +8,7 @@ Revises: 5
from collections.abc import Sequence
import sqlalchemy as sa
+from sqlalchemy import inspect
from sqlalchemy.dialects.postgresql import JSON
from alembic import op
@@ -20,21 +21,28 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
- # Drop the old column and create a new one with the new name and type
- # We need to do this because PostgreSQL doesn't support direct column renames with type changes
- op.add_column(
- "podcasts",
- sa.Column("podcast_transcript", JSON, nullable=False, server_default="{}"),
- )
+ bind = op.get_bind()
+ inspector = inspect(bind)
- # Copy data from old column to new column
- # Convert text to JSON by storing it as a JSON string value
- op.execute(
- "UPDATE podcasts SET podcast_transcript = jsonb_build_object('text', podcast_content) WHERE podcast_content != ''"
- )
+ columns = [col["name"] for col in inspector.get_columns("podcasts")]
+ if "podcast_transcript" not in columns:
+ op.add_column(
+ "podcasts",
+ sa.Column("podcast_transcript", JSON, nullable=False, server_default="{}"),
+ )
- # Drop the old column
- op.drop_column("podcasts", "podcast_content")
+ # Copy data from old column to new column
+ op.execute(
+ """
+ UPDATE podcasts
+ SET podcast_transcript = jsonb_build_object('text', podcast_content)
+ WHERE podcast_content != ''
+ """
+ )
+
+ # Drop the old column only if it exists
+ if "podcast_content" in columns:
+ op.drop_column("podcasts", "podcast_content")
def downgrade() -> None:
diff --git a/surfsense_backend/alembic/versions/7_remove_is_generated_column.py b/surfsense_backend/alembic/versions/7_remove_is_generated_column.py
index 0416944..17238c3 100644
--- a/surfsense_backend/alembic/versions/7_remove_is_generated_column.py
+++ b/surfsense_backend/alembic/versions/7_remove_is_generated_column.py
@@ -8,6 +8,7 @@ Revises: 6
from collections.abc import Sequence
import sqlalchemy as sa
+from sqlalchemy import inspect
from alembic import op
@@ -19,8 +20,14 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
- # Drop the is_generated column
- op.drop_column("podcasts", "is_generated")
+ # Get the current database connection
+ bind = op.get_bind()
+ inspector = inspect(bind)
+
+ # Check if the column exists before attempting to drop it
+ columns = [col["name"] for col in inspector.get_columns("podcasts")]
+ if "is_generated" in columns:
+ op.drop_column("podcasts", "is_generated")
def downgrade() -> None:
diff --git a/surfsense_backend/alembic/versions/8_add_content_hash_to_documents.py b/surfsense_backend/alembic/versions/8_add_content_hash_to_documents.py
index 10f68d4..6fa65a8 100644
--- a/surfsense_backend/alembic/versions/8_add_content_hash_to_documents.py
+++ b/surfsense_backend/alembic/versions/8_add_content_hash_to_documents.py
@@ -7,6 +7,7 @@ Revises: 7
from collections.abc import Sequence
import sqlalchemy as sa
+from sqlalchemy import inspect
from alembic import op
@@ -18,44 +19,53 @@ depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
- # Add content_hash column as nullable first to handle existing data
- op.add_column("documents", sa.Column("content_hash", sa.String(), nullable=True))
+ bind = op.get_bind()
+ inspector = inspect(bind)
+ columns = [col["name"] for col in inspector.get_columns("documents")]
- # Update existing documents to generate content hashes
- # Using SHA-256 hash of the content column with proper UTF-8 encoding
- op.execute("""
- UPDATE documents
- SET content_hash = encode(sha256(convert_to(content, 'UTF8')), 'hex')
- WHERE content_hash IS NULL
- """)
-
- # Handle duplicate content hashes by keeping only the oldest document for each hash
- # Delete newer documents with duplicate content hashes
- op.execute("""
- DELETE FROM documents
- WHERE id NOT IN (
- SELECT MIN(id)
- FROM documents
- GROUP BY content_hash
+ # Only add the column if it doesn't already exist
+ if "content_hash" not in columns:
+ op.add_column(
+ "documents", sa.Column("content_hash", sa.String(), nullable=True)
)
- """)
- # Now alter the column to match the model: nullable=False, index=True, unique=True
- op.alter_column(
- "documents", "content_hash", existing_type=sa.String(), nullable=False
- )
- op.create_index(
- op.f("ix_documents_content_hash"), "documents", ["content_hash"], unique=False
- )
- op.create_unique_constraint(
- op.f("uq_documents_content_hash"), "documents", ["content_hash"]
- )
+ # Populate the content_hash column
+ op.execute(
+ """
+ UPDATE documents
+ SET content_hash = encode(sha256(convert_to(content, 'UTF8')), 'hex')
+ WHERE content_hash IS NULL
+ """
+ )
+
+ op.execute(
+ """
+ DELETE FROM documents
+ WHERE id NOT IN (
+ SELECT MIN(id)
+ FROM documents
+ GROUP BY content_hash
+ )
+ """
+ )
+
+ op.alter_column(
+ "documents", "content_hash", existing_type=sa.String(), nullable=False
+ )
+ op.create_index(
+ op.f("ix_documents_content_hash"),
+ "documents",
+ ["content_hash"],
+ unique=False,
+ )
+ op.create_unique_constraint(
+ op.f("uq_documents_content_hash"), "documents", ["content_hash"]
+ )
+ else:
+ print("Column 'content_hash' already exists. Skipping column creation.")
def downgrade() -> None:
- # Remove constraints and index first
op.drop_constraint(op.f("uq_documents_content_hash"), "documents", type_="unique")
op.drop_index(op.f("ix_documents_content_hash"), table_name="documents")
-
- # Remove content_hash column from documents table
op.drop_column("documents", "content_hash")
diff --git a/surfsense_backend/alembic/versions/9_add_discord_connector_enum_and_documenttype.py b/surfsense_backend/alembic/versions/9_add_discord_connector_enum_and_documenttype.py
index 8be1e39..ed6f238 100644
--- a/surfsense_backend/alembic/versions/9_add_discord_connector_enum_and_documenttype.py
+++ b/surfsense_backend/alembic/versions/9_add_discord_connector_enum_and_documenttype.py
@@ -22,11 +22,38 @@ DOCUMENT_NEW_VALUE = "DISCORD_CONNECTOR"
def upgrade() -> None:
- """Upgrade schema - add DISCORD_CONNECTOR to connector and document enum."""
- # Add DISCORD_CONNECTOR to searchsourceconnectortype
- op.execute(f"ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}'")
- # Add DISCORD_CONNECTOR to documenttype
- op.execute(f"ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}'")
+ """Upgrade schema - add DISCORD_CONNECTOR to connector and document enum safely."""
+ # Add DISCORD_CONNECTOR to searchsourceconnectortype only if not exists
+ op.execute(
+ f"""
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_enum
+ WHERE enumlabel = '{CONNECTOR_NEW_VALUE}'
+ AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{CONNECTOR_ENUM}')
+ ) THEN
+ ALTER TYPE {CONNECTOR_ENUM} ADD VALUE '{CONNECTOR_NEW_VALUE}';
+ END IF;
+ END$$;
+ """
+ )
+
+ # Add DISCORD_CONNECTOR to documenttype only if not exists
+ op.execute(
+ f"""
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_enum
+ WHERE enumlabel = '{DOCUMENT_NEW_VALUE}'
+ AND enumtypid = (SELECT oid FROM pg_type WHERE typname = '{DOCUMENT_ENUM}')
+ ) THEN
+ ALTER TYPE {DOCUMENT_ENUM} ADD VALUE '{DOCUMENT_NEW_VALUE}';
+ END IF;
+ END$$;
+ """
+ )
def downgrade() -> None:
diff --git a/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py b/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py
index 3c7b3a7..0ef43f6 100644
--- a/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py
+++ b/surfsense_backend/alembic/versions/e55302644c51_add_github_connector_to_documenttype_.py
@@ -1,10 +1,3 @@
-"""Add GITHUB_CONNECTOR to DocumentType enum
-
-Revision ID: e55302644c51
-Revises: 1
-
-"""
-
from collections.abc import Sequence
from alembic import op
@@ -16,23 +9,34 @@ branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
# Define the ENUM type name and the new value
-ENUM_NAME = "documenttype" # Make sure this matches the name in your DB (usually lowercase class name)
+ENUM_NAME = "documenttype"
NEW_VALUE = "GITHUB_CONNECTOR"
def upgrade() -> None:
"""Upgrade schema."""
- op.execute(f"ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}'")
+ op.execute(
+ f"""
+ DO $$
+ BEGIN
+ IF NOT EXISTS (
+ SELECT 1 FROM pg_enum
+ WHERE enumlabel = '{NEW_VALUE}'
+ AND enumtypid = (
+ SELECT oid FROM pg_type WHERE typname = '{ENUM_NAME}'
+ )
+ ) THEN
+ ALTER TYPE {ENUM_NAME} ADD VALUE '{NEW_VALUE}';
+ END IF;
+ END$$;
+ """
+ )
-# Warning: This will delete all rows with the new value
def downgrade() -> None:
"""Downgrade schema - remove GITHUB_CONNECTOR from enum."""
-
- # The old type name
old_enum_name = f"{ENUM_NAME}_old"
- # Enum values *before* GITHUB_CONNECTOR was added
old_values = (
"EXTENSION",
"CRAWLED_URL",
@@ -43,25 +47,21 @@ def downgrade() -> None:
)
old_values_sql = ", ".join([f"'{v}'" for v in old_values])
- # Table and column names (adjust if different)
table_name = "documents"
column_name = "document_type"
- # 1. Rename the current enum type
- op.execute(f"ALTER TYPE {ENUM_NAME} RENAME TO {old_enum_name}")
+ # 1. Create the new enum type with the old values
+ op.execute(f"CREATE TYPE {old_enum_name} AS ENUM({old_values_sql})")
- # 2. Create the new enum type with the old values
- op.execute(f"CREATE TYPE {ENUM_NAME} AS ENUM({old_values_sql})")
-
- # 3. Update the table:
+ # 2. Delete rows using the new value
op.execute(f"DELETE FROM {table_name} WHERE {column_name}::text = '{NEW_VALUE}'")
- # 4. Alter the column to use the new enum type (casting old values)
+ # 3. Alter the column to use the old enum type
op.execute(
f"ALTER TABLE {table_name} ALTER COLUMN {column_name} "
- f"TYPE {ENUM_NAME} USING {column_name}::text::{ENUM_NAME}"
+ f"TYPE {old_enum_name} USING {column_name}::text::{old_enum_name}"
)
- # 5. Drop the old enum type
- op.execute(f"DROP TYPE {old_enum_name}")
- # ### end Alembic commands ###
+ # 4. Drop the current enum type and rename the old one
+ op.execute(f"DROP TYPE {ENUM_NAME}")
+ op.execute(f"ALTER TYPE {old_enum_name} RENAME TO {ENUM_NAME}")
diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py
index 67550d6..9831115 100644
--- a/surfsense_backend/app/agents/researcher/nodes.py
+++ b/surfsense_backend/app/agents/researcher/nodes.py
@@ -84,9 +84,9 @@ async def fetch_documents_by_ids(
"document": {
"id": doc.id,
"title": doc.title,
- "document_type": doc.document_type.value
- if doc.document_type
- else "UNKNOWN",
+ "document_type": (
+ doc.document_type.value if doc.document_type else "UNKNOWN"
+ ),
"metadata": doc.document_metadata or {},
},
"source": doc.document_type.value if doc.document_type else "UNKNOWN",
@@ -186,9 +186,11 @@ async def fetch_documents_by_ids(
title = f"GitHub: {doc.title}"
description = metadata.get(
"description",
- doc.content[:100] + "..."
- if len(doc.content) > 100
- else doc.content,
+ (
+ doc.content[:100] + "..."
+ if len(doc.content) > 100
+ else doc.content
+ ),
)
url = metadata.get("url", "")
@@ -204,9 +206,11 @@ async def fetch_documents_by_ids(
description = metadata.get(
"description",
- doc.content[:100] + "..."
- if len(doc.content) > 100
- else doc.content,
+ (
+ doc.content[:100] + "..."
+ if len(doc.content) > 100
+ else doc.content
+ ),
)
url = (
f"https://www.youtube.com/watch?v={video_id}"
@@ -238,6 +242,35 @@ async def fetch_documents_by_ids(
else:
url = ""
+ elif doc_type == "JIRA_CONNECTOR":
+ # Extract Jira-specific metadata
+ issue_key = metadata.get("issue_key", "Unknown Issue")
+ issue_title = metadata.get("issue_title", "Untitled Issue")
+ status = metadata.get("status", "")
+ priority = metadata.get("priority", "")
+ issue_type = metadata.get("issue_type", "")
+
+ title = f"Jira: {issue_key} - {issue_title}"
+ if status:
+ title += f" ({status})"
+
+ description = (
+ doc.content[:100] + "..."
+ if len(doc.content) > 100
+ else doc.content
+ )
+ if priority:
+ description += f" | Priority: {priority}"
+ if issue_type:
+ description += f" | Type: {issue_type}"
+
+ # Construct Jira URL if we have the base URL
+ base_url = metadata.get("base_url", "")
+ if base_url and issue_key:
+ url = f"{base_url}/browse/{issue_key}"
+ else:
+ url = ""
+
elif doc_type == "EXTENSION":
# Extract Extension-specific metadata
webpage_title = metadata.get("VisitedWebPageTitle", doc.title)
@@ -268,9 +301,11 @@ async def fetch_documents_by_ids(
"og:description",
metadata.get(
"ogDescription",
- doc.content[:100] + "..."
- if len(doc.content) > 100
- else doc.content,
+ (
+ doc.content[:100] + "..."
+ if len(doc.content) > 100
+ else doc.content
+ ),
),
)
url = metadata.get("url", "")
@@ -301,6 +336,7 @@ async def fetch_documents_by_ids(
"GITHUB_CONNECTOR": "GitHub (Selected)",
"YOUTUBE_VIDEO": "YouTube Videos (Selected)",
"DISCORD_CONNECTOR": "Discord (Selected)",
+ "JIRA_CONNECTOR": "Jira Issues (Selected)",
"EXTENSION": "Browser Extension (Selected)",
"CRAWLED_URL": "Web Pages (Selected)",
"FILE": "Files (Selected)",
@@ -376,10 +412,10 @@ async def write_answer_outline(
# Create the human message content
human_message_content = f"""
Now Please create an answer outline for the following query:
-
+
User Query: {reformulated_query}
Number of Sections: {num_sections}
-
+
Remember to format your response as valid JSON exactly matching this structure:
{{
"answer_outline": [
@@ -393,7 +429,7 @@ async def write_answer_outline(
}}
]
}}
-
+
Your output MUST be valid JSON in exactly this format. Do not include any other text or explanation.
"""
@@ -802,7 +838,9 @@ async def fetch_relevant_documents(
source_object,
linkup_chunks,
) = await connector_service.search_linkup(
- user_query=reformulated_query, user_id=user_id, mode=linkup_mode
+ user_query=reformulated_query,
+ user_id=user_id,
+ mode=linkup_mode,
)
# Add to sources and raw documents
@@ -845,6 +883,30 @@ async def fetch_relevant_documents(
}
)
+ elif connector == "JIRA_CONNECTOR":
+ source_object, jira_chunks = await connector_service.search_jira(
+ user_query=reformulated_query,
+ user_id=user_id,
+ search_space_id=search_space_id,
+ top_k=top_k,
+ search_mode=search_mode,
+ )
+
+ # Add to sources and raw documents
+ if source_object:
+ all_sources.append(source_object)
+ all_raw_documents.extend(jira_chunks)
+
+ # Stream found document count
+ if streaming_service and writer:
+ writer(
+ {
+ "yield_value": streaming_service.format_terminal_info_delta(
+ f"🎫 Found {len(jira_chunks)} Jira issues related to your query"
+ )
+ }
+ )
+
except Exception as e:
error_message = f"Error searching connector {connector}: {e!s}"
print(error_message)
@@ -1214,7 +1276,7 @@ async def process_sections(
# Combine the results into a final report with section titles
final_report = []
- for _, (section, content) in enumerate(
+ for _i, (section, content) in enumerate(
zip(answer_outline.answer_outline, processed_results, strict=False)
):
# Skip adding the section header since the content already contains the title
@@ -1725,11 +1787,11 @@ async def generate_further_questions(
# Create the human message content
human_message_content = f"""
{chat_history_xml}
-
+
{documents_xml}
-
+
Based on the chat history and available documents above, generate 3-5 contextually relevant follow-up questions that would naturally extend the conversation and provide additional value to the user. Make sure the questions can be reasonably answered using the available documents or knowledge base.
-
+
Your response MUST be valid JSON in exactly this format:
{{
"further_questions": [
@@ -1743,7 +1805,7 @@ async def generate_further_questions(
}}
]
}}
-
+
Do not include any other text or explanation. Only return the JSON.
"""
diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py
index eed0722..3f4d975 100644
--- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py
+++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py
@@ -15,7 +15,8 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
- YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos)
- GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions)
- LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management)
-- DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions)
+- JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking)
+- DISCORD_CONNECTOR: "Discord server conversations and shared content" (personal community communications)
- TAVILY_API: "Tavily search API results" (personalized search results)
- LINKUP_API: "Linkup search API results" (personalized search results)
@@ -71,7 +72,7 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel
Python's asyncio library provides tools for writing concurrent code using the async/await syntax. It's particularly useful for I/O-bound and high-level structured network code.
-
+
12
diff --git a/surfsense_backend/app/agents/researcher/utils.py b/surfsense_backend/app/agents/researcher/utils.py
index 908b3ab..e26788c 100644
--- a/surfsense_backend/app/agents/researcher/utils.py
+++ b/surfsense_backend/app/agents/researcher/utils.py
@@ -43,6 +43,8 @@ def get_connector_emoji(connector_name: str) -> str:
"NOTION_CONNECTOR": "📘",
"GITHUB_CONNECTOR": "🐙",
"LINEAR_CONNECTOR": "📊",
+ "JIRA_CONNECTOR": "🎫",
+ "DISCORD_CONNECTOR": "🗨️",
"TAVILY_API": "🔍",
"LINKUP_API": "🔗",
}
@@ -60,6 +62,8 @@ def get_connector_friendly_name(connector_name: str) -> str:
"NOTION_CONNECTOR": "Notion",
"GITHUB_CONNECTOR": "GitHub",
"LINEAR_CONNECTOR": "Linear",
+ "JIRA_CONNECTOR": "Jira",
+ "DISCORD_CONNECTOR": "Discord",
"TAVILY_API": "Tavily Search",
"LINKUP_API": "Linkup Search",
}
diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py
new file mode 100644
index 0000000..ef0e003
--- /dev/null
+++ b/surfsense_backend/app/connectors/jira_connector.py
@@ -0,0 +1,487 @@
+"""
+Jira Connector Module
+
+A module for retrieving data from Jira.
+Allows fetching issue lists and their comments, projects and more.
+"""
+
+import base64
+from datetime import datetime
+from typing import Any
+
+import requests
+
+
+class JiraConnector:
+ """Class for retrieving data from Jira."""
+
+ def __init__(
+ self,
+ base_url: str | None = None,
+ email: str | None = None,
+ api_token: str | None = None,
+ ):
+ """
+ Initialize the JiraConnector class.
+
+ Args:
+ base_url: Jira instance base URL (e.g., 'https://yourcompany.atlassian.net') (optional)
+ email: Jira account email address (optional)
+ api_token: Jira API token (optional)
+ """
+ self.base_url = base_url.rstrip("/") if base_url else None
+ self.email = email
+ self.api_token = api_token
+ self.api_version = "3" # Jira Cloud API version
+
+ def set_credentials(self, base_url: str, email: str, api_token: str) -> None:
+ """
+ Set the Jira credentials.
+
+ Args:
+ base_url: Jira instance base URL
+ email: Jira account email address
+ api_token: Jira API token
+ """
+ self.base_url = base_url.rstrip("/")
+ self.email = email
+ self.api_token = api_token
+
+ def set_email(self, email: str) -> None:
+ """
+ Set the Jira account email.
+
+ Args:
+ email: Jira account email address
+ """
+ self.email = email
+
+ def set_api_token(self, api_token: str) -> None:
+ """
+ Set the Jira API token.
+
+ Args:
+ api_token: Jira API token
+ """
+ self.api_token = api_token
+
+ def get_headers(self) -> dict[str, str]:
+ """
+ Get headers for Jira API requests using Basic Authentication.
+
+ Returns:
+ Dictionary of headers
+
+ Raises:
+ ValueError: If email, api_token, or base_url have not been set
+ """
+ if not all([self.base_url, self.email, self.api_token]):
+ raise ValueError(
+ "Jira credentials not initialized. Call set_credentials() first."
+ )
+
+ # Create Basic Auth header using email:api_token
+ auth_str = f"{self.email}:{self.api_token}"
+ auth_bytes = auth_str.encode("utf-8")
+ auth_header = "Basic " + base64.b64encode(auth_bytes).decode("ascii")
+
+ return {
+ "Content-Type": "application/json",
+ "Authorization": auth_header,
+ "Accept": "application/json",
+ }
+
+ def make_api_request(
+ self, endpoint: str, params: dict[str, Any] | None = None
+ ) -> dict[str, Any]:
+ """
+ Make a request to the Jira API.
+
+ Args:
+ endpoint: API endpoint (without base URL)
+ params: Query parameters for the request (optional)
+
+ Returns:
+ Response data from the API
+
+ Raises:
+ ValueError: If email, api_token, or base_url have not been set
+ Exception: If the API request fails
+ """
+ if not all([self.base_url, self.email, self.api_token]):
+ raise ValueError(
+ "Jira credentials not initialized. Call set_credentials() first."
+ )
+
+ url = f"{self.base_url}/rest/api/{self.api_version}/{endpoint}"
+ headers = self.get_headers()
+
+ response = requests.get(url, headers=headers, params=params, timeout=500)
+
+ if response.status_code == 200:
+ return response.json()
+ else:
+ raise Exception(
+ f"API request failed with status code {response.status_code}: {response.text}"
+ )
+
+ def get_all_projects(self) -> dict[str, Any]:
+ """
+ Fetch all projects from Jira.
+
+ Returns:
+ List of project objects
+
+ Raises:
+ ValueError: If credentials have not been set
+ Exception: If the API request fails
+ """
+ return self.make_api_request("project/search")
+
+ def get_all_issues(self, project_key: str | None = None) -> list[dict[str, Any]]:
+ """
+ Fetch all issues from Jira.
+
+ Args:
+ project_key: Optional project key to filter issues (e.g., 'PROJ')
+
+ Returns:
+ List of issue objects
+
+ Raises:
+ ValueError: If credentials have not been set
+ Exception: If the API request fails
+ """
+ jql = "ORDER BY created DESC"
+ if project_key:
+ jql = f'project = "{project_key}" ' + jql
+
+ fields = [
+ "summary",
+ "description",
+ "status",
+ "assignee",
+ "reporter",
+ "created",
+ "updated",
+ "priority",
+ "issuetype",
+ "project",
+ ]
+
+ params = {
+ "jql": jql,
+ "fields": ",".join(fields),
+ "maxResults": 100,
+ "startAt": 0,
+ }
+
+ all_issues = []
+ start_at = 0
+
+ while True:
+ params["startAt"] = start_at
+ result = self.make_api_request("search", params)
+
+ if not isinstance(result, dict) or "issues" not in result:
+ raise Exception("Invalid response from Jira API")
+
+ issues = result["issues"]
+ all_issues.extend(issues)
+
+ print(f"Fetched {len(issues)} issues (startAt={start_at})")
+
+ total = result.get("total", 0)
+ if start_at + len(issues) >= total:
+ break
+
+ start_at += len(issues)
+
+ return all_issues
+
+ def get_issues_by_date_range(
+ self,
+ start_date: str,
+ end_date: str,
+ include_comments: bool = True,
+ project_key: str | None = None,
+ ) -> tuple[list[dict[str, Any]], str | None]:
+ """
+ Fetch issues within a date range.
+
+ Args:
+ start_date: Start date in YYYY-MM-DD format
+ end_date: End date in YYYY-MM-DD format (inclusive)
+ include_comments: Whether to include comments in the response
+ project_key: Optional project key to filter issues
+
+ Returns:
+ Tuple containing (issues list, error message or None)
+ """
+ try:
+ # Build JQL query for date range
+ # Query issues that were either created OR updated within the date range
+ date_filter = (
+ f"(createdDate >= '{start_date}' AND createdDate <= '{end_date}')"
+ )
+ # TODO : This JQL needs some improvement to work as expected
+
+ _jql = f"{date_filter}"
+ if project_key:
+ _jql = (
+ f'project = "{project_key}" AND {date_filter} ORDER BY created DESC'
+ )
+
+ # Define fields to retrieve
+ fields = [
+ "summary",
+ "description",
+ "status",
+ "assignee",
+ "reporter",
+ "created",
+ "updated",
+ "priority",
+ "issuetype",
+ "project",
+ ]
+
+ if include_comments:
+ fields.append("comment")
+
+ params = {
+ # "jql": "", TODO : Add a JQL query to filter from a date range
+ "fields": ",".join(fields),
+ "maxResults": 100,
+ "startAt": 0,
+ }
+
+ all_issues = []
+ start_at = 0
+
+ while True:
+ params["startAt"] = start_at
+
+ result = self.make_api_request("search", params)
+
+ if not isinstance(result, dict) or "issues" not in result:
+ return [], "Invalid response from Jira API"
+
+ issues = result["issues"]
+ all_issues.extend(issues)
+
+ # Check if there are more issues to fetch
+ total = result.get("total", 0)
+ if start_at + len(issues) >= total:
+ break
+
+ start_at += len(issues)
+
+ if not all_issues:
+ return [], "No issues found in the specified date range."
+
+ return all_issues, None
+
+ except Exception as e:
+ return [], f"Error fetching issues: {e!s}"
+
+ def format_issue(self, issue: dict[str, Any]) -> dict[str, Any]:
+ """
+ Format an issue for easier consumption.
+
+ Args:
+ issue: The issue object from Jira API
+
+ Returns:
+ Formatted issue dictionary
+ """
+ fields = issue.get("fields", {})
+
+ # Extract basic issue details
+ formatted = {
+ "id": issue.get("id", ""),
+ "key": issue.get("key", ""),
+ "title": fields.get("summary", ""),
+ "description": fields.get("description", ""),
+ "status": (
+ fields.get("status", {}).get("name", "Unknown")
+ if fields.get("status")
+ else "Unknown"
+ ),
+ "status_category": (
+ fields.get("status", {})
+ .get("statusCategory", {})
+ .get("name", "Unknown")
+ if fields.get("status")
+ else "Unknown"
+ ),
+ "priority": (
+ fields.get("priority", {}).get("name", "Unknown")
+ if fields.get("priority")
+ else "Unknown"
+ ),
+ "issue_type": (
+ fields.get("issuetype", {}).get("name", "Unknown")
+ if fields.get("issuetype")
+ else "Unknown"
+ ),
+ "project": (
+ fields.get("project", {}).get("key", "Unknown")
+ if fields.get("project")
+ else "Unknown"
+ ),
+ "created_at": fields.get("created", ""),
+ "updated_at": fields.get("updated", ""),
+ "reporter": (
+ {
+ "account_id": (
+ fields.get("reporter", {}).get("accountId", "")
+ if fields.get("reporter")
+ else ""
+ ),
+ "display_name": (
+ fields.get("reporter", {}).get("displayName", "Unknown")
+ if fields.get("reporter")
+ else "Unknown"
+ ),
+ "email": (
+ fields.get("reporter", {}).get("emailAddress", "")
+ if fields.get("reporter")
+ else ""
+ ),
+ }
+ if fields.get("reporter")
+ else {"account_id": "", "display_name": "Unknown", "email": ""}
+ ),
+ "assignee": (
+ {
+ "account_id": fields.get("assignee", {}).get("accountId", ""),
+ "display_name": fields.get("assignee", {}).get(
+ "displayName", "Unknown"
+ ),
+ "email": fields.get("assignee", {}).get("emailAddress", ""),
+ }
+ if fields.get("assignee")
+ else None
+ ),
+ "comments": [],
+ }
+
+ # Extract comments if available
+ if "comment" in fields and "comments" in fields["comment"]:
+ for comment in fields["comment"]["comments"]:
+ formatted_comment = {
+ "id": comment.get("id", ""),
+ "body": comment.get("body", ""),
+ "created_at": comment.get("created", ""),
+ "updated_at": comment.get("updated", ""),
+ "author": (
+ {
+ "account_id": (
+ comment.get("author", {}).get("accountId", "")
+ if comment.get("author")
+ else ""
+ ),
+ "display_name": (
+ comment.get("author", {}).get("displayName", "Unknown")
+ if comment.get("author")
+ else "Unknown"
+ ),
+ "email": (
+ comment.get("author", {}).get("emailAddress", "")
+ if comment.get("author")
+ else ""
+ ),
+ }
+ if comment.get("author")
+ else {"account_id": "", "display_name": "Unknown", "email": ""}
+ ),
+ }
+ formatted["comments"].append(formatted_comment)
+
+ return formatted
+
+ def format_issue_to_markdown(self, issue: dict[str, Any]) -> str:
+ """
+ Convert an issue to markdown format.
+
+ Args:
+ issue: The issue object (either raw or formatted)
+
+ Returns:
+ Markdown string representation of the issue
+ """
+ # Format the issue if it's not already formatted
+ if "key" not in issue:
+ issue = self.format_issue(issue)
+
+ # Build the markdown content
+ markdown = (
+ f"# {issue.get('key', 'No Key')}: {issue.get('title', 'No Title')}\n\n"
+ )
+
+ if issue.get("status"):
+ markdown += f"**Status:** {issue['status']}\n"
+
+ if issue.get("priority"):
+ markdown += f"**Priority:** {issue['priority']}\n"
+
+ if issue.get("issue_type"):
+ markdown += f"**Type:** {issue['issue_type']}\n"
+
+ if issue.get("project"):
+ markdown += f"**Project:** {issue['project']}\n\n"
+
+ if issue.get("assignee") and issue["assignee"].get("display_name"):
+ markdown += f"**Assignee:** {issue['assignee']['display_name']}\n"
+
+ if issue.get("reporter") and issue["reporter"].get("display_name"):
+ markdown += f"**Reporter:** {issue['reporter']['display_name']}\n"
+
+ if issue.get("created_at"):
+ created_date = self.format_date(issue["created_at"])
+ markdown += f"**Created:** {created_date}\n"
+
+ if issue.get("updated_at"):
+ updated_date = self.format_date(issue["updated_at"])
+ markdown += f"**Updated:** {updated_date}\n\n"
+
+ if issue.get("description"):
+ markdown += f"## Description\n\n{issue['description']}\n\n"
+
+ if issue.get("comments"):
+ markdown += f"## Comments ({len(issue['comments'])})\n\n"
+
+ for comment in issue["comments"]:
+ author_name = "Unknown"
+ if comment.get("author") and comment["author"].get("display_name"):
+ author_name = comment["author"]["display_name"]
+
+ comment_date = "Unknown date"
+ if comment.get("created_at"):
+ comment_date = self.format_date(comment["created_at"])
+
+ markdown += f"### {author_name} ({comment_date})\n\n{comment.get('body', '')}\n\n---\n\n"
+
+ return markdown
+
+ @staticmethod
+ def format_date(iso_date: str) -> str:
+ """
+ Format an ISO date string to a more readable format.
+
+ Args:
+ iso_date: ISO format date string
+
+ Returns:
+ Formatted date string
+ """
+ if not iso_date or not isinstance(iso_date, str):
+ return "Unknown date"
+
+ try:
+ # Jira dates are typically in format: 2023-01-01T12:00:00.000+0000
+ dt = datetime.fromisoformat(iso_date.replace("Z", "+00:00"))
+ return dt.strftime("%Y-%m-%d %H:%M:%S")
+ except ValueError:
+ return iso_date
diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py
index 3d235d0..1a7aa57 100644
--- a/surfsense_backend/app/db.py
+++ b/surfsense_backend/app/db.py
@@ -3,6 +3,7 @@ from datetime import UTC, datetime
from enum import Enum
from fastapi import Depends
+from fastapi_users.db import SQLAlchemyBaseUserTableUUID, SQLAlchemyUserDatabase
from pgvector.sqlalchemy import Vector
from sqlalchemy import (
ARRAY,
@@ -26,13 +27,7 @@ from app.retriver.chunks_hybrid_search import ChucksHybridSearchRetriever
from app.retriver.documents_hybrid_search import DocumentHybridSearchRetriever
if config.AUTH_TYPE == "GOOGLE":
- from fastapi_users.db import (
- SQLAlchemyBaseOAuthAccountTableUUID,
- SQLAlchemyBaseUserTableUUID,
- SQLAlchemyUserDatabase,
- )
-else:
- from fastapi_users.db import SQLAlchemyBaseUserTableUUID, SQLAlchemyUserDatabase
+ from fastapi_users.db import SQLAlchemyBaseOAuthAccountTableUUID
DATABASE_URL = config.DATABASE_URL
@@ -47,6 +42,7 @@ class DocumentType(str, Enum):
GITHUB_CONNECTOR = "GITHUB_CONNECTOR"
LINEAR_CONNECTOR = "LINEAR_CONNECTOR"
DISCORD_CONNECTOR = "DISCORD_CONNECTOR"
+ JIRA_CONNECTOR = "JIRA_CONNECTOR"
class SearchSourceConnectorType(str, Enum):
@@ -58,6 +54,7 @@ class SearchSourceConnectorType(str, Enum):
GITHUB_CONNECTOR = "GITHUB_CONNECTOR"
LINEAR_CONNECTOR = "LINEAR_CONNECTOR"
DISCORD_CONNECTOR = "DISCORD_CONNECTOR"
+ JIRA_CONNECTOR = "JIRA_CONNECTOR"
class ChatType(str, Enum):
@@ -320,6 +317,7 @@ if config.AUTH_TYPE == "GOOGLE":
strategic_llm = relationship(
"LLMConfig", foreign_keys=[strategic_llm_id], post_update=True
)
+
else:
class User(SQLAlchemyBaseUserTableUUID, Base):
@@ -402,6 +400,7 @@ if config.AUTH_TYPE == "GOOGLE":
async def get_user_db(session: AsyncSession = Depends(get_async_session)):
yield SQLAlchemyUserDatabase(session, User, OAuthAccount)
+
else:
async def get_user_db(session: AsyncSession = Depends(get_async_session)):
diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py
index 47caa97..4c3d691 100644
--- a/surfsense_backend/app/routes/search_source_connectors_routes.py
+++ b/surfsense_backend/app/routes/search_source_connectors_routes.py
@@ -38,6 +38,7 @@ from app.schemas import (
from app.tasks.connectors_indexing_tasks import (
index_discord_messages,
index_github_repos,
+ index_jira_issues,
index_linear_issues,
index_notion_pages,
index_slack_messages,
@@ -336,6 +337,7 @@ async def index_connector_content(
- NOTION_CONNECTOR: Indexes pages from all accessible Notion pages
- GITHUB_CONNECTOR: Indexes code and documentation from GitHub repositories
- LINEAR_CONNECTOR: Indexes issues and comments from Linear
+ - JIRA_CONNECTOR: Indexes issues and comments from Jira
- DISCORD_CONNECTOR: Indexes messages from all accessible Discord channels
Args:
@@ -353,7 +355,9 @@ async def index_connector_content(
)
# Check if the search space belongs to the user
- await check_ownership(session, SearchSpace, search_space_id, user)
+ _search_space = await check_ownership(
+ session, SearchSpace, search_space_id, user
+ )
# Handle different connector types
response_message = ""
@@ -438,6 +442,21 @@ async def index_connector_content(
)
response_message = "Linear indexing started in the background."
+ elif connector.connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
+ # Run indexing in background
+ logger.info(
+ f"Triggering Jira indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}"
+ )
+ background_tasks.add_task(
+ run_jira_indexing_with_new_session,
+ connector_id,
+ search_space_id,
+ str(user.id),
+ indexing_from,
+ indexing_to,
+ )
+ response_message = "Jira indexing started in the background."
+
elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR:
# Run indexing in background
logger.info(
@@ -807,3 +826,61 @@ async def run_discord_indexing(
)
except Exception as e:
logger.error(f"Error in background Discord indexing task: {e!s}")
+
+
+# Add new helper functions for Jira indexing
+async def run_jira_indexing_with_new_session(
+ connector_id: int,
+ search_space_id: int,
+ user_id: str,
+ start_date: str,
+ end_date: str,
+):
+ """Wrapper to run Jira indexing with its own database session."""
+ logger.info(
+ f"Background task started: Indexing Jira connector {connector_id} into space {search_space_id} from {start_date} to {end_date}"
+ )
+ async with async_session_maker() as session:
+ await run_jira_indexing(
+ session, connector_id, search_space_id, user_id, start_date, end_date
+ )
+ logger.info(f"Background task finished: Indexing Jira connector {connector_id}")
+
+
+async def run_jira_indexing(
+ session: AsyncSession,
+ connector_id: int,
+ search_space_id: int,
+ user_id: str,
+ start_date: str,
+ end_date: str,
+):
+ """Runs the Jira indexing task and updates the timestamp."""
+ try:
+ indexed_count, error_message = await index_jira_issues(
+ session,
+ connector_id,
+ search_space_id,
+ user_id,
+ start_date,
+ end_date,
+ update_last_indexed=False,
+ )
+ if error_message:
+ logger.error(
+ f"Jira indexing failed for connector {connector_id}: {error_message}"
+ )
+ # Optionally update status in DB to indicate failure
+ else:
+ logger.info(
+ f"Jira indexing successful for connector {connector_id}. Indexed {indexed_count} documents."
+ )
+ # Update the last indexed timestamp only on success
+ await update_connector_last_indexed(session, connector_id)
+ await session.commit() # Commit timestamp update
+ except Exception as e:
+ logger.error(
+ f"Critical error in run_jira_indexing for connector {connector_id}: {e}",
+ exc_info=True,
+ )
+ # Optionally update status in DB to indicate failure
diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py
index 719a9f9..9c43d07 100644
--- a/surfsense_backend/app/schemas/search_source_connector.py
+++ b/surfsense_backend/app/schemas/search_source_connector.py
@@ -123,6 +123,25 @@ class SearchSourceConnectorBase(BaseModel):
# Ensure the bot token is not empty
if not config.get("DISCORD_BOT_TOKEN"):
raise ValueError("DISCORD_BOT_TOKEN cannot be empty")
+ elif connector_type == SearchSourceConnectorType.JIRA_CONNECTOR:
+ # For JIRA_CONNECTOR, require JIRA_EMAIL, JIRA_API_TOKEN and JIRA_BASE_URL
+ allowed_keys = ["JIRA_EMAIL", "JIRA_API_TOKEN", "JIRA_BASE_URL"]
+ if set(config.keys()) != set(allowed_keys):
+ raise ValueError(
+ f"For JIRA_CONNECTOR connector type, config must only contain these keys: {allowed_keys}"
+ )
+
+ # Ensure the email is not empty
+ if not config.get("JIRA_EMAIL"):
+ raise ValueError("JIRA_EMAIL cannot be empty")
+
+ # Ensure the API token is not empty
+ if not config.get("JIRA_API_TOKEN"):
+ raise ValueError("JIRA_API_TOKEN cannot be empty")
+
+ # Ensure the base URL is not empty
+ if not config.get("JIRA_BASE_URL"):
+ raise ValueError("JIRA_BASE_URL cannot be empty")
return config
diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py
index 33001e2..1c6d612 100644
--- a/surfsense_backend/app/services/connector_service.py
+++ b/surfsense_backend/app/services/connector_service.py
@@ -1,4 +1,5 @@
import asyncio
+from typing import Any
from linkup import LinkupClient
from sqlalchemy import func
@@ -204,7 +205,9 @@ class ConnectorService:
return result_object, files_chunks
- def _transform_document_results(self, document_results: list[dict]) -> list[dict]:
+ def _transform_document_results(
+ self, document_results: list[dict[str, Any]]
+ ) -> list[dict[str, Any]]:
"""
Transform results from document_retriever.hybrid_search() to match the format
expected by the processing code.
@@ -608,6 +611,7 @@ class ConnectorService:
visit_duration = metadata.get(
"VisitedWebPageVisitDurationInMilliseconds", ""
)
+ _browsing_session_id = metadata.get("BrowsingSessionId", "")
# Create a more descriptive title for extension data
title = webpage_title
@@ -948,6 +952,127 @@ class ConnectorService:
return result_object, linear_chunks
+ async def search_jira(
+ self,
+ user_query: str,
+ user_id: str,
+ search_space_id: int,
+ top_k: int = 20,
+ search_mode: SearchMode = SearchMode.CHUNKS,
+ ) -> tuple:
+ """
+ Search for Jira issues and comments and return both the source information and langchain documents
+
+ Args:
+ user_query: The user's query
+ user_id: The user's ID
+ search_space_id: The search space ID to search in
+ top_k: Maximum number of results to return
+ search_mode: Search mode (CHUNKS or DOCUMENTS)
+
+ Returns:
+ tuple: (sources_info, langchain_documents)
+ """
+ if search_mode == SearchMode.CHUNKS:
+ jira_chunks = await self.chunk_retriever.hybrid_search(
+ query_text=user_query,
+ top_k=top_k,
+ user_id=user_id,
+ search_space_id=search_space_id,
+ document_type="JIRA_CONNECTOR",
+ )
+ elif search_mode == SearchMode.DOCUMENTS:
+ jira_chunks = await self.document_retriever.hybrid_search(
+ query_text=user_query,
+ top_k=top_k,
+ user_id=user_id,
+ search_space_id=search_space_id,
+ document_type="JIRA_CONNECTOR",
+ )
+ # Transform document retriever results to match expected format
+ jira_chunks = self._transform_document_results(jira_chunks)
+
+ # Early return if no results
+ if not jira_chunks:
+ return {
+ "id": 30,
+ "name": "Jira Issues",
+ "type": "JIRA_CONNECTOR",
+ "sources": [],
+ }, []
+
+ # Process each chunk and create sources directly without deduplication
+ sources_list = []
+ async with self.counter_lock:
+ for _i, chunk in enumerate(jira_chunks):
+ # Extract document metadata
+ document = chunk.get("document", {})
+ metadata = document.get("metadata", {})
+
+ # Extract Jira-specific metadata
+ issue_key = metadata.get("issue_key", "")
+ issue_title = metadata.get("issue_title", "Untitled Issue")
+ status = metadata.get("status", "")
+ priority = metadata.get("priority", "")
+ issue_type = metadata.get("issue_type", "")
+ comment_count = metadata.get("comment_count", 0)
+
+ # Create a more descriptive title for Jira issues
+ title = f"Jira: {issue_key} - {issue_title}"
+ if status:
+ title += f" ({status})"
+
+ # Create a more descriptive description for Jira issues
+ description = chunk.get("content", "")[:100]
+ if len(description) == 100:
+ description += "..."
+
+ # Add priority and type info to description
+ info_parts = []
+ if priority:
+ info_parts.append(f"Priority: {priority}")
+ if issue_type:
+ info_parts.append(f"Type: {issue_type}")
+ if comment_count:
+ info_parts.append(f"Comments: {comment_count}")
+
+ if info_parts:
+ if description:
+ description += f" | {' | '.join(info_parts)}"
+ else:
+ description = " | ".join(info_parts)
+
+ # For URL, we could construct a URL to the Jira issue if we have the base URL
+ # For now, use a generic placeholder
+ url = ""
+ if issue_key and metadata.get("base_url"):
+ url = f"{metadata.get('base_url')}/browse/{issue_key}"
+
+ source = {
+ "id": document.get("id", self.source_id_counter),
+ "title": title,
+ "description": description,
+ "url": url,
+ "issue_key": issue_key,
+ "status": status,
+ "priority": priority,
+ "issue_type": issue_type,
+ "comment_count": comment_count,
+ }
+
+ self.source_id_counter += 1
+ sources_list.append(source)
+
+ # Create result object
+ result_object = {
+ "id": 10, # Assign a unique ID for the Jira connector
+ "name": "Jira Issues",
+ "type": "JIRA_CONNECTOR",
+ "sources": sources_list,
+ }
+
+ return result_object, jira_chunks
+
async def search_linkup(
self, user_query: str, user_id: str, mode: str = "standard"
) -> tuple:
@@ -1013,12 +1138,12 @@ class ConnectorService:
# Create a source entry
source = {
"id": self.source_id_counter,
- "title": result.name
- if hasattr(result, "name")
- else "Linkup Result",
- "description": result.content[:100]
- if hasattr(result, "content")
- else "",
+ "title": (
+ result.name if hasattr(result, "name") else "Linkup Result"
+ ),
+ "description": (
+ result.content[:100] if hasattr(result, "content") else ""
+ ),
"url": result.url if hasattr(result, "url") else "",
}
sources_list.append(source)
@@ -1030,9 +1155,11 @@ class ConnectorService:
"score": 1.0, # Default score since not provided by Linkup
"document": {
"id": self.source_id_counter,
- "title": result.name
- if hasattr(result, "name")
- else "Linkup Result",
+ "title": (
+ result.name
+ if hasattr(result, "name")
+ else "Linkup Result"
+ ),
"document_type": "LINKUP_API",
"metadata": {
"url": result.url if hasattr(result, "url") else "",
diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py
index 053b8ba..e028a47 100644
--- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py
+++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py
@@ -10,6 +10,7 @@ from sqlalchemy.future import select
from app.config import config
from app.connectors.discord_connector import DiscordConnector
from app.connectors.github_connector import GitHubConnector
+from app.connectors.jira_connector import JiraConnector
from app.connectors.linear_connector import LinearConnector
from app.connectors.notion_history import NotionHistoryConnector
from app.connectors.slack_history import SlackHistory
@@ -1374,9 +1375,9 @@ async def index_linear_issues(
# Process each issue
for issue in issues:
try:
- issue_id = issue.get("id")
- issue_identifier = issue.get("identifier", "")
- issue_title = issue.get("title", "")
+ issue_id = issue.get("key")
+ issue_identifier = issue.get("id", "")
+ issue_title = issue.get("key", "")
if not issue_id or not issue_title:
logger.warning(
@@ -1978,3 +1979,353 @@ async def index_discord_messages(
)
logger.error(f"Failed to index Discord messages: {e!s}", exc_info=True)
return 0, f"Failed to index Discord messages: {e!s}"
+
+
+async def index_jira_issues(
+ session: AsyncSession,
+ connector_id: int,
+ search_space_id: int,
+ user_id: str,
+ start_date: str | None = None,
+ end_date: str | None = None,
+ update_last_indexed: bool = True,
+) -> tuple[int, str | None]:
+ """
+ Index Jira issues and comments.
+
+ Args:
+ session: Database session
+ connector_id: ID of the Jira connector
+ search_space_id: ID of the search space to store documents in
+ user_id: User ID
+ start_date: Start date for indexing (YYYY-MM-DD format)
+ end_date: End date for indexing (YYYY-MM-DD format)
+ update_last_indexed: Whether to update the last_indexed_at timestamp (default: True)
+
+ Returns:
+ Tuple containing (number of documents indexed, error message or None)
+ """
+ task_logger = TaskLoggingService(session, search_space_id)
+
+ # Log task start
+ log_entry = await task_logger.log_task_start(
+ task_name="jira_issues_indexing",
+ source="connector_indexing_task",
+ message=f"Starting Jira issues indexing for connector {connector_id}",
+ metadata={
+ "connector_id": connector_id,
+ "user_id": str(user_id),
+ "start_date": start_date,
+ "end_date": end_date,
+ },
+ )
+
+ try:
+ # Get the connector from the database
+ result = await session.execute(
+ select(SearchSourceConnector).filter(
+ SearchSourceConnector.id == connector_id,
+ SearchSourceConnector.connector_type
+ == SearchSourceConnectorType.JIRA_CONNECTOR,
+ )
+ )
+ connector = result.scalars().first()
+
+ if not connector:
+ await task_logger.log_task_failure(
+ log_entry,
+ f"Connector with ID {connector_id} not found",
+ "Connector not found",
+ {"error_type": "ConnectorNotFound"},
+ )
+ return 0, f"Connector with ID {connector_id} not found"
+
+ # Get the Jira credentials from the connector config
+ jira_email = connector.config.get("JIRA_EMAIL")
+ jira_api_token = connector.config.get("JIRA_API_TOKEN")
+ jira_base_url = connector.config.get("JIRA_BASE_URL")
+
+ if not jira_email or not jira_api_token or not jira_base_url:
+ await task_logger.log_task_failure(
+ log_entry,
+ f"Jira credentials not found in connector config for connector {connector_id}",
+ "Missing Jira credentials",
+ {"error_type": "MissingCredentials"},
+ )
+ return 0, "Jira credentials not found in connector config"
+
+ # Initialize Jira client
+ await task_logger.log_task_progress(
+ log_entry,
+ f"Initializing Jira client for connector {connector_id}",
+ {"stage": "client_initialization"},
+ )
+
+ jira_client = JiraConnector(
+ base_url=jira_base_url, email=jira_email, api_token=jira_api_token
+ )
+
+ # Calculate date range
+ if start_date is None or end_date is None:
+ # Fall back to calculating dates based on last_indexed_at
+ calculated_end_date = datetime.now()
+
+ # Use last_indexed_at as start date if available, otherwise use 365 days ago
+ if connector.last_indexed_at:
+ # Convert dates to be comparable (both timezone-naive)
+ last_indexed_naive = (
+ connector.last_indexed_at.replace(tzinfo=None)
+ if connector.last_indexed_at.tzinfo
+ else connector.last_indexed_at
+ )
+
+ # Check if last_indexed_at is in the future or after end_date
+ if last_indexed_naive > calculated_end_date:
+ logger.warning(
+ f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 365 days ago instead."
+ )
+ calculated_start_date = calculated_end_date - timedelta(days=365)
+ else:
+ calculated_start_date = last_indexed_naive
+ logger.info(
+ f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date"
+ )
+ else:
+ calculated_start_date = calculated_end_date - timedelta(
+ days=365
+ ) # Use 365 days as default
+ logger.info(
+ f"No last_indexed_at found, using {calculated_start_date.strftime('%Y-%m-%d')} (365 days ago) as start date"
+ )
+
+ # Use calculated dates if not provided
+ start_date_str = (
+ start_date if start_date else calculated_start_date.strftime("%Y-%m-%d")
+ )
+ end_date_str = (
+ end_date if end_date else calculated_end_date.strftime("%Y-%m-%d")
+ )
+ else:
+ # Use provided dates
+ start_date_str = start_date
+ end_date_str = end_date
+
+ await task_logger.log_task_progress(
+ log_entry,
+ f"Fetching Jira issues from {start_date_str} to {end_date_str}",
+ {
+ "stage": "fetching_issues",
+ "start_date": start_date_str,
+ "end_date": end_date_str,
+ },
+ )
+
+ # Get issues within date range
+ try:
+ issues, error = jira_client.get_issues_by_date_range(
+ start_date=start_date_str, end_date=end_date_str, include_comments=True
+ )
+
+ if error:
+ logger.error(f"Failed to get Jira issues: {error}")
+
+ # Don't treat "No issues found" as an error that should stop indexing
+ if "No issues found" in error:
+ logger.info(
+ "No issues found is not a critical error, continuing with update"
+ )
+ if update_last_indexed:
+ connector.last_indexed_at = datetime.now()
+ await session.commit()
+ logger.info(
+ f"Updated last_indexed_at to {connector.last_indexed_at} despite no issues found"
+ )
+
+ await task_logger.log_task_success(
+ log_entry,
+ f"No Jira issues found in date range {start_date_str} to {end_date_str}",
+ {"issues_found": 0},
+ )
+ return 0, None
+ else:
+ await task_logger.log_task_failure(
+ log_entry,
+ f"Failed to get Jira issues: {error}",
+ "API Error",
+ {"error_type": "APIError"},
+ )
+ return 0, f"Failed to get Jira issues: {error}"
+
+ logger.info(f"Retrieved {len(issues)} issues from Jira API")
+
+ except Exception as e:
+ logger.error(f"Error fetching Jira issues: {e!s}", exc_info=True)
+ return 0, f"Error fetching Jira issues: {e!s}"
+
+ # Process and index each issue
+ documents_indexed = 0
+ skipped_issues = []
+ documents_skipped = 0
+
+ for issue in issues:
+ try:
+ issue_id = issue.get("key")
+ issue_identifier = issue.get("key", "")
+ issue_title = issue.get("id", "")
+
+ if not issue_id or not issue_title:
+ logger.warning(
+ f"Skipping issue with missing ID or title: {issue_id or 'Unknown'}"
+ )
+ skipped_issues.append(
+ f"{issue_identifier or 'Unknown'} (missing data)"
+ )
+ documents_skipped += 1
+ continue
+
+ # Format the issue for better readability
+ formatted_issue = jira_client.format_issue(issue)
+
+ # Convert to markdown
+ issue_content = jira_client.format_issue_to_markdown(formatted_issue)
+
+ if not issue_content:
+ logger.warning(
+ f"Skipping issue with no content: {issue_identifier} - {issue_title}"
+ )
+ skipped_issues.append(f"{issue_identifier} (no content)")
+ documents_skipped += 1
+ continue
+
+ # Create a simple summary
+ summary_content = f"Jira Issue {issue_identifier}: {issue_title}\n\nStatus: {formatted_issue.get('status', 'Unknown')}\n\n"
+ if formatted_issue.get("description"):
+ summary_content += (
+ f"Description: {formatted_issue.get('description')}\n\n"
+ )
+
+ # Add comment count
+ comment_count = len(formatted_issue.get("comments", []))
+ summary_content += f"Comments: {comment_count}"
+
+ # Generate content hash
+ content_hash = generate_content_hash(issue_content, search_space_id)
+
+ # Check if document already exists
+ existing_doc_by_hash_result = await session.execute(
+ select(Document).where(Document.content_hash == content_hash)
+ )
+ existing_document_by_hash = (
+ existing_doc_by_hash_result.scalars().first()
+ )
+
+ if existing_document_by_hash:
+ logger.info(
+ f"Document with content hash {content_hash} already exists for issue {issue_identifier}. Skipping processing."
+ )
+ documents_skipped += 1
+ continue
+
+ # Generate embedding for the summary
+ summary_embedding = config.embedding_model_instance.embed(
+ summary_content
+ )
+
+ # Process chunks - using the full issue content with comments
+ chunks = [
+ Chunk(
+ content=chunk.text,
+ embedding=config.embedding_model_instance.embed(chunk.text),
+ )
+ for chunk in config.chunker_instance.chunk(issue_content)
+ ]
+
+ # Create and store new document
+ logger.info(
+ f"Creating new document for issue {issue_identifier} - {issue_title}"
+ )
+ document = Document(
+ search_space_id=search_space_id,
+ title=f"Jira - {issue_identifier}: {issue_title}",
+ document_type=DocumentType.JIRA_CONNECTOR,
+ document_metadata={
+ "issue_id": issue_id,
+ "issue_identifier": issue_identifier,
+ "issue_title": issue_title,
+ "state": formatted_issue.get("status", "Unknown"),
+ "comment_count": comment_count,
+ "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+ },
+ content=summary_content,
+ content_hash=content_hash,
+ embedding=summary_embedding,
+ chunks=chunks,
+ )
+
+ session.add(document)
+ documents_indexed += 1
+ logger.info(
+ f"Successfully indexed new issue {issue_identifier} - {issue_title}"
+ )
+
+ except Exception as e:
+ logger.error(
+ f"Error processing issue {issue.get('identifier', 'Unknown')}: {e!s}",
+ exc_info=True,
+ )
+ skipped_issues.append(
+ f"{issue.get('identifier', 'Unknown')} (processing error)"
+ )
+ documents_skipped += 1
+ continue # Skip this issue and continue with others
+
+ # Update the last_indexed_at timestamp for the connector only if requested
+ total_processed = documents_indexed
+ if update_last_indexed:
+ connector.last_indexed_at = datetime.now()
+ logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}")
+
+ # Commit all changes
+ await session.commit()
+ logger.info("Successfully committed all JIRA document changes to database")
+
+ # Log success
+ await task_logger.log_task_success(
+ log_entry,
+ f"Successfully completed JIRA indexing for connector {connector_id}",
+ {
+ "issues_processed": total_processed,
+ "documents_indexed": documents_indexed,
+ "documents_skipped": documents_skipped,
+ "skipped_issues_count": len(skipped_issues),
+ },
+ )
+
+ logger.info(
+ f"JIRA indexing completed: {documents_indexed} new issues, {documents_skipped} skipped"
+ )
+ return (
+ total_processed,
+ None,
+ ) # Return None as the error message to indicate success
+
+ except SQLAlchemyError as db_error:
+ await session.rollback()
+ await task_logger.log_task_failure(
+ log_entry,
+ f"Database error during JIRA indexing for connector {connector_id}",
+ str(db_error),
+ {"error_type": "SQLAlchemyError"},
+ )
+ logger.error(f"Database error: {db_error!s}", exc_info=True)
+ return 0, f"Database error: {db_error!s}"
+ except Exception as e:
+ await session.rollback()
+ await task_logger.log_task_failure(
+ log_entry,
+ f"Failed to index JIRA issues for connector {connector_id}",
+ str(e),
+ {"error_type": type(e).__name__},
+ )
+ logger.error(f"Failed to index JIRA issues: {e!s}", exc_info=True)
+ return 0, f"Failed to index JIRA issues: {e!s}"
diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx
index 34db58f..918a625 100644
--- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx
+++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx
@@ -9,12 +9,12 @@ import { ArrowLeft, Check, Loader2, Github } from "lucide-react";
import { Form } from "@/components/ui/form";
import { Button } from "@/components/ui/button";
import {
- Card,
- CardContent,
- CardDescription,
- CardFooter,
- CardHeader,
- CardTitle,
+ Card,
+ CardContent,
+ CardDescription,
+ CardFooter,
+ CardHeader,
+ CardTitle,
} from "@/components/ui/card";
// Import Utils, Types, Hook, and Components
@@ -27,201 +27,227 @@ import { EditSimpleTokenForm } from "@/components/editConnector/EditSimpleTokenF
import { getConnectorIcon } from "@/components/chat";
export default function EditConnectorPage() {
- const router = useRouter();
- const params = useParams();
- const searchSpaceId = params.search_space_id as string;
- // Ensure connectorId is parsed safely
- const connectorIdParam = params.connector_id as string;
- const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN;
+ const router = useRouter();
+ const params = useParams();
+ const searchSpaceId = params.search_space_id as string;
+ // Ensure connectorId is parsed safely
+ const connectorIdParam = params.connector_id as string;
+ const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN;
- // Use the custom hook to manage state and logic
- const {
- connectorsLoading,
- connector,
- isSaving,
- editForm,
- patForm, // Needed for GitHub child component
- handleSaveChanges,
- // GitHub specific props for the child component
- editMode,
- setEditMode, // Pass down if needed by GitHub component
- originalPat,
- currentSelectedRepos,
- fetchedRepos,
- setFetchedRepos,
- newSelectedRepos,
- setNewSelectedRepos,
- isFetchingRepos,
- handleFetchRepositories,
- handleRepoSelectionChange,
- } = useConnectorEditPage(connectorId, searchSpaceId);
+ // Use the custom hook to manage state and logic
+ const {
+ connectorsLoading,
+ connector,
+ isSaving,
+ editForm,
+ patForm, // Needed for GitHub child component
+ handleSaveChanges,
+ // GitHub specific props for the child component
+ editMode,
+ setEditMode, // Pass down if needed by GitHub component
+ originalPat,
+ currentSelectedRepos,
+ fetchedRepos,
+ setFetchedRepos,
+ newSelectedRepos,
+ setNewSelectedRepos,
+ isFetchingRepos,
+ handleFetchRepositories,
+ handleRepoSelectionChange,
+ } = useConnectorEditPage(connectorId, searchSpaceId);
- // Redirect if connectorId is not a valid number after parsing
- useEffect(() => {
- if (isNaN(connectorId)) {
- toast.error("Invalid Connector ID.");
- router.push(`/dashboard/${searchSpaceId}/connectors`);
- }
- }, [connectorId, router, searchSpaceId]);
+ // Redirect if connectorId is not a valid number after parsing
+ useEffect(() => {
+ if (isNaN(connectorId)) {
+ toast.error("Invalid Connector ID.");
+ router.push(`/dashboard/${searchSpaceId}/connectors`);
+ }
+ }, [connectorId, router, searchSpaceId]);
- // Loading State
- if (connectorsLoading || !connector) {
- // Handle NaN case before showing skeleton
- if (isNaN(connectorId)) return null;
- return ;
- }
+ // Loading State
+ if (connectorsLoading || !connector) {
+ // Handle NaN case before showing skeleton
+ if (isNaN(connectorId)) return null;
+ return ;
+ }
- // Main Render using data/handlers from the hook
- return (
-
-
+ // Main Render using data/handlers from the hook
+ return (
+