SurfSense/surfsense_backend/app/tasks/connector_indexers/clickup_indexer.py

"""
ClickUp connector indexer.
"""

from datetime import datetime

from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import config
from app.connectors.clickup_connector import ClickUpConnector
from app.db import Document, DocumentType, SearchSourceConnectorType
from app.services.task_logging_service import TaskLoggingService
from app.utils.document_converters import generate_content_hash

from .base import (
    check_duplicate_document_by_hash,
    create_document_chunks,
    get_connector_by_id,
    logger,
    update_connector_last_indexed,
)


async def index_clickup_tasks(
    session: AsyncSession,
    connector_id: int,
    search_space_id: int,
    user_id: str,
    start_date: str | None = None,
    end_date: str | None = None,
    update_last_indexed: bool = True,
) -> tuple[int, str | None]:
    """
    Index tasks from ClickUp workspace.

    Args:
        session: Database session
        connector_id: ID of the ClickUp connector
        search_space_id: ID of the search space
        user_id: ID of the user
        start_date: Start date for filtering tasks (YYYY-MM-DD format)
        end_date: End date for filtering tasks (YYYY-MM-DD format)
        update_last_indexed: Whether to update the last_indexed_at timestamp

    Returns:
        Tuple of (number of indexed tasks, error message if any)
    """
    task_logger = TaskLoggingService(session, search_space_id)

    # Log task start
    log_entry = await task_logger.log_task_start(
        task_name="clickup_tasks_indexing",
        source="connector_indexing_task",
        message=f"Starting ClickUp tasks indexing for connector {connector_id}",
        metadata={
            "connector_id": connector_id,
            "start_date": start_date,
            "end_date": end_date,
        },
    )

    try:
        # Get connector configuration
        connector = await get_connector_by_id(
            session, connector_id, SearchSourceConnectorType.CLICKUP_CONNECTOR
        )

        if not connector:
            error_msg = f"ClickUp connector with ID {connector_id} not found"
            await task_logger.log_task_failure(
                log_entry,
                f"Connector with ID {connector_id} not found or is not a ClickUp connector",
                "Connector not found",
                {"error_type": "ConnectorNotFound"},
            )
            return 0, error_msg

        # Extract ClickUp configuration
        clickup_api_token = connector.config.get("CLICKUP_API_TOKEN")

        if not clickup_api_token:
            error_msg = "ClickUp API token not found in connector configuration"
            await task_logger.log_task_failure(
                log_entry,
                f"ClickUp API token not found in connector config for connector {connector_id}",
                "Missing ClickUp token",
                {"error_type": "MissingToken"},
            )
            return 0, error_msg

        await task_logger.log_task_progress(
            log_entry,
            f"Initializing ClickUp client for connector {connector_id}",
            {"stage": "client_initialization"},
        )

        clickup_client = ClickUpConnector(api_token=clickup_api_token)

        # Get authorized workspaces
        await task_logger.log_task_progress(
            log_entry,
            "Fetching authorized ClickUp workspaces",
            {"stage": "workspace_fetching"},
        )

        workspaces_response = clickup_client.get_authorized_workspaces()
        workspaces = workspaces_response.get("teams", [])

        if not workspaces:
            error_msg = "No authorized ClickUp workspaces found"
            await task_logger.log_task_failure(
                log_entry,
                f"No authorized ClickUp workspaces found for connector {connector_id}",
                "No workspaces found",
                {"error_type": "NoWorkspacesFound"},
            )
            return 0, error_msg

        documents_indexed = 0
        documents_skipped = 0

        # Iterate workspaces and fetch tasks
        for workspace in workspaces:
            workspace_id = workspace.get("id")
            workspace_name = workspace.get("name", "Unknown Workspace")
            if not workspace_id:
                continue

            await task_logger.log_task_progress(
                log_entry,
                f"Processing workspace: {workspace_name}",
                {"stage": "workspace_processing", "workspace_id": workspace_id},
            )

            # Fetch tasks for date range if provided
            if start_date and end_date:
                tasks, error = clickup_client.get_tasks_in_date_range(
                    workspace_id=workspace_id,
                    start_date=start_date,
                    end_date=end_date,
                    include_closed=True,
                )
                if error:
                    logger.warning(
                        f"Error fetching tasks from workspace {workspace_name}: {error}"
                    )
                    continue
            else:
                tasks = clickup_client.get_workspace_tasks(
                    workspace_id=workspace_id, include_closed=True
                )

            await task_logger.log_task_progress(
                log_entry,
                f"Found {len(tasks)} tasks in workspace {workspace_name}",
                {"stage": "tasks_found", "task_count": len(tasks)},
            )

            for task in tasks:
                try:
                    task_id = task.get("id")
                    task_name = task.get("name", "Untitled Task")
                    task_description = task.get("description", "")
                    task_status = task.get("status", {}).get("status", "Unknown")
                    task_priority = (
                        task.get("priority", {}).get("priority", "Unknown")
                        if task.get("priority")
                        else "None"
                    )
                    task_assignees = task.get("assignees", [])
                    task_due_date = task.get("due_date")
                    task_created = task.get("date_created")
                    task_updated = task.get("date_updated")

                    task_list = task.get("list", {})
                    task_list_name = task_list.get("name", "Unknown List")
                    task_space = task.get("space", {})
                    task_space_name = task_space.get("name", "Unknown Space")

                    # Build task content string
                    content_parts: list[str] = [f"Task: {task_name}"]
                    if task_description:
                        content_parts.append(f"Description: {task_description}")
                    content_parts.extend(
                        [
                            f"Status: {task_status}",
                            f"Priority: {task_priority}",
                            f"List: {task_list_name}",
                            f"Space: {task_space_name}",
                        ]
                    )
                    if task_assignees:
                        assignee_names = [
                            assignee.get("username", "Unknown")
                            for assignee in task_assignees
                        ]
                        content_parts.append(
                            f"Assignees: {', '.join(assignee_names)}"
                        )
                    if task_due_date:
                        content_parts.append(f"Due Date: {task_due_date}")

                    task_content = "\n".join(content_parts)
                    if not task_content.strip():
                        logger.warning(f"Skipping task with no content: {task_name}")
                        documents_skipped += 1
                        continue

                    # Hash for duplicates
                    content_hash = generate_content_hash(task_content, search_space_id)
                    existing_document_by_hash = await check_duplicate_document_by_hash(
                        session, content_hash
                    )
                    if existing_document_by_hash:
                        logger.info(
                            f"Document with content hash {content_hash} already exists for task {task_name}. Skipping processing."
                        )
                        documents_skipped += 1
                        continue

                    # Embedding and chunks
                    summary_embedding = config.embedding_model_instance.embed(
                        task_content
                    )
                    chunks = await create_document_chunks(task_content)

                    document = Document(
                        search_space_id=search_space_id,
                        title=f"Task - {task_name}",
                        document_type=DocumentType.CLICKUP_CONNECTOR,
                        document_metadata={
                            "task_id": task_id,
                            "task_name": task_name,
                            "task_status": task_status,
                            "task_priority": task_priority,
                            "task_assignees": task_assignees,
                            "task_due_date": task_due_date,
                            "task_created": task_created,
                            "task_updated": task_updated,
                            "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
                        },
                        content=task_content,
                        content_hash=content_hash,
                        embedding=summary_embedding,
                        chunks=chunks,
                    )

                    session.add(document)
                    documents_indexed += 1
                    logger.info(f"Successfully indexed new task {task_name}")

                except Exception as e:
                    logger.error(
                        f"Error processing task {task.get('name', 'Unknown')}: {e!s}",
                        exc_info=True,
                    )
                    documents_skipped += 1

        total_processed = documents_indexed

        if total_processed > 0:
            await update_connector_last_indexed(session, connector, update_last_indexed)

        await session.commit()

        await task_logger.log_task_success(
            log_entry,
            f"Successfully completed clickup indexing for connector {connector_id}",
            {
                "pages_processed": total_processed,
                "documents_indexed": documents_indexed,
                "documents_skipped": documents_skipped,
            },
        )

        logger.info(
            f"clickup indexing completed: {documents_indexed} new tasks, {documents_skipped} skipped"
        )
        return total_processed, None

    except SQLAlchemyError as db_error:
        await session.rollback()
        await task_logger.log_task_failure(
            log_entry,
            f"Database error during ClickUp indexing for connector {connector_id}",
            str(db_error),
            {"error_type": "SQLAlchemyError"},
        )
        logger.error(f"Database error: {db_error!s}", exc_info=True)
        return 0, f"Database error: {db_error!s}"
    except Exception as e:
        await session.rollback()
        await task_logger.log_task_failure(
            log_entry,
            f"Failed to index ClickUp tasks for connector {connector_id}",
            str(e),
            {"error_type": type(e).__name__},
        )
        logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True)
        return 0, f"Failed to index ClickUp tasks: {e!s}"