vrr/SurfSense
2
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2025-09-01 10:09:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add coderabbit suggestions

Browse source
This commit is contained in:
CREDO23 2025-07-30 22:25:47 +02:00
parent 9a98742f81
commit 4cb00735ac
4 changed files with 76 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
surfsense_backend/app/connectors/clickup_connector.py
View file

@ -5,7 +5,6 @@ A module for retrieving data from ClickUp.
Allows fetching tasks from workspaces and lists.
"""
from datetime import datetime
from typing import Any
import requests
@ -169,14 +168,6 @@ class ClickUpConnector:
Tuple containing (tasks list, error message or None)
"""
try:
# Convert dates to Unix timestamps (milliseconds)
start_timestamp = int(
datetime.strptime(start_date, "%Y-%m-%d").timestamp() * 1000
)
end_timestamp = int(
datetime.strptime(end_date, "%Y-%m-%d").timestamp() * 1000
)
# TODO : Include date range in api request
params = {

141
surfsense_backend/app/tasks/connectors_indexing_tasks.py
View file

@ -1,10 +1,8 @@
import asyncio
import hashlib
import logging
from datetime import UTC, datetime, timedelta
from slack_sdk.errors import SlackApiError
from sqlalchemy import func
from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select
@ -2742,7 +2740,6 @@ async def index_clickup_tasks(
)
try:
# Get connector configuration
result = await session.execute(
select(SearchSourceConnector).filter(
@ -2828,7 +2825,9 @@ async def index_clickup_tasks(
include_closed=True,
)
if error:
logger.warning(f"Error fetching tasks from workspace {workspace_name}: {error}")
logger.warning(
f"Error fetching tasks from workspace {workspace_name}: {error}"
)
continue
else:
tasks = clickup_client.get_workspace_tasks(
@ -2848,12 +2847,15 @@ async def index_clickup_tasks(
task_name = task.get("name", "Untitled Task")
task_description = task.get("description", "")
task_status = task.get("status", {}).get("status", "Unknown")
task_priority = task.get("priority", {}).get("priority", "Unknown") if task.get("priority") else "None"
task_priority = (
task.get("priority", {}).get("priority", "Unknown")
if task.get("priority")
else "None"
)
task_assignees = task.get("assignees", [])
task_due_date = task.get("due_date")
task_created = task.get("date_created")
task_updated = task.get("date_updated")
task_url = task.get("url", "")
# Get list and space information
task_list = task.get("list", {})
@ -2867,15 +2869,20 @@ async def index_clickup_tasks(
if task_description:
content_parts.append(f"Description: {task_description}")
content_parts.extend([
f"Status: {task_status}",
f"Priority: {task_priority}",
f"List: {task_list_name}",
f"Space: {task_space_name}",
])
content_parts.extend(
[
f"Status: {task_status}",
f"Priority: {task_priority}",
f"List: {task_list_name}",
f"Space: {task_space_name}",
]
)
if task_assignees:
assignee_names = [assignee.get("username", "Unknown") for assignee in task_assignees]
assignee_names = [
assignee.get("username", "Unknown")
for assignee in task_assignees
]
content_parts.append(f"Assignees: {', '.join(assignee_names)}")
if task_due_date:
@ -2887,53 +2894,34 @@ async def index_clickup_tasks(
logger.warning(f"Skipping task with no content: {task_name}")
continue
# Create document metadata
document_metadata = {
"task_id": task_id,
"task_name": task_name,
"task_url": task_url,
"task_status": task_status,
"task_priority": task_priority,
"task_assignees": task_assignees,
"task_due_date": task_due_date,
"task_created": task_created,
"task_updated": task_updated,
"task_list_name": task_list_name,
"task_space_name": task_space_name,
"workspace_id": workspace_id,
"workspace_name": workspace_name,
"connector_id": connector_id,
"source": "CLICKUP_CONNECTOR",
}
# Generate content hash
content_hash = generate_content_hash(task_content, search_space_id)
# Check if document already exists
existing_doc_by_hash_result = await session.execute(
select(Document).where(Document.content_hash == content_hash)
select(Document).where(Document.content_hash == content_hash)
)
existing_document_by_hash = (
existing_doc_by_hash_result.scalars().first()
existing_doc_by_hash_result.scalars().first()
)
if existing_document_by_hash:
logger.info(
f"Document with content hash {content_hash} already exists for task {task_name}. Skipping processing."
f"Document with content hash {content_hash} already exists for task {task_name}. Skipping processing."
)
documents_skipped += 1
continue
# Generate embedding for the summary
summary_embedding = config.embedding_model_instance.embed(
task_content
task_content
)
# Process chunks - using the full page content with comments
chunks = [
Chunk(
content=chunk.text,
embedding=config.embedding_model_instance.embed(chunk.text),
content=chunk.text,
embedding=config.embedding_model_instance.embed(chunk.text),
)
for chunk in config.chunker_instance.chunk(task_content)
]
@ -2942,24 +2930,24 @@ async def index_clickup_tasks(
logger.info(f"Creating new document for task {task_name}")
document = Document(
search_space_id=search_space_id,
title=f"Task - {task_name}",
document_type=DocumentType.CLICKUP_CONNECTOR,
document_metadata={
"task_id": task_id,
"task_name": task_name,
"task_status": task_status,
"task_priority": task_priority,
"task_assignees": task_assignees,
"task_due_date": task_due_date,
"task_created": task_created,
"task_updated": task_updated,
"indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
},
content=task_content,
content_hash=content_hash,
embedding=summary_embedding,
chunks=chunks,
search_space_id=search_space_id,
title=f"Task - {task_name}",
document_type=DocumentType.CLICKUP_CONNECTOR,
document_metadata={
"task_id": task_id,
"task_name": task_name,
"task_status": task_status,
"task_priority": task_priority,
"task_assignees": task_assignees,
"task_due_date": task_due_date,
"task_created": task_created,
"task_updated": task_updated,
"indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
},
content=task_content,
content_hash=content_hash,
embedding=summary_embedding,
chunks=chunks,
)
session.add(document)
@ -2968,12 +2956,11 @@ async def index_clickup_tasks(
except Exception as e:
logger.error(
f"Error processing task {task.get('name', 'Unknown')}: {e!s}",
exc_info=True,
f"Error processing task {task.get('name', 'Unknown')}: {e!s}",
exc_info=True,
)
documents_skipped += 1
# Update the last_indexed_at timestamp for the connector only if requested
total_processed = documents_indexed
if update_last_indexed:
@ -3006,22 +2993,22 @@ async def index_clickup_tasks(
) # Return None as the error message to indicate success
except SQLAlchemyError as db_error:
await session.rollback()
await task_logger.log_task_failure(
log_entry,
f"Database error during Cickup indexing for connector {connector_id}",
str(db_error),
{"error_type": "SQLAlchemyError"},
)
logger.error(f"Database error: {db_error!s}", exc_info=True)
return 0, f"Database error: {db_error!s}"
await session.rollback()
await task_logger.log_task_failure(
log_entry,
f"Database error during Cickup indexing for connector {connector_id}",
str(db_error),
{"error_type": "SQLAlchemyError"},
)
logger.error(f"Database error: {db_error!s}", exc_info=True)
return 0, f"Database error: {db_error!s}"
except Exception as e:
await session.rollback()
await task_logger.log_task_failure(
log_entry,
f"Failed to index ClickUp tasks for connector {connector_id}",
str(e),
{"error_type": type(e).__name__},
)
logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True)
return 0, f"Failed to index ClickUp tasks: {e!s}"
await session.rollback()
await task_logger.log_task_failure(
log_entry,
f"Failed to index ClickUp tasks for connector {connector_id}",
str(e),
{"error_type": type(e).__name__},
)
logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True)
return 0, f"Failed to index ClickUp tasks: {e!s}"

28
surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx
View file

@ -1,11 +1,15 @@
"use client";
import { useState } from "react";
import { useRouter } from "next/navigation";
import { zodResolver } from "@hookform/resolvers/zod";
import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { useState } from "react";
import { useForm } from "react-hook-form";
import { toast } from "sonner";
import * as z from "zod";
import { Button } from "@/components/ui/button";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import {
Form,
FormControl,
@ -16,10 +20,6 @@ import {
FormMessage,
} from "@/components/ui/form";
import { Input } from "@/components/ui/input";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react";
import Link from "next/link";
import { toast } from "sonner";
import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors";
interface ClickUpConnectorPageProps {
@ -103,7 +103,8 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
<CardHeader>
<CardTitle>ClickUp Configuration</CardTitle>
<CardDescription>
Enter your ClickUp API token to connect your workspace. You can generate a personal API token from your ClickUp settings.
Enter your ClickUp API token to connect your workspace. You can generate a personal API
token from your ClickUp settings.
</CardDescription>
</CardHeader>
<CardContent>
@ -116,10 +117,7 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
<FormItem>
<FormLabel>Connector Name</FormLabel>
<FormControl>
<Input
placeholder="ClickUp Connector"
{...field}
/>
<Input placeholder="ClickUp Connector" {...field} />
</FormControl>
<FormDescription>
A friendly name to identify this ClickUp connector.
@ -199,15 +197,11 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
</CardHeader>
<CardContent className="space-y-4">
<div className="space-y-2">
<p className="text-sm text-muted-foreground">
1. Log in to your ClickUp account
</p>
<p className="text-sm text-muted-foreground">1. Log in to your ClickUp account</p>
<p className="text-sm text-muted-foreground">
2. Click your avatar in the upper-right corner and select "Settings"
</p>
<p className="text-sm text-muted-foreground">
3. In the sidebar, click "Apps"
</p>
<p className="text-sm text-muted-foreground">3. In the sidebar, click "Apps"</p>
<p className="text-sm text-muted-foreground">
4. Under "API Token", click "Generate" or "Regenerate"
</p>

2
surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx
View file

@ -8,6 +8,7 @@ import {
IconBrandSlack,
IconBrandWindows,
IconBrandZoom,
IconChecklist,
IconChevronDown,
IconChevronRight,
IconLayoutKanban,
@ -15,7 +16,6 @@ import {
IconMail,
IconTicket,
IconWorldWww,
IconChecklist,
} from "@tabler/icons-react";
import { AnimatePresence, motion, type Variants } from "framer-motion";
import Link from "next/link";