vrr/SurfSense
2
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2025-09-02 02:29:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add coderabbit suggestions

Browse source
This commit is contained in:
CREDO23 2025-07-30 22:25:47 +02:00
parent 9a98742f81
commit 4cb00735ac
4 changed files with 76 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
surfsense_backend/app/connectors/clickup_connector.py
View file

@ -5,7 +5,6 @@ A module for retrieving data from ClickUp.
Allows fetching tasks from workspaces and lists. Allows fetching tasks from workspaces and lists.
""" """
from datetime import datetime
from typing import Any from typing import Any
import requests import requests
@ -169,14 +168,6 @@ class ClickUpConnector:
Tuple containing (tasks list, error message or None) Tuple containing (tasks list, error message or None)
""" """
try: try:
# Convert dates to Unix timestamps (milliseconds)
start_timestamp = int(
datetime.strptime(start_date, "%Y-%m-%d").timestamp() * 1000
)
end_timestamp = int(
datetime.strptime(end_date, "%Y-%m-%d").timestamp() * 1000
)
# TODO : Include date range in api request # TODO : Include date range in api request
params = { params = {

141
surfsense_backend/app/tasks/connectors_indexing_tasks.py
View file

@ -1,10 +1,8 @@
import asyncio import asyncio
import hashlib
import logging import logging
from datetime import UTC, datetime, timedelta from datetime import UTC, datetime, timedelta
from slack_sdk.errors import SlackApiError from slack_sdk.errors import SlackApiError
from sqlalchemy import func
from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.exc import SQLAlchemyError
from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.future import select from sqlalchemy.future import select
@ -2742,7 +2740,6 @@ async def index_clickup_tasks(
) )
try: try:
# Get connector configuration # Get connector configuration
result = await session.execute( result = await session.execute(
select(SearchSourceConnector).filter( select(SearchSourceConnector).filter(
@ -2828,7 +2825,9 @@ async def index_clickup_tasks(
include_closed=True, include_closed=True,
) )
if error: if error:
logger.warning(f"Error fetching tasks from workspace {workspace_name}: {error}") logger.warning(
f"Error fetching tasks from workspace {workspace_name}: {error}"
)
continue continue
else: else:
tasks = clickup_client.get_workspace_tasks( tasks = clickup_client.get_workspace_tasks(
@ -2848,12 +2847,15 @@ async def index_clickup_tasks(
task_name = task.get("name", "Untitled Task") task_name = task.get("name", "Untitled Task")
task_description = task.get("description", "") task_description = task.get("description", "")
task_status = task.get("status", {}).get("status", "Unknown") task_status = task.get("status", {}).get("status", "Unknown")
task_priority = task.get("priority", {}).get("priority", "Unknown") if task.get("priority") else "None" task_priority = (
task.get("priority", {}).get("priority", "Unknown")
if task.get("priority")
else "None"
)
task_assignees = task.get("assignees", []) task_assignees = task.get("assignees", [])
task_due_date = task.get("due_date") task_due_date = task.get("due_date")
task_created = task.get("date_created") task_created = task.get("date_created")
task_updated = task.get("date_updated") task_updated = task.get("date_updated")
task_url = task.get("url", "")
# Get list and space information # Get list and space information
task_list = task.get("list", {}) task_list = task.get("list", {})
@ -2867,15 +2869,20 @@ async def index_clickup_tasks(
if task_description: if task_description:
content_parts.append(f"Description: {task_description}") content_parts.append(f"Description: {task_description}")
content_parts.extend([ content_parts.extend(
f"Status: {task_status}", [
f"Priority: {task_priority}", f"Status: {task_status}",
f"List: {task_list_name}", f"Priority: {task_priority}",
f"Space: {task_space_name}", f"List: {task_list_name}",
]) f"Space: {task_space_name}",
]
)
if task_assignees: if task_assignees:
assignee_names = [assignee.get("username", "Unknown") for assignee in task_assignees] assignee_names = [
assignee.get("username", "Unknown")
for assignee in task_assignees
]
content_parts.append(f"Assignees: {', '.join(assignee_names)}") content_parts.append(f"Assignees: {', '.join(assignee_names)}")
if task_due_date: if task_due_date:
@ -2887,53 +2894,34 @@ async def index_clickup_tasks(
logger.warning(f"Skipping task with no content: {task_name}") logger.warning(f"Skipping task with no content: {task_name}")
continue continue
# Create document metadata
document_metadata = {
"task_id": task_id,
"task_name": task_name,
"task_url": task_url,
"task_status": task_status,
"task_priority": task_priority,
"task_assignees": task_assignees,
"task_due_date": task_due_date,
"task_created": task_created,
"task_updated": task_updated,
"task_list_name": task_list_name,
"task_space_name": task_space_name,
"workspace_id": workspace_id,
"workspace_name": workspace_name,
"connector_id": connector_id,
"source": "CLICKUP_CONNECTOR",
}
# Generate content hash # Generate content hash
content_hash = generate_content_hash(task_content, search_space_id) content_hash = generate_content_hash(task_content, search_space_id)
# Check if document already exists # Check if document already exists
existing_doc_by_hash_result = await session.execute( existing_doc_by_hash_result = await session.execute(
select(Document).where(Document.content_hash == content_hash) select(Document).where(Document.content_hash == content_hash)
) )
existing_document_by_hash = ( existing_document_by_hash = (
existing_doc_by_hash_result.scalars().first() existing_doc_by_hash_result.scalars().first()
) )
if existing_document_by_hash: if existing_document_by_hash:
logger.info( logger.info(
f"Document with content hash {content_hash} already exists for task {task_name}. Skipping processing." f"Document with content hash {content_hash} already exists for task {task_name}. Skipping processing."
) )
documents_skipped += 1 documents_skipped += 1
continue continue
# Generate embedding for the summary # Generate embedding for the summary
summary_embedding = config.embedding_model_instance.embed( summary_embedding = config.embedding_model_instance.embed(
task_content task_content
) )
# Process chunks - using the full page content with comments # Process chunks - using the full page content with comments
chunks = [ chunks = [
Chunk( Chunk(
content=chunk.text, content=chunk.text,
embedding=config.embedding_model_instance.embed(chunk.text), embedding=config.embedding_model_instance.embed(chunk.text),
) )
for chunk in config.chunker_instance.chunk(task_content) for chunk in config.chunker_instance.chunk(task_content)
] ]
@ -2942,24 +2930,24 @@ async def index_clickup_tasks(
logger.info(f"Creating new document for task {task_name}") logger.info(f"Creating new document for task {task_name}")
document = Document( document = Document(
search_space_id=search_space_id, search_space_id=search_space_id,
title=f"Task - {task_name}", title=f"Task - {task_name}",
document_type=DocumentType.CLICKUP_CONNECTOR, document_type=DocumentType.CLICKUP_CONNECTOR,
document_metadata={ document_metadata={
"task_id": task_id, "task_id": task_id,
"task_name": task_name, "task_name": task_name,
"task_status": task_status, "task_status": task_status,
"task_priority": task_priority, "task_priority": task_priority,
"task_assignees": task_assignees, "task_assignees": task_assignees,
"task_due_date": task_due_date, "task_due_date": task_due_date,
"task_created": task_created, "task_created": task_created,
"task_updated": task_updated, "task_updated": task_updated,
"indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
}, },
content=task_content, content=task_content,
content_hash=content_hash, content_hash=content_hash,
embedding=summary_embedding, embedding=summary_embedding,
chunks=chunks, chunks=chunks,
) )
session.add(document) session.add(document)
@ -2968,12 +2956,11 @@ async def index_clickup_tasks(
except Exception as e: except Exception as e:
logger.error( logger.error(
f"Error processing task {task.get('name', 'Unknown')}: {e!s}", f"Error processing task {task.get('name', 'Unknown')}: {e!s}",
exc_info=True, exc_info=True,
) )
documents_skipped += 1 documents_skipped += 1
# Update the last_indexed_at timestamp for the connector only if requested # Update the last_indexed_at timestamp for the connector only if requested
total_processed = documents_indexed total_processed = documents_indexed
if update_last_indexed: if update_last_indexed:
@ -3006,22 +2993,22 @@ async def index_clickup_tasks(
) # Return None as the error message to indicate success ) # Return None as the error message to indicate success
except SQLAlchemyError as db_error: except SQLAlchemyError as db_error:
await session.rollback() await session.rollback()
await task_logger.log_task_failure( await task_logger.log_task_failure(
log_entry, log_entry,
f"Database error during Cickup indexing for connector {connector_id}", f"Database error during Cickup indexing for connector {connector_id}",
str(db_error), str(db_error),
{"error_type": "SQLAlchemyError"}, {"error_type": "SQLAlchemyError"},
) )
logger.error(f"Database error: {db_error!s}", exc_info=True) logger.error(f"Database error: {db_error!s}", exc_info=True)
return 0, f"Database error: {db_error!s}" return 0, f"Database error: {db_error!s}"
except Exception as e: except Exception as e:
await session.rollback() await session.rollback()
await task_logger.log_task_failure( await task_logger.log_task_failure(
log_entry, log_entry,
f"Failed to index ClickUp tasks for connector {connector_id}", f"Failed to index ClickUp tasks for connector {connector_id}",
str(e), str(e),
{"error_type": type(e).__name__}, {"error_type": type(e).__name__},
) )
logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True) logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True)
return 0, f"Failed to index ClickUp tasks: {e!s}" return 0, f"Failed to index ClickUp tasks: {e!s}"

28
surfsense_web/app/dashboard/[search_space_id]/connectors/add/clickup-connector/page.tsx
View file

@ -1,11 +1,15 @@
"use client"; "use client";
import { useState } from "react";
import { useRouter } from "next/navigation";
import { zodResolver } from "@hookform/resolvers/zod"; import { zodResolver } from "@hookform/resolvers/zod";
import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react";
import Link from "next/link";
import { useRouter } from "next/navigation";
import { useState } from "react";
import { useForm } from "react-hook-form"; import { useForm } from "react-hook-form";
import { toast } from "sonner";
import * as z from "zod"; import * as z from "zod";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { import {
Form, Form,
FormControl, FormControl,
@ -16,10 +20,6 @@ import {
FormMessage, FormMessage,
} from "@/components/ui/form"; } from "@/components/ui/form";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card";
import { ArrowLeft, ExternalLink, Eye, EyeOff } from "lucide-react";
import Link from "next/link";
import { toast } from "sonner";
import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors";
interface ClickUpConnectorPageProps { interface ClickUpConnectorPageProps {
@ -103,7 +103,8 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
<CardHeader> <CardHeader>
<CardTitle>ClickUp Configuration</CardTitle> <CardTitle>ClickUp Configuration</CardTitle>
<CardDescription> <CardDescription>
Enter your ClickUp API token to connect your workspace. You can generate a personal API token from your ClickUp settings. Enter your ClickUp API token to connect your workspace. You can generate a personal API
token from your ClickUp settings.
</CardDescription> </CardDescription>
</CardHeader> </CardHeader>
<CardContent> <CardContent>
@ -116,10 +117,7 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
<FormItem> <FormItem>
<FormLabel>Connector Name</FormLabel> <FormLabel>Connector Name</FormLabel>
<FormControl> <FormControl>
<Input <Input placeholder="ClickUp Connector" {...field} />
placeholder="ClickUp Connector"
{...field}
/>
</FormControl> </FormControl>
<FormDescription> <FormDescription>
A friendly name to identify this ClickUp connector. A friendly name to identify this ClickUp connector.
@ -199,15 +197,11 @@ export default function ClickUpConnectorPage({ params }: ClickUpConnectorPagePro
</CardHeader> </CardHeader>
<CardContent className="space-y-4"> <CardContent className="space-y-4">
<div className="space-y-2"> <div className="space-y-2">
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">1. Log in to your ClickUp account</p>
1. Log in to your ClickUp account
</p>
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">
2. Click your avatar in the upper-right corner and select "Settings" 2. Click your avatar in the upper-right corner and select "Settings"
</p> </p>
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">3. In the sidebar, click "Apps"</p>
3. In the sidebar, click "Apps"
</p>
<p className="text-sm text-muted-foreground"> <p className="text-sm text-muted-foreground">
4. Under "API Token", click "Generate" or "Regenerate" 4. Under "API Token", click "Generate" or "Regenerate"
</p> </p>

2
surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx
View file

@ -8,6 +8,7 @@ import {
IconBrandSlack, IconBrandSlack,
IconBrandWindows, IconBrandWindows,
IconBrandZoom, IconBrandZoom,
IconChecklist,
IconChevronDown, IconChevronDown,
IconChevronRight, IconChevronRight,
IconLayoutKanban, IconLayoutKanban,
@ -15,7 +16,6 @@ import {
IconMail, IconMail,
IconTicket, IconTicket,
IconWorldWww, IconWorldWww,
IconChecklist,
} from "@tabler/icons-react"; } from "@tabler/icons-react";
import { AnimatePresence, motion, type Variants } from "framer-motion"; import { AnimatePresence, motion, type Variants } from "framer-motion";
import Link from "next/link"; import Link from "next/link";