From 655352fc09cc35586b9d0b63dc54c86ac85940b2 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Fri, 25 Jul 2025 00:17:06 +0200 Subject: [PATCH] fix issues indexing with jira connector --- ...cb2962bf19c1099cfe708e42daa0097f94976.json | 1 + .../app/connectors/jira_connector.py | 2 +- .../app/tasks/connectors_indexing_tasks.py | 289 ++- .../documents/(manage)/page.tsx | 2026 +++++++++-------- 4 files changed, 1262 insertions(+), 1056 deletions(-) create mode 100644 node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json diff --git a/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json b/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json new file mode 100644 index 0000000..f10b141 --- /dev/null +++ b/node_modules/.cache/prettier/.prettier-caches/a2ecb2962bf19c1099cfe708e42daa0097f94976.json @@ -0,0 +1 @@ +{"2d0ec64d93969318101ee479b664221b32241665":{"files":{"surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx":["k+EnkTKWK6V5ATYvtszJhNn43kI=",true]},"modified":1753395412751}} \ No newline at end of file diff --git a/surfsense_backend/app/connectors/jira_connector.py b/surfsense_backend/app/connectors/jira_connector.py index 2325a66..eeecb0f 100644 --- a/surfsense_backend/app/connectors/jira_connector.py +++ b/surfsense_backend/app/connectors/jira_connector.py @@ -252,7 +252,7 @@ class JiraConnector: fields.append("comment") params = { - "jql": jql, + "jql": "", "fields": ",".join(fields), "maxResults": 100, "startAt": 0, diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index b01a2a1..fae6417 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -1,5 +1,4 @@ import asyncio -import json import logging from datetime import datetime, timedelta, timezone from typing import Optional, Tuple @@ -1376,9 +1375,9 @@ async def index_linear_issues( # Process each issue for issue in issues: try: - issue_id = issue.get("id") - issue_identifier = issue.get("identifier", "") - issue_title = issue.get("title", "") + issue_id = issue.get("key") + issue_identifier = issue.get("id", "") + issue_title = issue.get("key", "") if not issue_id or not issue_title: logger.warning( @@ -2026,11 +2025,13 @@ async def index_jira_issues( try: # Get the connector from the database result = await session.execute( - select(SearchSourceConnector).where( - SearchSourceConnector.id == connector_id + select(SearchSourceConnector).filter( + SearchSourceConnector.id == connector_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.JIRA_CONNECTOR, ) ) - connector = result.scalar_one_or_none() + connector = result.scalars().first() if not connector: await task_logger.log_task_failure( @@ -2071,15 +2072,43 @@ async def index_jira_issues( # Fall back to calculating dates based on last_indexed_at calculated_end_date = datetime.now() + # Use last_indexed_at as start date if available, otherwise use 365 days ago if connector.last_indexed_at: - calculated_start_date = connector.last_indexed_at - else: - # If never indexed, go back 30 days - calculated_start_date = calculated_end_date - timedelta(days=30) + # Convert dates to be comparable (both timezone-naive) + last_indexed_naive = ( + connector.last_indexed_at.replace(tzinfo=None) + if connector.last_indexed_at.tzinfo + else connector.last_indexed_at + ) - start_date_str = calculated_start_date.strftime("%Y-%m-%d") - end_date_str = calculated_end_date.strftime("%Y-%m-%d") + # Check if last_indexed_at is in the future or after end_date + if last_indexed_naive > calculated_end_date: + logger.warning( + f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 365 days ago instead." + ) + calculated_start_date = calculated_end_date - timedelta(days=365) + else: + calculated_start_date = last_indexed_naive + logger.info( + f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date" + ) + else: + calculated_start_date = calculated_end_date - timedelta( + days=365 + ) # Use 365 days as default + logger.info( + f"No last_indexed_at found, using {calculated_start_date.strftime('%Y-%m-%d')} (365 days ago) as start date" + ) + + # Use calculated dates if not provided + start_date_str = ( + start_date if start_date else calculated_start_date.strftime("%Y-%m-%d") + ) + end_date_str = ( + end_date if end_date else calculated_end_date.strftime("%Y-%m-%d") + ) else: + # Use provided dates start_date_str = start_date end_date_str = end_date @@ -2099,8 +2128,6 @@ async def index_jira_issues( start_date=start_date_str, end_date=end_date_str, include_comments=True ) - print(json.dumps(issues, indent=2)) - if error: logger.error(f"Failed to get Jira issues: {error}") @@ -2133,146 +2160,174 @@ async def index_jira_issues( logger.info(f"Retrieved {len(issues)} issues from Jira API") - await task_logger.log_task_progress( - log_entry, - f"Retrieved {len(issues)} issues from Jira API", - {"stage": "processing_issues", "issues_found": len(issues)}, - ) - except Exception as e: - await task_logger.log_task_failure( - log_entry, - f"Error fetching Jira issues: {str(e)}", - "Fetch Error", - {"error_type": type(e).__name__}, - ) logger.error(f"Error fetching Jira issues: {str(e)}", exc_info=True) return 0, f"Error fetching Jira issues: {str(e)}" # Process and index each issue - indexed_count = 0 + documents_indexed = 0 + skipped_issues = [] + documents_skipped = 0 for issue in issues: try: + issue_id = issue.get("key") + issue_identifier = issue.get("key", "") + issue_title = issue.get("id", "") + + if not issue_id or not issue_title: + logger.warning( + f"Skipping issue with missing ID or title: {issue_id or 'Unknown'}" + ) + skipped_issues.append( + f"{issue_identifier or 'Unknown'} (missing data)" + ) + documents_skipped += 1 + continue + # Format the issue for better readability formatted_issue = jira_client.format_issue(issue) # Convert to markdown - issue_markdown = jira_client.format_issue_to_markdown(formatted_issue) + issue_content = jira_client.format_issue_to_markdown(formatted_issue) - # Create document metadata - metadata = { - "issue_key": formatted_issue.get("key", ""), - "issue_title": formatted_issue.get("title", ""), - "status": formatted_issue.get("status", ""), - "priority": formatted_issue.get("priority", ""), - "issue_type": formatted_issue.get("issue_type", ""), - "project": formatted_issue.get("project", ""), - "assignee": ( - formatted_issue.get("assignee", {}).get("display_name", "") - if formatted_issue.get("assignee") - else "" - ), - "reporter": formatted_issue.get("reporter", {}).get( - "display_name", "" - ), - "created_at": formatted_issue.get("created_at", ""), - "updated_at": formatted_issue.get("updated_at", ""), - "comment_count": len(formatted_issue.get("comments", [])), - "connector_id": connector_id, - "source": "jira", - "base_url": jira_base_url, - } - - # Generate content hash - content_hash = generate_content_hash(issue_markdown) - - # Check if document already exists - existing_doc_result = await session.execute( - select(Document).where(Document.content_hash == content_hash) - ) - existing_doc = existing_doc_result.scalar_one_or_none() - - if existing_doc: - logger.debug( - f"Document with hash {content_hash} already exists, skipping" + if not issue_content: + logger.warning( + f"Skipping issue with no content: {issue_identifier} - {issue_title}" ) + skipped_issues.append(f"{issue_identifier} (no content)") + documents_skipped += 1 continue - # Create new document - document = Document( - title=f"Jira: {formatted_issue.get('key', 'Unknown')} - {formatted_issue.get('title', 'Untitled')}", - document_type=DocumentType.JIRA_CONNECTOR, - document_metadata=metadata, - content=issue_markdown, - content_hash=content_hash, - search_space_id=search_space_id, + # Create a simple summary + summary_content = f"Jira Issue {issue_identifier}: {issue_title}\n\nStatus: {formatted_issue.get('status', 'Unknown')}\n\n" + if formatted_issue.get("description"): + summary_content += ( + f"Description: {formatted_issue.get('description')}\n\n" + ) + + # Add comment count + comment_count = len(formatted_issue.get("comments", [])) + summary_content += f"Comments: {comment_count}" + + # Generate content hash + content_hash = generate_content_hash(issue_content, search_space_id) + + # Check if document already exists + existing_doc_by_hash_result = await session.execute( + select(Document).where(Document.content_hash == content_hash) + ) + existing_document_by_hash = ( + existing_doc_by_hash_result.scalars().first() ) - # Generate embedding - embedding = await config.embedding_model_instance.get_embedding( - issue_markdown + if existing_document_by_hash: + logger.info( + f"Document with content hash {content_hash} already exists for issue {issue_identifier}. Skipping processing." + ) + documents_skipped += 1 + continue + + # Generate embedding for the summary + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) + + # Process chunks - using the full issue content with comments + chunks = [ + Chunk( + content=chunk.text, + embedding=config.embedding_model_instance.embed(chunk.text), + ) + for chunk in config.chunker_instance.chunk(issue_content) + ] + + # Create and store new document + logger.info( + f"Creating new document for issue {issue_identifier} - {issue_title}" + ) + document = Document( + search_space_id=search_space_id, + title=f"Jira - {issue_identifier}: {issue_title}", + document_type=DocumentType.JIRA_CONNECTOR, + document_metadata={ + "issue_id": issue_id, + "issue_identifier": issue_identifier, + "issue_title": issue_title, + "state": formatted_issue.get("status", "Unknown"), + "comment_count": comment_count, + "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + }, + content=summary_content, + content_hash=content_hash, + embedding=summary_embedding, + chunks=chunks, ) - document.embedding = embedding session.add(document) - await session.flush() # Flush to get the document ID - - # Create chunks for the document - chunks = await config.chunking_model_instance.chunk_document( - issue_markdown - ) - - for chunk_content in chunks: - chunk_embedding = ( - await config.embedding_model_instance.get_embedding( - chunk_content - ) - ) - - chunk = Chunk( - content=chunk_content, - embedding=chunk_embedding, - document_id=document.id, - ) - session.add(chunk) - - indexed_count += 1 - logger.debug( - f"Indexed Jira issue: {formatted_issue.get('key', 'Unknown')}" + documents_indexed += 1 + logger.info( + f"Successfully indexed new issue {issue_identifier} - {issue_title}" ) except Exception as e: logger.error( - f"Error processing Jira issue {issue.get('key', 'Unknown')}: {str(e)}", + f"Error processing issue {issue.get('identifier', 'Unknown')}: {str(e)}", exc_info=True, ) - continue + skipped_issues.append( + f"{issue.get('identifier', 'Unknown')} (processing error)" + ) + documents_skipped += 1 + continue # Skip this issue and continue with others + + # Update the last_indexed_at timestamp for the connector only if requested + total_processed = documents_indexed + if update_last_indexed: + connector.last_indexed_at = datetime.now() + logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}") # Commit all changes await session.commit() + logger.info("Successfully committed all JIRA document changes to database") - # Update last_indexed_at timestamp - if update_last_indexed: - connector.last_indexed_at = datetime.now() - await session.commit() - logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}") - + # Log success await task_logger.log_task_success( log_entry, - f"Successfully indexed {indexed_count} Jira issues", - {"issues_indexed": indexed_count}, + f"Successfully completed JIRA indexing for connector {connector_id}", + { + "issues_processed": total_processed, + "documents_indexed": documents_indexed, + "documents_skipped": documents_skipped, + "skipped_issues_count": len(skipped_issues), + }, ) - logger.info(f"Successfully indexed {indexed_count} Jira issues") - return indexed_count, None + logger.info( + f"JIRA indexing completed: {documents_indexed} new issues, {documents_skipped} skipped" + ) + return ( + total_processed, + None, + ) # Return None as the error message to indicate success - except Exception as e: + except SQLAlchemyError as db_error: + await session.rollback() await task_logger.log_task_failure( log_entry, - f"Failed to index Jira issues: {str(e)}", + f"Database error during JIRA indexing for connector {connector_id}", + str(db_error), + {"error_type": "SQLAlchemyError"}, + ) + logger.error(f"Database error: {str(db_error)}", exc_info=True) + return 0, f"Database error: {str(db_error)}" + except Exception as e: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Failed to index JIRA issues for connector {connector_id}", str(e), {"error_type": type(e).__name__}, ) - logger.error(f"Failed to index Jira issues: {str(e)}", exc_info=True) - return 0, f"Failed to index Jira issues: {str(e)}" + logger.error(f"Failed to index JIRA issues: {str(e)}", exc_info=True) + return 0, f"Failed to index JIRA issues: {str(e)}" diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 7d9aa3c..3a4bd33 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -3,88 +3,111 @@ import { DocumentViewer } from "@/components/document-viewer"; import { JsonMetadataViewer } from "@/components/json-metadata-viewer"; import { - AlertDialog, - AlertDialogAction, - AlertDialogCancel, - AlertDialogContent, - AlertDialogDescription, - AlertDialogFooter, - AlertDialogHeader, - AlertDialogTitle, - AlertDialogTrigger, + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, + AlertDialogTrigger, } from "@/components/ui/alert-dialog"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { - DropdownMenu, - DropdownMenuCheckboxItem, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuLabel, - DropdownMenuSeparator, - DropdownMenuTrigger, + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, } from "@/components/ui/dropdown-menu"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; -import { Pagination, PaginationContent, PaginationItem } from "@/components/ui/pagination"; -import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover"; import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, + Pagination, + PaginationContent, + PaginationItem, +} from "@/components/ui/pagination"; +import { + Popover, + PopoverContent, + PopoverTrigger, +} from "@/components/ui/popover"; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, } from "@/components/ui/select"; import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, } from "@/components/ui/table"; import { useDocuments } from "@/hooks/use-documents"; import { cn } from "@/lib/utils"; -import { IconBrandDiscord, IconBrandGithub, IconBrandNotion, IconBrandSlack, IconBrandYoutube, IconLayoutKanban } from "@tabler/icons-react"; import { - ColumnDef, - ColumnFiltersState, - FilterFn, - PaginationState, - Row, - SortingState, - VisibilityState, - flexRender, - getCoreRowModel, - getFacetedUniqueValues, - getFilteredRowModel, - getPaginationRowModel, - getSortedRowModel, - useReactTable, + IconBrandDiscord, + IconBrandGithub, + IconBrandNotion, + IconBrandSlack, + IconBrandYoutube, + IconLayoutKanban, + IconBrandTrello, +} from "@tabler/icons-react"; +import { + ColumnDef, + ColumnFiltersState, + FilterFn, + PaginationState, + Row, + SortingState, + VisibilityState, + flexRender, + getCoreRowModel, + getFacetedUniqueValues, + getFilteredRowModel, + getPaginationRowModel, + getSortedRowModel, + useReactTable, } from "@tanstack/react-table"; import { AnimatePresence, motion } from "framer-motion"; import { - AlertCircle, - ChevronDown, - ChevronFirst, - ChevronLast, - ChevronLeft, - ChevronRight, - ChevronUp, - CircleAlert, - CircleX, - Columns3, - File, - FileX, - Filter, - Globe, - ListFilter, - MoreHorizontal, - Trash, - Webhook + AlertCircle, + ChevronDown, + ChevronFirst, + ChevronLast, + ChevronLeft, + ChevronRight, + ChevronUp, + CircleAlert, + CircleX, + Columns3, + File, + FileX, + Filter, + Globe, + ListFilter, + MoreHorizontal, + Trash, + Webhook, } from "lucide-react"; import { useParams } from "next/navigation"; -import React, { useContext, useEffect, useId, useMemo, useRef, useState } from "react"; +import React, { + useContext, + useEffect, + useId, + useMemo, + useRef, + useState, +} from "react"; import ReactMarkdown from "react-markdown"; import rehypeRaw from "rehype-raw"; import rehypeSanitize from "rehype-sanitize"; @@ -93,938 +116,1065 @@ import { toast } from "sonner"; // Define animation variants for reuse const fadeInScale = { - hidden: { opacity: 0, scale: 0.95 }, - visible: { - opacity: 1, - scale: 1, - transition: { type: "spring", stiffness: 300, damping: 30 } - }, - exit: { - opacity: 0, - scale: 0.95, - transition: { duration: 0.15 } - } + hidden: { opacity: 0, scale: 0.95 }, + visible: { + opacity: 1, + scale: 1, + transition: { type: "spring", stiffness: 300, damping: 30 }, + }, + exit: { + opacity: 0, + scale: 0.95, + transition: { duration: 0.15 }, + }, }; type Document = { - id: number; - title: string; - document_type: "EXTENSION" | "CRAWLED_URL" | "SLACK_CONNECTOR" | "NOTION_CONNECTOR" | "FILE" | "YOUTUBE_VIDEO" | "LINEAR_CONNECTOR" | "DISCORD_CONNECTOR"; - document_metadata: any; - content: string; - created_at: string; - search_space_id: number; + id: number; + title: string; + document_type: + | "EXTENSION" + | "CRAWLED_URL" + | "SLACK_CONNECTOR" + | "NOTION_CONNECTOR" + | "FILE" + | "YOUTUBE_VIDEO" + | "LINEAR_CONNECTOR" + | "DISCORD_CONNECTOR"; + document_metadata: any; + content: string; + created_at: string; + search_space_id: number; }; // Custom filter function for multi-column searching -const multiColumnFilterFn: FilterFn = (row, columnId, filterValue) => { - const searchableRowContent = `${row.original.title}`.toLowerCase(); - const searchTerm = (filterValue ?? "").toLowerCase(); - return searchableRowContent.includes(searchTerm); +const multiColumnFilterFn: FilterFn = ( + row, + columnId, + filterValue, +) => { + const searchableRowContent = `${row.original.title}`.toLowerCase(); + const searchTerm = (filterValue ?? "").toLowerCase(); + return searchableRowContent.includes(searchTerm); }; -const statusFilterFn: FilterFn = (row, columnId, filterValue: string[]) => { - if (!filterValue?.length) return true; - const status = row.getValue(columnId) as string; - return filterValue.includes(status); +const statusFilterFn: FilterFn = ( + row, + columnId, + filterValue: string[], +) => { + if (!filterValue?.length) return true; + const status = row.getValue(columnId) as string; + return filterValue.includes(status); }; // Add document type icons mapping const documentTypeIcons = { - EXTENSION: Webhook, - CRAWLED_URL: Globe, - SLACK_CONNECTOR: IconBrandSlack, - NOTION_CONNECTOR: IconBrandNotion, - FILE: File, - YOUTUBE_VIDEO: IconBrandYoutube, - GITHUB_CONNECTOR: IconBrandGithub, - LINEAR_CONNECTOR: IconLayoutKanban, - DISCORD_CONNECTOR: IconBrandDiscord, + EXTENSION: Webhook, + CRAWLED_URL: Globe, + SLACK_CONNECTOR: IconBrandSlack, + NOTION_CONNECTOR: IconBrandNotion, + FILE: File, + YOUTUBE_VIDEO: IconBrandYoutube, + GITHUB_CONNECTOR: IconBrandGithub, + LINEAR_CONNECTOR: IconLayoutKanban, + JIRA_CONNECTOR: IconBrandTrello, + DISCORD_CONNECTOR: IconBrandDiscord, } as const; const columns: ColumnDef[] = [ - { - id: "select", - header: ({ table }) => ( - table.toggleAllPageRowsSelected(!!value)} - aria-label="Select all" - /> - ), - cell: ({ row }) => ( - row.toggleSelected(!!value)} - aria-label="Select row" - /> - ), - size: 28, - enableSorting: false, - enableHiding: false, + { + id: "select", + header: ({ table }) => ( + table.toggleAllPageRowsSelected(!!value)} + aria-label="Select all" + /> + ), + cell: ({ row }) => ( + row.toggleSelected(!!value)} + aria-label="Select row" + /> + ), + size: 28, + enableSorting: false, + enableHiding: false, + }, + { + header: "Title", + accessorKey: "title", + cell: ({ row }) => { + const Icon = documentTypeIcons[row.original.document_type]; + return ( + + + {row.getValue("title")} + + ); }, - { - header: "Title", - accessorKey: "title", - cell: ({ row }) => { - const Icon = documentTypeIcons[row.original.document_type]; - return ( - - - {row.getValue("title")} - - ); - }, - size: 250, + size: 250, + }, + { + header: "Type", + accessorKey: "document_type", + cell: ({ row }) => { + const type = row.getValue( + "document_type", + ) as keyof typeof documentTypeIcons; + const Icon = documentTypeIcons[type]; + return ( +
+
+ +
+ + {type + .split("_") + .map((word) => word.charAt(0) + word.slice(1).toLowerCase()) + .join(" ")} + +
+ ); }, - { - header: "Type", - accessorKey: "document_type", - cell: ({ row }) => { - const type = row.getValue("document_type") as keyof typeof documentTypeIcons; - const Icon = documentTypeIcons[type]; - return ( -
-
- -
- - {type.split('_').map(word => word.charAt(0) + word.slice(1).toLowerCase()).join(' ')} - -
- ); - }, - size: 180, + size: 180, + }, + { + header: "Content Summary", + accessorKey: "content", + cell: ({ row }) => { + const content = row.getValue("content") as string; + const title = row.getValue("title") as string; + + // Create a truncated preview (first 150 characters) + const previewContent = + content.length > 150 ? content.substring(0, 150) + "..." : content; + + return ( + + ); }, - { - header: "Content Summary", - accessorKey: "content", - cell: ({ row }) => { - const content = row.getValue("content") as string; - const title = row.getValue("title") as string; - - // Create a truncated preview (first 150 characters) - const previewContent = content.length > 150 - ? content.substring(0, 150) + "..." - : content; - - return ( - - ); - }, - size: 300, - }, - { - header: "Created At", - accessorKey: "created_at", - cell: ({ row }) => { - const date = new Date(row.getValue("created_at")); - return date.toLocaleDateString(); - }, - size: 120, - }, - { - id: "actions", - header: () => Actions, - cell: ({ row }) => , - size: 60, - enableHiding: false, + size: 300, + }, + { + header: "Created At", + accessorKey: "created_at", + cell: ({ row }) => { + const date = new Date(row.getValue("created_at")); + return date.toLocaleDateString(); }, + size: 120, + }, + { + id: "actions", + header: () => Actions, + cell: ({ row }) => , + size: 60, + enableHiding: false, + }, ]; // Create a context to share the deleteDocument function const DocumentsContext = React.createContext<{ - deleteDocument: (id: number) => Promise; - refreshDocuments: () => Promise; + deleteDocument: (id: number) => Promise; + refreshDocuments: () => Promise; } | null>(null); export default function DocumentsTable() { - const id = useId(); - const params = useParams(); - const searchSpaceId = Number(params.search_space_id); - const { documents, loading, error, refreshDocuments, deleteDocument } = useDocuments(searchSpaceId); - - // console.log("Search Space ID:", searchSpaceId); - // console.log("Documents loaded:", documents?.length); - - useEffect(() => { - console.log("Delete document function available:", !!deleteDocument); - }, [deleteDocument]); - - const [columnFilters, setColumnFilters] = useState([]); - const [columnVisibility, setColumnVisibility] = useState({}); - const [pagination, setPagination] = useState({ - pageIndex: 0, - pageSize: 10, - }); - const inputRef = useRef(null); + const id = useId(); + const params = useParams(); + const searchSpaceId = Number(params.search_space_id); + const { documents, loading, error, refreshDocuments, deleteDocument } = + useDocuments(searchSpaceId); - const [sorting, setSorting] = useState([ - { - id: "title", - desc: false, - }, - ]); + // console.log("Search Space ID:", searchSpaceId); + // console.log("Documents loaded:", documents?.length); - const [data, setData] = useState([]); - - useEffect(() => { - if (documents) { - setData(documents); - } - }, [documents]); + useEffect(() => { + console.log("Delete document function available:", !!deleteDocument); + }, [deleteDocument]); - const handleDeleteRows = async () => { - const selectedRows = table.getSelectedRowModel().rows; - // console.log("Deleting selected rows:", selectedRows.length); - - if (selectedRows.length === 0) { - toast.error("No rows selected"); - return; - } - - // Create an array of promises for each delete operation - const deletePromises = selectedRows.map(row => { - // console.log("Deleting row with ID:", row.original.id); - return deleteDocument(row.original.id); - }); - - try { - // Execute all delete operations - const results = await Promise.all(deletePromises); - // console.log("Delete results:", results); - - // Check if all deletions were successful - const allSuccessful = results.every(result => result === true); - - if (allSuccessful) { - toast.success(`Successfully deleted ${selectedRows.length} document(s)`); - } else { - toast.error("Some documents could not be deleted"); - } - - // Refresh the documents list after all deletions - await refreshDocuments(); - table.resetRowSelection(); - } catch (error: any) { - console.error("Error deleting documents:", error); - toast.error("Error deleting documents"); - } - }; + const [columnFilters, setColumnFilters] = useState([]); + const [columnVisibility, setColumnVisibility] = useState({}); + const [pagination, setPagination] = useState({ + pageIndex: 0, + pageSize: 10, + }); + const inputRef = useRef(null); - const table = useReactTable({ - data, - columns, - getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), - onSortingChange: setSorting, - enableSortingRemoval: false, - getPaginationRowModel: getPaginationRowModel(), - onPaginationChange: setPagination, - onColumnFiltersChange: setColumnFilters, - onColumnVisibilityChange: setColumnVisibility, - getFilteredRowModel: getFilteredRowModel(), - getFacetedUniqueValues: getFacetedUniqueValues(), - state: { - sorting, - pagination, - columnFilters, - columnVisibility, - }, + const [sorting, setSorting] = useState([ + { + id: "title", + desc: false, + }, + ]); + + const [data, setData] = useState([]); + + useEffect(() => { + if (documents) { + setData(documents); + } + }, [documents]); + + const handleDeleteRows = async () => { + const selectedRows = table.getSelectedRowModel().rows; + // console.log("Deleting selected rows:", selectedRows.length); + + if (selectedRows.length === 0) { + toast.error("No rows selected"); + return; + } + + // Create an array of promises for each delete operation + const deletePromises = selectedRows.map((row) => { + // console.log("Deleting row with ID:", row.original.id); + return deleteDocument(row.original.id); }); - // Get unique status values - const uniqueStatusValues = useMemo(() => { - const statusColumn = table.getColumn("document_type"); + try { + // Execute all delete operations + const results = await Promise.all(deletePromises); + // console.log("Delete results:", results); - if (!statusColumn) return []; + // Check if all deletions were successful + const allSuccessful = results.every((result) => result === true); - const values = Array.from(statusColumn.getFacetedUniqueValues().keys()); + if (allSuccessful) { + toast.success( + `Successfully deleted ${selectedRows.length} document(s)`, + ); + } else { + toast.error("Some documents could not be deleted"); + } - return values.sort(); - }, [table.getColumn("document_type")?.getFacetedUniqueValues()]); + // Refresh the documents list after all deletions + await refreshDocuments(); + table.resetRowSelection(); + } catch (error: any) { + console.error("Error deleting documents:", error); + toast.error("Error deleting documents"); + } + }; - // Get counts for each status - const statusCounts = useMemo(() => { - const statusColumn = table.getColumn("document_type"); - if (!statusColumn) return new Map(); - return statusColumn.getFacetedUniqueValues(); - }, [table.getColumn("document_type")?.getFacetedUniqueValues()]); + const table = useReactTable({ + data, + columns, + getCoreRowModel: getCoreRowModel(), + getSortedRowModel: getSortedRowModel(), + onSortingChange: setSorting, + enableSortingRemoval: false, + getPaginationRowModel: getPaginationRowModel(), + onPaginationChange: setPagination, + onColumnFiltersChange: setColumnFilters, + onColumnVisibilityChange: setColumnVisibility, + getFilteredRowModel: getFilteredRowModel(), + getFacetedUniqueValues: getFacetedUniqueValues(), + state: { + sorting, + pagination, + columnFilters, + columnVisibility, + }, + }); - const selectedStatuses = useMemo(() => { - const filterValue = table.getColumn("document_type")?.getFilterValue() as string[]; - return filterValue ?? []; - }, [table.getColumn("document_type")?.getFilterValue()]); + // Get unique status values + const uniqueStatusValues = useMemo(() => { + const statusColumn = table.getColumn("document_type"); - const handleStatusChange = (checked: boolean, value: string) => { - const filterValue = table.getColumn("document_type")?.getFilterValue() as string[]; - const newFilterValue = filterValue ? [...filterValue] : []; + if (!statusColumn) return []; - if (checked) { - newFilterValue.push(value); - } else { - const index = newFilterValue.indexOf(value); - if (index > -1) { - newFilterValue.splice(index, 1); - } - } + const values = Array.from(statusColumn.getFacetedUniqueValues().keys()); - table.getColumn("document_type")?.setFilterValue(newFilterValue.length ? newFilterValue : undefined); - }; + return values.sort(); + }, [table.getColumn("document_type")?.getFacetedUniqueValues()]); - return ( - Promise.resolve(false)), - refreshDocuments: refreshDocuments || (() => Promise.resolve()) - }}> + // Get counts for each status + const statusCounts = useMemo(() => { + const statusColumn = table.getColumn("document_type"); + if (!statusColumn) return new Map(); + return statusColumn.getFacetedUniqueValues(); + }, [table.getColumn("document_type")?.getFacetedUniqueValues()]); + + const selectedStatuses = useMemo(() => { + const filterValue = table + .getColumn("document_type") + ?.getFilterValue() as string[]; + return filterValue ?? []; + }, [table.getColumn("document_type")?.getFilterValue()]); + + const handleStatusChange = (checked: boolean, value: string) => { + const filterValue = table + .getColumn("document_type") + ?.getFilterValue() as string[]; + const newFilterValue = filterValue ? [...filterValue] : []; + + if (checked) { + newFilterValue.push(value); + } else { + const index = newFilterValue.indexOf(value); + if (index > -1) { + newFilterValue.splice(index, 1); + } + } + + table + .getColumn("document_type") + ?.setFilterValue(newFilterValue.length ? newFilterValue : undefined); + }; + + return ( + Promise.resolve(false)), + refreshDocuments: refreshDocuments || (() => Promise.resolve()), + }} + > + + {/* Filters */} + +
+ {/* Filter by name or email */} - {/* Filters */} - + table.getColumn("title")?.setFilterValue(e.target.value) + } + placeholder="Filter by title..." + type="text" + aria-label="Filter by title" + /> + + + {Boolean(table.getColumn("title")?.getFilterValue()) && ( + { + table.getColumn("title")?.setFilterValue(""); + if (inputRef.current) { + inputRef.current.focus(); + } + }} + initial={{ opacity: 0, rotate: -90 }} + animate={{ opacity: 1, rotate: 0 }} + exit={{ opacity: 0, rotate: 90 }} + whileHover={{ scale: 1.1 }} + whileTap={{ scale: 0.9 }} > -
- {/* Filter by name or email */} - - table.getColumn("title")?.setFilterValue(e.target.value)} - placeholder="Filter by title..." - type="text" - aria-label="Filter by title" - /> - - - {Boolean(table.getColumn("title")?.getFilterValue()) && ( - { - table.getColumn("title")?.setFilterValue(""); - if (inputRef.current) { - inputRef.current.focus(); - } - }} - initial={{ opacity: 0, rotate: -90 }} - animate={{ opacity: 1, rotate: 0 }} - exit={{ opacity: 0, rotate: 90 }} - whileHover={{ scale: 1.1 }} - whileTap={{ scale: 0.9 }} - > - - )} - - {/* Filter by status */} - - - - - - - - -
-
Filters
-
- - {uniqueStatusValues.map((value, i) => ( - - handleStatusChange(checked, value)} - /> - - - ))} - -
-
- - - - {/* Toggle columns visibility */} - - - - - - - - - Toggle columns - {table - .getAllColumns() - .filter((column) => column.getCanHide()) - .map((column) => { - return ( - column.toggleVisibility(!!value)} - onSelect={(event) => event.preventDefault()} - > - {column.id} - - ); - })} - - - +
+ - {/* Table */} - + {loading ? ( +
+
+
+

+ Loading documents... +

+
+
+ ) : error ? ( +
+
+ +

+ Error loading documents +

+ -
-
- ) : data.length === 0 ? ( -
-
- -

No documents found

-
-
- ) : ( - - - {table.getHeaderGroups().map((headerGroup) => ( - - {headerGroup.headers.map((header) => { - return ( - - {header.isPlaceholder ? null : header.column.getCanSort() ? ( -
{ - // Enhanced keyboard handling for sorting - if ( - header.column.getCanSort() && - (e.key === "Enter" || e.key === " ") - ) { - e.preventDefault(); - header.column.getToggleSortingHandler()?.(e); - } - }} - tabIndex={header.column.getCanSort() ? 0 : undefined} - > - {flexRender(header.column.columnDef.header, header.getContext())} - {{ - asc: ( -
- ) : ( - flexRender(header.column.columnDef.header, header.getContext()) - )} - - ); - })} - - ))} - - - - {table.getRowModel().rows?.length ? ( - table.getRowModel().rows.map((row, index) => ( - - {row.getVisibleCells().map((cell) => ( - - {flexRender(cell.column.columnDef.cell, cell.getContext())} - - ))} - - )) - ) : ( - - - No documents found. - - - )} - - -
- )} - - - {/* Pagination */} -
- {/* Results per page */} - - - - - {/* Page number information */} - -

- - {table.getState().pagination.pageIndex * table.getState().pagination.pageSize + 1}- - {Math.min( - Math.max( - table.getState().pagination.pageIndex * table.getState().pagination.pageSize + - table.getState().pagination.pageSize, - 0, - ), - table.getRowCount(), - )} - {" "} - of {table.getRowCount().toString()} -

- + {header.isPlaceholder ? null : header.column.getCanSort() ? ( +
{ + // Enhanced keyboard handling for sorting + if ( + header.column.getCanSort() && + (e.key === "Enter" || e.key === " ") + ) { + e.preventDefault(); + header.column.getToggleSortingHandler()?.(e); + } + }} + tabIndex={ + header.column.getCanSort() ? 0 : undefined + } + > + {flexRender( + header.column.columnDef.header, + header.getContext(), + )} + {{ + asc: ( +
+ ) : ( + flexRender( + header.column.columnDef.header, + header.getContext(), + ) + )} + + ); + })} + + ))} + + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row, index) => ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender( + cell.column.columnDef.cell, + cell.getContext(), + )} + + ))} + + )) + ) : ( + + + No documents found. + + + )} + + + + )} + - {/* Pagination buttons */} -
- - - {/* First page button */} - - - - - - {/* Previous page button */} - - - - - - {/* Next page button */} - - - - - - {/* Last page button */} - - - - - - - -
-
- - - ); + {/* Pagination */} +
+ {/* Results per page */} + + + + + {/* Page number information */} + +

+ + {table.getState().pagination.pageIndex * + table.getState().pagination.pageSize + + 1} + - + {Math.min( + Math.max( + table.getState().pagination.pageIndex * + table.getState().pagination.pageSize + + table.getState().pagination.pageSize, + 0, + ), + table.getRowCount(), + )} + {" "} + of{" "} + + {table.getRowCount().toString()} + +

+ + + {/* Pagination buttons */} +
+ + + {/* First page button */} + + + + + + {/* Previous page button */} + + + + + + {/* Next page button */} + + + + + + {/* Last page button */} + + + + + + + +
+
+ + + ); } function RowActions({ row }: { row: Row }) { - const [isOpen, setIsOpen] = useState(false); - const [isDeleting, setIsDeleting] = useState(false); - const { deleteDocument, refreshDocuments } = useContext(DocumentsContext)!; - const document = row.original; + const [isOpen, setIsOpen] = useState(false); + const [isDeleting, setIsDeleting] = useState(false); + const { deleteDocument, refreshDocuments } = useContext(DocumentsContext)!; + const document = row.original; - const handleDelete = async () => { - setIsDeleting(true); - try { - await deleteDocument(document.id); - toast.success("Document deleted successfully"); - await refreshDocuments(); - } catch (error) { - console.error("Error deleting document:", error); - toast.error("Failed to delete document"); - } finally { - setIsDeleting(false); - setIsOpen(false); - } - }; + const handleDelete = async () => { + setIsDeleting(true); + try { + await deleteDocument(document.id); + toast.success("Document deleted successfully"); + await refreshDocuments(); + } catch (error) { + console.error("Error deleting document:", error); + toast.error("Failed to delete document"); + } finally { + setIsDeleting(false); + setIsOpen(false); + } + }; - return ( -
- - - - - - e.preventDefault()}> - View Metadata - - } - /> - - - - { - e.preventDefault(); - setIsOpen(true); - }} - > - Delete - - - - - Are you sure? - - This action cannot be undone. This will permanently delete the document. - - - - Cancel - { - e.preventDefault(); - handleDelete(); - }} - disabled={isDeleting} - > - {isDeleting ? "Deleting..." : "Delete"} - - - - - - -
- ); + return ( +
+ + + + + + e.preventDefault()}> + View Metadata + + } + /> + + + + { + e.preventDefault(); + setIsOpen(true); + }} + > + Delete + + + + + Are you sure? + + This action cannot be undone. This will permanently delete the + document. + + + + Cancel + { + e.preventDefault(); + handleDelete(); + }} + disabled={isDeleting} + > + {isDeleting ? "Deleting..." : "Delete"} + + + + + + +
+ ); } export { DocumentsTable }; -