diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 8a003ff..786c19d 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -9,17 +9,18 @@ POST /search-source-connectors/{connector_id}/index - Index content from a conne Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR, GITHUB_CONNECTOR, LINEAR_CONNECTOR). """ -from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks +from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks, Body from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError from typing import List, Dict, Any from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace, async_session_maker -from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead +from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase from app.users import current_active_user from app.utils.check_ownership import check_ownership -from pydantic import ValidationError +from pydantic import BaseModel, Field, ValidationError from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues +from app.connectors.github_connector import GitHubConnector from datetime import datetime, timezone, timedelta import logging @@ -28,6 +29,34 @@ logger = logging.getLogger(__name__) router = APIRouter() +# Use Pydantic's BaseModel here +class GitHubPATRequest(BaseModel): + github_pat: str = Field(..., description="GitHub Personal Access Token") + +# --- New Endpoint to list GitHub Repositories --- +@router.post("/github/repositories/", response_model=List[Dict[str, Any]]) +async def list_github_repositories( + pat_request: GitHubPATRequest, + user: User = Depends(current_active_user) # Ensure the user is logged in +): + """ + Fetches a list of repositories accessible by the provided GitHub PAT. + The PAT is used for this request only and is not stored. + """ + try: + # Initialize GitHubConnector with the provided PAT + github_client = GitHubConnector(token=pat_request.github_pat) + # Fetch repositories + repositories = github_client.get_user_repositories() + return repositories + except ValueError as e: + # Handle invalid token error specifically + logger.error(f"GitHub PAT validation failed for user {user.id}: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid GitHub PAT: {str(e)}") + except Exception as e: + logger.error(f"Failed to fetch GitHub repositories for user {user.id}: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to fetch GitHub repositories.") + @router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, @@ -76,6 +105,7 @@ async def create_search_source_connector( await session.rollback() raise except Exception as e: + logger.error(f"Failed to create search source connector: {str(e)}") await session.rollback() raise HTTPException( status_code=500, @@ -130,54 +160,84 @@ async def update_search_source_connector( ): """ Update a search source connector. - - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, etc.). - The config must contain the appropriate keys for the connector type. + Handles partial updates, including merging changes into the 'config' field. """ - try: - db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + + # Convert the sparse update data (only fields present in request) to a dict + update_data = connector_update.model_dump(exclude_unset=True) + + # Special handling for 'config' field + if "config" in update_data: + incoming_config = update_data["config"] # Config data from the request + existing_config = db_connector.config if db_connector.config else {} # Current config from DB - # If connector type is being changed, check if one of that type already exists - if connector_update.connector_type != db_connector.connector_type: + # Merge incoming config into existing config + # This preserves existing keys (like GITHUB_PAT) if they are not in the incoming data + merged_config = existing_config.copy() + merged_config.update(incoming_config) + + # -- Validation after merging -- + # Validate the *merged* config based on the connector type + # We need the connector type - use the one from the update if provided, else the existing one + current_connector_type = connector_update.connector_type if connector_update.connector_type is not None else db_connector.connector_type + + try: + # We can reuse the base validator by creating a temporary base model instance + # Note: This assumes 'name' and 'is_indexable' are not crucial for config validation itself + temp_data_for_validation = { + "name": db_connector.name, # Use existing name + "connector_type": current_connector_type, + "is_indexable": db_connector.is_indexable, # Use existing value + "last_indexed_at": db_connector.last_indexed_at, # Not used by validator + "config": merged_config + } + SearchSourceConnectorBase.model_validate(temp_data_for_validation) + except ValidationError as e: + # Raise specific validation error for the merged config + raise HTTPException( + status_code=422, + detail=f"Validation error for merged config: {str(e)}" + ) + + # If validation passes, update the main update_data dict with the merged config + update_data["config"] = merged_config + + # Apply all updates (including the potentially merged config) + for key, value in update_data.items(): + # Prevent changing connector_type if it causes a duplicate (check moved here) + if key == "connector_type" and value != db_connector.connector_type: result = await session.execute( select(SearchSourceConnector) .filter( SearchSourceConnector.user_id == user.id, - SearchSourceConnector.connector_type == connector_update.connector_type, + SearchSourceConnector.connector_type == value, SearchSourceConnector.id != connector_id ) ) existing_connector = result.scalars().first() - if existing_connector: raise HTTPException( status_code=409, - detail=f"A connector with type {connector_update.connector_type} already exists. Each user can have only one connector of each type." + detail=f"A connector with type {value} already exists. Each user can have only one connector of each type." ) - update_data = connector_update.model_dump(exclude_unset=True) - for key, value in update_data.items(): - setattr(db_connector, key, value) + setattr(db_connector, key, value) + + try: await session.commit() await session.refresh(db_connector) return db_connector - except ValidationError as e: - await session.rollback() - raise HTTPException( - status_code=422, - detail=f"Validation error: {str(e)}" - ) except IntegrityError as e: await session.rollback() + # This might occur if connector_type constraint is violated somehow after the check raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {str(e)}" + detail=f"Database integrity error during update: {str(e)}" ) - except HTTPException: - await session.rollback() - raise except Exception as e: await session.rollback() + logger.error(f"Failed to update search source connector {connector_id}: {e}", exc_info=True) raise HTTPException( status_code=500, detail=f"Failed to update search source connector: {str(e)}" diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 032e486..6accc12 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -1,16 +1,15 @@ from datetime import datetime import uuid -from typing import Dict, Any +from typing import Dict, Any, Optional from pydantic import BaseModel, field_validator from .base import IDModel, TimestampModel from app.db import SearchSourceConnectorType -from fastapi import HTTPException class SearchSourceConnectorBase(BaseModel): name: str connector_type: SearchSourceConnectorType is_indexable: bool - last_indexed_at: datetime | None + last_indexed_at: Optional[datetime] = None config: Dict[str, Any] @field_validator('config') @@ -59,8 +58,8 @@ class SearchSourceConnectorBase(BaseModel): raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty") elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: - # For GITHUB_CONNECTOR, only allow GITHUB_PAT - allowed_keys = ["GITHUB_PAT"] + # For GITHUB_CONNECTOR, only allow GITHUB_PAT and repo_full_names + allowed_keys = ["GITHUB_PAT", "repo_full_names"] if set(config.keys()) != set(allowed_keys): raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") @@ -68,6 +67,10 @@ class SearchSourceConnectorBase(BaseModel): if not config.get("GITHUB_PAT"): raise ValueError("GITHUB_PAT cannot be empty") + # Ensure the repo_full_names is present and is a non-empty list + repo_full_names = config.get("repo_full_names") + if not isinstance(repo_full_names, list) or not repo_full_names: + raise ValueError("repo_full_names must be a non-empty list of strings") elif connector_type == SearchSourceConnectorType.LINEAR_CONNECTOR: # For LINEAR_CONNECTOR, only allow LINEAR_API_KEY allowed_keys = ["LINEAR_API_KEY"] @@ -83,8 +86,12 @@ class SearchSourceConnectorBase(BaseModel): class SearchSourceConnectorCreate(SearchSourceConnectorBase): pass -class SearchSourceConnectorUpdate(SearchSourceConnectorBase): - pass +class SearchSourceConnectorUpdate(BaseModel): + name: Optional[str] = None + connector_type: Optional[SearchSourceConnectorType] = None + is_indexable: Optional[bool] = None + last_indexed_at: Optional[datetime] = None + config: Optional[Dict[str, Any]] = None class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel): user_id: uuid.UUID diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index dbdb24c..7c21062 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Dict, Any, Tuple +from typing import Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.future import select @@ -639,10 +639,15 @@ async def index_github_repos( if not connector: return 0, f"Connector with ID {connector_id} not found or is not a GitHub connector" - # 2. Get the GitHub PAT from the connector config + # 2. Get the GitHub PAT and selected repositories from the connector config github_pat = connector.config.get("GITHUB_PAT") + repo_full_names_to_index = connector.config.get("repo_full_names") + if not github_pat: return 0, "GitHub Personal Access Token (PAT) not found in connector config" + + if not repo_full_names_to_index or not isinstance(repo_full_names_to_index, list): + return 0, "'repo_full_names' not found or is not a list in connector config" # 3. Initialize GitHub connector client try: @@ -650,13 +655,10 @@ async def index_github_repos( except ValueError as e: return 0, f"Failed to initialize GitHub client: {str(e)}" - # 4. Get list of accessible repositories - repositories = github_client.get_user_repositories() - if not repositories: - logger.info("No accessible GitHub repositories found for the provided token.") - return 0, "No accessible GitHub repositories found." - - logger.info(f"Found {len(repositories)} repositories to potentially index.") + # 4. Validate selected repositories + # For simplicity, we'll proceed with the list provided. + # If a repo is inaccessible, get_repository_files will likely fail gracefully later. + logger.info(f"Starting indexing for {len(repo_full_names_to_index)} selected repositories.") # 5. Get existing documents for this search space and connector type to prevent duplicates existing_docs_result = await session.execute( @@ -671,11 +673,10 @@ async def index_github_repos( existing_docs_lookup = {doc.document_metadata.get("full_path"): doc for doc in existing_docs if doc.document_metadata.get("full_path")} logger.info(f"Found {len(existing_docs_lookup)} existing GitHub documents in database for search space {search_space_id}") - # 6. Iterate through repositories and index files - for repo_info in repositories: - repo_full_name = repo_info.get("full_name") - if not repo_full_name: - logger.warning(f"Skipping repository with missing full_name: {repo_info.get('name')}") + # 6. Iterate through selected repositories and index files + for repo_full_name in repo_full_names_to_index: + if not repo_full_name or not isinstance(repo_full_name, str): + logger.warning(f"Skipping invalid repository entry: {repo_full_name}") continue logger.info(f"Processing repository: {repo_full_name}") diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 8bfec63..24fe626 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -206,7 +206,7 @@ export default function ConnectorsPage() { + + + + + + {/* TODO: Dynamic icon */} + Edit {getConnectorTypeDisplay(connector.connector_type)} Connector + + Modify connector name and configuration. + + +
+ {/* Pass hook's handleSaveChanges */} + + + {/* Pass form control from hook */} + + +
+ +

Configuration

+ + {/* == GitHub == */} + {connector.connector_type === 'GITHUB_CONNECTOR' && ( + + )} + + {/* == Slack == */} + {connector.connector_type === 'SLACK_CONNECTOR' && ( + + )} + {/* == Notion == */} + {connector.connector_type === 'NOTION_CONNECTOR' && ( + + )} + {/* == Serper == */} + {connector.connector_type === 'SERPER_API' && ( + + )} + {/* == Tavily == */} + {connector.connector_type === 'TAVILY_API' && ( + + )} + + {/* == Linear == */} + {connector.connector_type === 'LINEAR_CONNECTOR' && ( + + )} + + + + + +
+ + + + + ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index e841639..ad6ceb7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -70,7 +70,7 @@ export default function EditConnectorPage() { const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); - + console.log("connector", connector); // Initialize the form const form = useForm({ resolver: zodResolver(apiConnectorFormSchema), diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 45534d6..fc7a602 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -7,7 +7,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useForm } from "react-hook-form"; import * as z from "zod"; import { toast } from "sonner"; -import { ArrowLeft, Check, Info, Loader2, Github } from "lucide-react"; +import { ArrowLeft, Check, Info, Loader2, Github, CircleAlert, ListChecks } from "lucide-react"; // Assuming useSearchSourceConnectors hook exists and works similarly import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; @@ -42,9 +42,10 @@ import { AccordionTrigger, } from "@/components/ui/accordion"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Checkbox } from "@/components/ui/checkbox"; -// Define the form schema with Zod for GitHub -const githubConnectorFormSchema = z.object({ +// Define the form schema with Zod for GitHub PAT entry step +const githubPatFormSchema = z.object({ name: z.string().min(3, { message: "Connector name must be at least 3 characters.", }), @@ -58,61 +59,144 @@ const githubConnectorFormSchema = z.object({ }); // Define the type for the form values -type GithubConnectorFormValues = z.infer; +type GithubPatFormValues = z.infer; + +// Type for fetched GitHub repositories +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} export default function GithubConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); // Assuming this hook exists + const [step, setStep] = useState<'enter_pat' | 'select_repos'>('enter_pat'); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + const [isCreatingConnector, setIsCreatingConnector] = useState(false); + const [repositories, setRepositories] = useState([]); + const [selectedRepos, setSelectedRepos] = useState([]); + const [connectorName, setConnectorName] = useState("GitHub Connector"); + const [validatedPat, setValidatedPat] = useState(""); // Store the validated PAT - // Initialize the form - const form = useForm({ - resolver: zodResolver(githubConnectorFormSchema), + const { createConnector } = useSearchSourceConnectors(); + + // Initialize the form for PAT entry + const form = useForm({ + resolver: zodResolver(githubPatFormSchema), defaultValues: { - name: "GitHub Connector", + name: connectorName, github_pat: "", }, }); - // Handle form submission - const onSubmit = async (values: GithubConnectorFormValues) => { - setIsSubmitting(true); + // Function to fetch repositories using the new backend endpoint + const fetchRepositories = async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setConnectorName(values.name); // Store the name + setValidatedPat(values.github_pat); // Store the PAT temporarily + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) { + throw new Error('No authentication token found'); + } + + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ github_pat: values.github_pat }) + } + ); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); + } + + const data: GithubRepo[] = await response.json(); + setRepositories(data); + setStep('select_repos'); // Move to the next step + toast.success(`Found ${data.length} repositories.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + const errorMessage = error instanceof Error ? error.message : "Failed to fetch repositories. Please check the PAT and try again."; + toast.error(errorMessage); + } finally { + setIsFetchingRepos(false); + } + }; + + // Handle final connector creation + const handleCreateConnector = async () => { + if (selectedRepos.length === 0) { + toast.warning("Please select at least one repository to index."); + return; + } + + setIsCreatingConnector(true); try { await createConnector({ - name: values.name, + name: connectorName, // Use the stored name connector_type: "GITHUB_CONNECTOR", config: { - GITHUB_PAT: values.github_pat, + GITHUB_PAT: validatedPat, // Use the stored validated PAT + repo_full_names: selectedRepos, // Add the selected repo names }, - is_indexable: true, // GitHub connector is indexable - last_indexed_at: null, // New connector hasn't been indexed + is_indexable: true, + last_indexed_at: null, }); toast.success("GitHub connector created successfully!"); - - // Navigate back to connectors management page (or the add page) router.push(`/dashboard/${searchSpaceId}/connectors`); - } catch (error) { // Added type check for error + } catch (error) { console.error("Error creating GitHub connector:", error); - // Display specific backend error message if available - const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector. Please check the PAT and permissions."; + const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector."; toast.error(errorMessage); } finally { - setIsSubmitting(false); + setIsCreatingConnector(false); } }; + // Handle checkbox changes + const handleRepoSelection = (repoFullName: string, checked: boolean) => { + setSelectedRepos(prev => + checked + ? [...prev, repoFullName] + : prev.filter(name => name !== repoFullName) + ); + }; + return (
- Connect GitHub Account + + {step === 'enter_pat' ? : } + {step === 'enter_pat' ? "Connect GitHub Account" : "Select Repositories to Index"} + - Integrate with GitHub using a Personal Access Token (PAT) to search and retrieve information from accessible repositories. This connector can index your code and documentation. + {step === 'enter_pat' + ? "Provide a name and GitHub Personal Access Token (PAT) to fetch accessible repositories." + : `Select which repositories you want SurfSense to index for search. Found ${repositories.length} repositories accessible via your PAT.` + } - - - - GitHub Personal Access Token (PAT) Required - - You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to use this connector. You can create one from your - - GitHub Developer Settings - . - - -
- - ( - - Connector Name - - - - - A friendly name to identify this GitHub connection. - - - - )} - /> + + {step === 'enter_pat' && ( + + + + GitHub Personal Access Token (PAT) Required + + You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to fetch repositories. You can create one from your{' '} + + GitHub Developer Settings + . The PAT will be used to fetch repositories and then stored securely to enable indexing. + + - ( - - GitHub Personal Access Token (PAT) - - - - - Your GitHub PAT will be encrypted and stored securely. Ensure it has the necessary 'repo' scopes. - - - - )} - /> - -
- -
- - - + /> + + ( + + GitHub Personal Access Token (PAT) + + + + + Enter your GitHub PAT here to fetch your repositories. It will be stored encrypted later. + + + + )} + /> + +
+ +
+ +
+ )} + + {step === 'select_repos' && ( + + {repositories.length === 0 ? ( + + + No Repositories Found + + No repositories were found or accessible with the provided PAT. Please check the token and its permissions, then go back and try again. + + + ) : ( +
+ Repositories ({selectedRepos.length} selected) +
+ {repositories.map((repo) => ( +
+ handleRepoSelection(repo.full_name, !!checked)} + /> + +
+ ))} +
+ + Select the repositories you wish to index. Only checked repositories will be processed. + + +
+ + +
+
+ )} + + )} + +

What you get with GitHub integration:

    -
  • Search through code and documentation in your repositories
    • +
  • Search through code and documentation in your selected repositories
  • Access READMEs, Markdown files, and common code files
  • Connect your project knowledge directly to your search space
    • -
  • Index your repositories for enhanced search capabilities
    • +
  • Index your selected repositories for enhanced search capabilities
 @@ -237,27 +398,20 @@ export default function GithubConnectorPage() {

How it works

- The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. It fetches information about repositories accessible to the token and indexes relevant files (code, markdown, text). + The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. First, it fetches a list of repositories accessible to the token. You then select which repositories you want to index. The connector indexes relevant files (code, markdown, text) from only the selected repositories.

  • The connector indexes files based on common code and documentation extensions.
  • Large files (over 1MB) are skipped during indexing.
    • +
  • Only selected repositories are indexed.
  • Indexing runs periodically (check connector settings for frequency) to keep content up-to-date.
- Step 1: Create a GitHub PAT - - - - Token Security - - Treat your PAT like a password. Store it securely and consider using fine-grained tokens if possible. - - - + Step 1: Generate GitHub PAT +

Generating a Token:

@@ -280,9 +434,13 @@ export default function GithubConnectorPage() { Step 2: Connect in SurfSense
    -
  1. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field on the "Connect GitHub" tab.
    2. -
  2. Optionally, give the connector a custom name.
    3. -
  3. Click the Connect GitHub button.
    4. +
  4. Navigate to the "Connect GitHub" tab.
    5. +
  5. Enter a name for your connector.
    6. +
  6. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field.
    7. +
  7. Click Fetch Repositories.
    8. +
  8. If the PAT is valid, you'll see a list of your accessible repositories.
    9. +
  9. Select the repositories you want SurfSense to index using the checkboxes.
    10. +
  10. Click the Create Connector button.
  11. If the connection is successful, you will be redirected and can start indexing from the Connectors page.
 diff --git a/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx new file mode 100644 index 0000000..dc1320f --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { Skeleton } from "@/components/ui/skeleton"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; + +export function EditConnectorLoadingSkeleton() { + return ( +
+ + + + + + + + + + + +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditConnectorNameForm.tsx b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx new file mode 100644 index 0000000..3f18820 --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditConnectorNameFormProps { + control: Control; // Use Control if type is available +} + +export function EditConnectorNameForm({ control }: EditConnectorNameFormProps) { + return ( + ( + + Connector Name + + + + )} + /> + ); +} diff --git a/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx new file mode 100644 index 0000000..17f83f7 --- /dev/null +++ b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx @@ -0,0 +1,160 @@ +import React from 'react'; +import { UseFormReturn } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Edit, KeyRound, Loader2, CircleAlert } from 'lucide-react'; + +// Types needed from parent +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} +type GithubPatFormValues = { github_pat: string; }; +type EditMode = 'viewing' | 'editing_repos'; + +interface EditGitHubConnectorConfigProps { + // State from parent + editMode: EditMode; + originalPat: string; + currentSelectedRepos: string[]; + fetchedRepos: GithubRepo[] | null; + newSelectedRepos: string[]; + isFetchingRepos: boolean; + // Forms from parent + patForm: UseFormReturn; + // Handlers from parent + setEditMode: (mode: EditMode) => void; + handleFetchRepositories: (values: GithubPatFormValues) => Promise; + handleRepoSelectionChange: (repoFullName: string, checked: boolean) => void; + setNewSelectedRepos: React.Dispatch>; + setFetchedRepos: React.Dispatch>; +} + +export function EditGitHubConnectorConfig({ + editMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + newSelectedRepos, + isFetchingRepos, + patForm, + setEditMode, + handleFetchRepositories, + handleRepoSelectionChange, + setNewSelectedRepos, + setFetchedRepos +}: EditGitHubConnectorConfigProps) { + + return ( +
+

Repository Selection & Access

+ + {/* Viewing Mode */} + {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected)

+ )} + + To change repo selections or update the PAT, click above. +
+ )} + + {/* Editing Mode */} + {editMode === 'editing_repos' && ( +
+ {/* PAT Input */} +
+ ( + + GitHub PAT + + Enter PAT to fetch/update repos or if you need to update the stored token. + + + )} + /> + +
+ + {/* Repo List */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + + + No Repositories Found + Check PAT & permissions. + + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} + /> + +
+ ))} +
+
+ ) + )} + +
+ )} +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx new file mode 100644 index 0000000..c0c8032 --- /dev/null +++ b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { KeyRound } from 'lucide-react'; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditSimpleTokenFormProps { + control: Control; + fieldName: string; // e.g., "SLACK_BOT_TOKEN" + fieldLabel: string; // e.g., "Slack Bot Token" + fieldDescription: string; + placeholder?: string; +} + +export function EditSimpleTokenForm({ + control, + fieldName, + fieldLabel, + fieldDescription, + placeholder +}: EditSimpleTokenFormProps) { + return ( + ( + + {fieldLabel} + + {fieldDescription} + + + )} + /> + ); +} diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts new file mode 100644 index 0000000..364f235 --- /dev/null +++ b/surfsense_web/components/editConnector/types.ts @@ -0,0 +1,34 @@ +import * as z from "zod"; + +// Types +export interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} + +export type EditMode = 'viewing' | 'editing_repos'; + +// Schemas +export const githubPatSchema = z.object({ + github_pat: z.string() + .min(20, { message: "GitHub Personal Access Token seems too short." }) + .refine(pat => pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { + message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", + }), +}); +export type GithubPatFormValues = z.infer; + +export const editConnectorSchema = z.object({ + name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), + SLACK_BOT_TOKEN: z.string().optional(), + NOTION_INTEGRATION_TOKEN: z.string().optional(), + SERPER_API_KEY: z.string().optional(), + TAVILY_API_KEY: z.string().optional(), + LINEAR_API_KEY: z.string().optional(), +}); +export type EditConnectorFormValues = z.infer; diff --git a/surfsense_web/hooks/useConnectorEditPage.ts b/surfsense_web/hooks/useConnectorEditPage.ts new file mode 100644 index 0000000..d767202 --- /dev/null +++ b/surfsense_web/hooks/useConnectorEditPage.ts @@ -0,0 +1,240 @@ +import { useState, useEffect, useCallback } from 'react'; +import { useRouter } from 'next/navigation'; +import { useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { toast } from 'sonner'; +import { useSearchSourceConnectors, SearchSourceConnector } from '@/hooks/useSearchSourceConnectors'; +import { + GithubRepo, + EditMode, + githubPatSchema, + editConnectorSchema, + GithubPatFormValues, + EditConnectorFormValues +} from '@/components/editConnector/types'; + +export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { + const router = useRouter(); + const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + + // State managed by the hook + const [connector, setConnector] = useState(null); + const [originalConfig, setOriginalConfig] = useState | null>(null); + const [isSaving, setIsSaving] = useState(false); + const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); + const [originalPat, setOriginalPat] = useState(""); + const [editMode, setEditMode] = useState('viewing'); + const [fetchedRepos, setFetchedRepos] = useState(null); + const [newSelectedRepos, setNewSelectedRepos] = useState([]); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + + // Forms managed by the hook + const patForm = useForm({ + resolver: zodResolver(githubPatSchema), + defaultValues: { github_pat: "" }, + }); + const editForm = useForm({ + resolver: zodResolver(editConnectorSchema), + defaultValues: { + name: "", + SLACK_BOT_TOKEN: "", + NOTION_INTEGRATION_TOKEN: "", + SERPER_API_KEY: "", + TAVILY_API_KEY: "", + LINEAR_API_KEY: "" + }, + }); + + // Effect to load initial data + useEffect(() => { + if (!connectorsLoading && connectors.length > 0 && !connector) { + const currentConnector = connectors.find(c => c.id === connectorId); + if (currentConnector) { + setConnector(currentConnector); + const config = currentConnector.config || {}; + setOriginalConfig(config); + editForm.reset({ + name: currentConnector.name, + SLACK_BOT_TOKEN: config.SLACK_BOT_TOKEN || "", + NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", + SERPER_API_KEY: config.SERPER_API_KEY || "", + TAVILY_API_KEY: config.TAVILY_API_KEY || "", + LINEAR_API_KEY: config.LINEAR_API_KEY || "" + }); + if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { + const savedRepos = config.repo_full_names || []; + const savedPat = config.GITHUB_PAT || ""; + setCurrentSelectedRepos(savedRepos); + setNewSelectedRepos(savedRepos); + setOriginalPat(savedPat); + patForm.reset({ github_pat: savedPat }); + setEditMode('viewing'); + } + } else { + toast.error("Connector not found."); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } + } + }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); + + // Handlers managed by the hook + const handleFetchRepositories = useCallback(async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setFetchedRepos(null); + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) throw new Error('No auth token'); + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ github_pat: values.github_pat }) } + ); + if (!response.ok) { const err = await response.json(); throw new Error(err.detail || 'Fetch failed'); } + const data: GithubRepo[] = await response.json(); + setFetchedRepos(data); + setNewSelectedRepos(currentSelectedRepos); + toast.success(`Found ${data.length} repos.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); + } finally { setIsFetchingRepos(false); } + }, [currentSelectedRepos]); // Added dependency + + const handleRepoSelectionChange = useCallback((repoFullName: string, checked: boolean) => { + setNewSelectedRepos(prev => checked ? [...prev, repoFullName] : prev.filter(name => name !== repoFullName)); + }, []); + + const handleSaveChanges = useCallback(async (formData: EditConnectorFormValues) => { + if (!connector || !originalConfig) return; + setIsSaving(true); + const updatePayload: Partial = {}; + let configChanged = false; + let newConfig: Record | null = null; + + if (formData.name !== connector.name) { + updatePayload.name = formData.name; + } + + switch (connector.connector_type) { + case 'GITHUB_CONNECTOR': + const currentPatInForm = patForm.getValues('github_pat'); + const patChanged = currentPatInForm !== originalPat; + const initialRepoSet = new Set(currentSelectedRepos); + const newRepoSet = new Set(newSelectedRepos); + const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); + if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { + if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { + toast.error("Invalid GitHub PAT format. Cannot save."); setIsSaving(false); return; + } + newConfig = { GITHUB_PAT: currentPatInForm, repo_full_names: newSelectedRepos }; + if (reposChanged && newSelectedRepos.length === 0) { toast.warning("Warning: No repositories selected."); } + } + break; + case 'SLACK_CONNECTOR': + if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { + if (!formData.SLACK_BOT_TOKEN) { toast.error("Slack Token empty."); setIsSaving(false); return; } + newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; + } + break; + case 'NOTION_CONNECTOR': + if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { + if (!formData.NOTION_INTEGRATION_TOKEN) { toast.error("Notion Token empty."); setIsSaving(false); return; } + newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; + } + break; + case 'SERPER_API': + if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { + if (!formData.SERPER_API_KEY) { toast.error("Serper Key empty."); setIsSaving(false); return; } + newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; + } + break; + case 'TAVILY_API': + if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { + if (!formData.TAVILY_API_KEY) { toast.error("Tavily Key empty."); setIsSaving(false); return; } + newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; + } + break; + + case 'LINEAR_CONNECTOR': + if (formData.LINEAR_API_KEY !== originalConfig.LINEAR_API_KEY) { + if (!formData.LINEAR_API_KEY) { + toast.error("Linear API Key cannot be empty."); + setIsSaving(false); + return; + } + newConfig = { LINEAR_API_KEY: formData.LINEAR_API_KEY }; + } + break; + } + + if (newConfig !== null) { + updatePayload.config = newConfig; + configChanged = true; + } + + if (Object.keys(updatePayload).length === 0) { + toast.info("No changes detected."); + setIsSaving(false); + if (connector.connector_type === 'GITHUB_CONNECTOR') { setEditMode('viewing'); patForm.reset({ github_pat: originalPat }); } + return; + } + + try { + await updateConnector(connectorId, updatePayload); + toast.success("Connector updated!"); + const newlySavedConfig = updatePayload.config || originalConfig; + setOriginalConfig(newlySavedConfig); + if (updatePayload.name) { + setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); + } + if (configChanged) { + if (connector.connector_type === 'GITHUB_CONNECTOR') { + const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; + setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); + setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); + setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); + patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); + } else if(connector.connector_type === 'SLACK_CONNECTOR') { + editForm.setValue('SLACK_BOT_TOKEN', newlySavedConfig.SLACK_BOT_TOKEN || ""); + } else if(connector.connector_type === 'NOTION_CONNECTOR') { + editForm.setValue('NOTION_INTEGRATION_TOKEN', newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""); + } else if(connector.connector_type === 'SERPER_API') { + editForm.setValue('SERPER_API_KEY', newlySavedConfig.SERPER_API_KEY || ""); + } else if(connector.connector_type === 'TAVILY_API') { + editForm.setValue('TAVILY_API_KEY', newlySavedConfig.TAVILY_API_KEY || ""); + } else if(connector.connector_type === 'LINEAR_CONNECTOR') { + editForm.setValue('LINEAR_API_KEY', newlySavedConfig.LINEAR_API_KEY || ""); + } + } + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + setFetchedRepos(null); + } + // Resetting simple form values is handled by useEffect if connector state updates + } catch (error) { + console.error("Error updating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to update connector."); + } finally { setIsSaving(false); } + }, [connector, originalConfig, updateConnector, connectorId, patForm, originalPat, currentSelectedRepos, newSelectedRepos, editMode, fetchedRepos, editForm]); // Added editForm to dependencies + + // Return values needed by the component + return { + connectorsLoading, + connector, + isSaving, + editForm, + patForm, + handleSaveChanges, + // GitHub specific props + editMode, + setEditMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + setFetchedRepos, + newSelectedRepos, + setNewSelectedRepos, + isFetchingRepos, + handleFetchRepositories, + handleRepoSelectionChange, + }; +} diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts new file mode 100644 index 0000000..5efc593 --- /dev/null +++ b/surfsense_web/lib/connectors/utils.ts @@ -0,0 +1,12 @@ +// Helper function to get connector type display name +export const getConnectorTypeDisplay = (type: string): string => { + const typeMap: Record = { + "SERPER_API": "Serper API", + "TAVILY_API": "Tavily API", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + "LINEAR_CONNECTOR": "Linear", + }; + return typeMap[type] || type; +};