From ae8c74a5aa1637243cad798d5f2db78af6f2d626 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 19:59:38 -0700 Subject: [PATCH 1/6] select repos when adding gh connector --- .../routes/search_source_connectors_routes.py | 33 +- .../app/schemas/search_source_connector.py | 10 +- .../app/tasks/connectors_indexing_tasks.py | 29 +- .../connectors/add/github-connector/page.tsx | 396 ++++++++++++------ 4 files changed, 330 insertions(+), 138 deletions(-) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 482a825..5bfe0a9 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -9,7 +9,7 @@ POST /search-source-connectors/{connector_id}/index - Index content from a conne Note: Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR, NOTION_CONNECTOR). """ -from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks +from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks, Body from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError @@ -18,8 +18,9 @@ from app.db import get_async_session, User, SearchSourceConnector, SearchSourceC from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead from app.users import current_active_user from app.utils.check_ownership import check_ownership -from pydantic import ValidationError +from pydantic import ValidationError, BaseModel, Field from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos +from app.connectors.github_connector import GitHubConnector from datetime import datetime, timezone import logging @@ -28,6 +29,34 @@ logger = logging.getLogger(__name__) router = APIRouter() +# --- New Schema for GitHub PAT --- +class GitHubPATRequest(BaseModel): + github_pat: str = Field(..., description="GitHub Personal Access Token") + +# --- New Endpoint to list GitHub Repositories --- +@router.post("/github/repositories/", response_model=List[Dict[str, Any]]) +async def list_github_repositories( + pat_request: GitHubPATRequest, + user: User = Depends(current_active_user) # Ensure the user is logged in +): + """ + Fetches a list of repositories accessible by the provided GitHub PAT. + The PAT is used for this request only and is not stored. + """ + try: + # Initialize GitHubConnector with the provided PAT + github_client = GitHubConnector(token=pat_request.github_pat) + # Fetch repositories + repositories = github_client.get_user_repositories() + return repositories + except ValueError as e: + # Handle invalid token error specifically + logger.error(f"GitHub PAT validation failed for user {user.id}: {str(e)}") + raise HTTPException(status_code=400, detail=f"Invalid GitHub PAT: {str(e)}") + except Exception as e: + logger.error(f"Failed to fetch GitHub repositories for user {user.id}: {str(e)}") + raise HTTPException(status_code=500, detail="Failed to fetch GitHub repositories.") + @router.post("/search-source-connectors/", response_model=SearchSourceConnectorRead) async def create_search_source_connector( connector: SearchSourceConnectorCreate, diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 41e1086..1005a63 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -4,7 +4,6 @@ from typing import Dict, Any from pydantic import BaseModel, field_validator from .base import IDModel, TimestampModel from app.db import SearchSourceConnectorType -from fastapi import HTTPException class SearchSourceConnectorBase(BaseModel): name: str @@ -59,14 +58,19 @@ class SearchSourceConnectorBase(BaseModel): raise ValueError("NOTION_INTEGRATION_TOKEN cannot be empty") elif connector_type == SearchSourceConnectorType.GITHUB_CONNECTOR: - # For GITHUB_CONNECTOR, only allow GITHUB_PAT - allowed_keys = ["GITHUB_PAT"] + # For GITHUB_CONNECTOR, only allow GITHUB_PAT and repo_full_names + allowed_keys = ["GITHUB_PAT", "repo_full_names"] if set(config.keys()) != set(allowed_keys): raise ValueError(f"For GITHUB_CONNECTOR connector type, config must only contain these keys: {allowed_keys}") # Ensure the token is not empty if not config.get("GITHUB_PAT"): raise ValueError("GITHUB_PAT cannot be empty") + + # Ensure the repo_full_names is present and is a non-empty list + repo_full_names = config.get("repo_full_names") + if not isinstance(repo_full_names, list) or not repo_full_names: + raise ValueError("repo_full_names must be a non-empty list of strings") return config diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 670fa26..31a7d4d 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -1,4 +1,4 @@ -from typing import Optional, List, Dict, Any, Tuple +from typing import Optional, Tuple from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.future import select @@ -626,10 +626,15 @@ async def index_github_repos( if not connector: return 0, f"Connector with ID {connector_id} not found or is not a GitHub connector" - # 2. Get the GitHub PAT from the connector config + # 2. Get the GitHub PAT and selected repositories from the connector config github_pat = connector.config.get("GITHUB_PAT") + repo_full_names_to_index = connector.config.get("repo_full_names") + if not github_pat: return 0, "GitHub Personal Access Token (PAT) not found in connector config" + + if not repo_full_names_to_index or not isinstance(repo_full_names_to_index, list): + return 0, "'repo_full_names' not found or is not a list in connector config" # 3. Initialize GitHub connector client try: @@ -637,13 +642,10 @@ async def index_github_repos( except ValueError as e: return 0, f"Failed to initialize GitHub client: {str(e)}" - # 4. Get list of accessible repositories - repositories = github_client.get_user_repositories() - if not repositories: - logger.info("No accessible GitHub repositories found for the provided token.") - return 0, "No accessible GitHub repositories found." - - logger.info(f"Found {len(repositories)} repositories to potentially index.") + # 4. Validate selected repositories + # For simplicity, we'll proceed with the list provided. + # If a repo is inaccessible, get_repository_files will likely fail gracefully later. + logger.info(f"Starting indexing for {len(repo_full_names_to_index)} selected repositories.") # 5. Get existing documents for this search space and connector type to prevent duplicates existing_docs_result = await session.execute( @@ -658,11 +660,10 @@ async def index_github_repos( existing_docs_lookup = {doc.document_metadata.get("full_path"): doc for doc in existing_docs if doc.document_metadata.get("full_path")} logger.info(f"Found {len(existing_docs_lookup)} existing GitHub documents in database for search space {search_space_id}") - # 6. Iterate through repositories and index files - for repo_info in repositories: - repo_full_name = repo_info.get("full_name") - if not repo_full_name: - logger.warning(f"Skipping repository with missing full_name: {repo_info.get('name')}") + # 6. Iterate through selected repositories and index files + for repo_full_name in repo_full_names_to_index: + if not repo_full_name or not isinstance(repo_full_name, str): + logger.warning(f"Skipping invalid repository entry: {repo_full_name}") continue logger.info(f"Processing repository: {repo_full_name}") diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx index 45534d6..fc7a602 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/github-connector/page.tsx @@ -7,7 +7,7 @@ import { zodResolver } from "@hookform/resolvers/zod"; import { useForm } from "react-hook-form"; import * as z from "zod"; import { toast } from "sonner"; -import { ArrowLeft, Check, Info, Loader2, Github } from "lucide-react"; +import { ArrowLeft, Check, Info, Loader2, Github, CircleAlert, ListChecks } from "lucide-react"; // Assuming useSearchSourceConnectors hook exists and works similarly import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; @@ -42,9 +42,10 @@ import { AccordionTrigger, } from "@/components/ui/accordion"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { Checkbox } from "@/components/ui/checkbox"; -// Define the form schema with Zod for GitHub -const githubConnectorFormSchema = z.object({ +// Define the form schema with Zod for GitHub PAT entry step +const githubPatFormSchema = z.object({ name: z.string().min(3, { message: "Connector name must be at least 3 characters.", }), @@ -58,61 +59,144 @@ const githubConnectorFormSchema = z.object({ }); // Define the type for the form values -type GithubConnectorFormValues = z.infer; +type GithubPatFormValues = z.infer; + +// Type for fetched GitHub repositories +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} export default function GithubConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const [isSubmitting, setIsSubmitting] = useState(false); - const { createConnector } = useSearchSourceConnectors(); // Assuming this hook exists + const [step, setStep] = useState<'enter_pat' | 'select_repos'>('enter_pat'); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + const [isCreatingConnector, setIsCreatingConnector] = useState(false); + const [repositories, setRepositories] = useState([]); + const [selectedRepos, setSelectedRepos] = useState([]); + const [connectorName, setConnectorName] = useState("GitHub Connector"); + const [validatedPat, setValidatedPat] = useState(""); // Store the validated PAT - // Initialize the form - const form = useForm({ - resolver: zodResolver(githubConnectorFormSchema), + const { createConnector } = useSearchSourceConnectors(); + + // Initialize the form for PAT entry + const form = useForm({ + resolver: zodResolver(githubPatFormSchema), defaultValues: { - name: "GitHub Connector", + name: connectorName, github_pat: "", }, }); - // Handle form submission - const onSubmit = async (values: GithubConnectorFormValues) => { - setIsSubmitting(true); + // Function to fetch repositories using the new backend endpoint + const fetchRepositories = async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setConnectorName(values.name); // Store the name + setValidatedPat(values.github_pat); // Store the PAT temporarily + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) { + throw new Error('No authentication token found'); + } + + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${token}` + }, + body: JSON.stringify({ github_pat: values.github_pat }) + } + ); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); + } + + const data: GithubRepo[] = await response.json(); + setRepositories(data); + setStep('select_repos'); // Move to the next step + toast.success(`Found ${data.length} repositories.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + const errorMessage = error instanceof Error ? error.message : "Failed to fetch repositories. Please check the PAT and try again."; + toast.error(errorMessage); + } finally { + setIsFetchingRepos(false); + } + }; + + // Handle final connector creation + const handleCreateConnector = async () => { + if (selectedRepos.length === 0) { + toast.warning("Please select at least one repository to index."); + return; + } + + setIsCreatingConnector(true); try { await createConnector({ - name: values.name, + name: connectorName, // Use the stored name connector_type: "GITHUB_CONNECTOR", config: { - GITHUB_PAT: values.github_pat, + GITHUB_PAT: validatedPat, // Use the stored validated PAT + repo_full_names: selectedRepos, // Add the selected repo names }, - is_indexable: true, // GitHub connector is indexable - last_indexed_at: null, // New connector hasn't been indexed + is_indexable: true, + last_indexed_at: null, }); toast.success("GitHub connector created successfully!"); - - // Navigate back to connectors management page (or the add page) router.push(`/dashboard/${searchSpaceId}/connectors`); - } catch (error) { // Added type check for error + } catch (error) { console.error("Error creating GitHub connector:", error); - // Display specific backend error message if available - const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector. Please check the PAT and permissions."; + const errorMessage = error instanceof Error ? error.message : "Failed to create GitHub connector."; toast.error(errorMessage); } finally { - setIsSubmitting(false); + setIsCreatingConnector(false); } }; + // Handle checkbox changes + const handleRepoSelection = (repoFullName: string, checked: boolean) => { + setSelectedRepos(prev => + checked + ? [...prev, repoFullName] + : prev.filter(name => name !== repoFullName) + ); + }; + return (
- Connect GitHub Account + + {step === 'enter_pat' ? : } + {step === 'enter_pat' ? "Connect GitHub Account" : "Select Repositories to Index"} + - Integrate with GitHub using a Personal Access Token (PAT) to search and retrieve information from accessible repositories. This connector can index your code and documentation. + {step === 'enter_pat' + ? "Provide a name and GitHub Personal Access Token (PAT) to fetch accessible repositories." + : `Select which repositories you want SurfSense to index for search. Found ${repositories.length} repositories accessible via your PAT.` + } - - - - GitHub Personal Access Token (PAT) Required - - You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to use this connector. You can create one from your - - GitHub Developer Settings - . - - -
- - ( - - Connector Name - - - - - A friendly name to identify this GitHub connection. - - - - )} - /> + + {step === 'enter_pat' && ( + + + + GitHub Personal Access Token (PAT) Required + + You'll need a GitHub PAT with the appropriate scopes (e.g., 'repo') to fetch repositories. You can create one from your{' '} + + GitHub Developer Settings + . The PAT will be used to fetch repositories and then stored securely to enable indexing. + + - ( - - GitHub Personal Access Token (PAT) - - - - - Your GitHub PAT will be encrypted and stored securely. Ensure it has the necessary 'repo' scopes. - - - - )} - /> - -
- -
- - - + /> + + ( + + GitHub Personal Access Token (PAT) + + + + + Enter your GitHub PAT here to fetch your repositories. It will be stored encrypted later. + + + + )} + /> + +
+ +
+ +
+ )} + + {step === 'select_repos' && ( + + {repositories.length === 0 ? ( + + + No Repositories Found + + No repositories were found or accessible with the provided PAT. Please check the token and its permissions, then go back and try again. + + + ) : ( +
+ Repositories ({selectedRepos.length} selected) +
+ {repositories.map((repo) => ( +
+ handleRepoSelection(repo.full_name, !!checked)} + /> + +
+ ))} +
+ + Select the repositories you wish to index. Only checked repositories will be processed. + + +
+ + +
+
+ )} + + )} + +

What you get with GitHub integration:

    -
  • Search through code and documentation in your repositories
    • +
  • Search through code and documentation in your selected repositories
  • Access READMEs, Markdown files, and common code files
  • Connect your project knowledge directly to your search space
    • -
  • Index your repositories for enhanced search capabilities
    • +
  • Index your selected repositories for enhanced search capabilities
 @@ -237,27 +398,20 @@ export default function GithubConnectorPage() {

How it works

- The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. It fetches information about repositories accessible to the token and indexes relevant files (code, markdown, text). + The GitHub connector uses a Personal Access Token (PAT) to authenticate with the GitHub API. First, it fetches a list of repositories accessible to the token. You then select which repositories you want to index. The connector indexes relevant files (code, markdown, text) from only the selected repositories.

  • The connector indexes files based on common code and documentation extensions.
  • Large files (over 1MB) are skipped during indexing.
    • +
  • Only selected repositories are indexed.
  • Indexing runs periodically (check connector settings for frequency) to keep content up-to-date.
- Step 1: Create a GitHub PAT - - - - Token Security - - Treat your PAT like a password. Store it securely and consider using fine-grained tokens if possible. - - - + Step 1: Generate GitHub PAT +

Generating a Token:

@@ -280,9 +434,13 @@ export default function GithubConnectorPage() { Step 2: Connect in SurfSense
    -
  1. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field on the "Connect GitHub" tab.
    2. -
  2. Optionally, give the connector a custom name.
    3. -
  3. Click the Connect GitHub button.
    4. +
  4. Navigate to the "Connect GitHub" tab.
    5. +
  5. Enter a name for your connector.
    6. +
  6. Paste the copied GitHub PAT into the "GitHub Personal Access Token (PAT)" field.
    7. +
  7. Click Fetch Repositories.
    8. +
  8. If the PAT is valid, you'll see a list of your accessible repositories.
    9. +
  9. Select the repositories you want SurfSense to index using the checkboxes.
    10. +
  10. Click the Create Connector button.
  11. If the connection is successful, you will be redirected and can start indexing from the Connectors page.
 From 5176569e30e2e717e4305e3fd198510116985c43 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 20:29:50 -0700 Subject: [PATCH 2/6] edit repos for gh connector --- .../routes/search_source_connectors_routes.py | 78 +++- .../app/schemas/search_source_connector.py | 12 +- .../connectors/(manage)/page.tsx | 2 +- .../connectors/[connector_id]/edit/page.tsx | 442 ++++++++++++++++++ .../connectors/[connector_id]/page.tsx | 2 +- 5 files changed, 506 insertions(+), 30 deletions(-) create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 5bfe0a9..f843029 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -15,7 +15,7 @@ from sqlalchemy.future import select from sqlalchemy.exc import IntegrityError from typing import List, Dict, Any from app.db import get_async_session, User, SearchSourceConnector, SearchSourceConnectorType, SearchSpace, async_session_maker -from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead +from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase from app.users import current_active_user from app.utils.check_ownership import check_ownership from pydantic import ValidationError, BaseModel, Field @@ -159,54 +159,84 @@ async def update_search_source_connector( ): """ Update a search source connector. - - Each user can have only one connector of each type (SERPER_API, TAVILY_API, SLACK_CONNECTOR). - The config must contain the appropriate keys for the connector type. + Handles partial updates, including merging changes into the 'config' field. """ - try: - db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + db_connector = await check_ownership(session, SearchSourceConnector, connector_id, user) + + # Convert the sparse update data (only fields present in request) to a dict + update_data = connector_update.model_dump(exclude_unset=True) + + # Special handling for 'config' field + if "config" in update_data: + incoming_config = update_data["config"] # Config data from the request + existing_config = db_connector.config if db_connector.config else {} # Current config from DB - # If connector type is being changed, check if one of that type already exists - if connector_update.connector_type != db_connector.connector_type: + # Merge incoming config into existing config + # This preserves existing keys (like GITHUB_PAT) if they are not in the incoming data + merged_config = existing_config.copy() + merged_config.update(incoming_config) + + # -- Validation after merging -- + # Validate the *merged* config based on the connector type + # We need the connector type - use the one from the update if provided, else the existing one + current_connector_type = connector_update.connector_type if connector_update.connector_type is not None else db_connector.connector_type + + try: + # We can reuse the base validator by creating a temporary base model instance + # Note: This assumes 'name' and 'is_indexable' are not crucial for config validation itself + temp_data_for_validation = { + "name": db_connector.name, # Use existing name + "connector_type": current_connector_type, + "is_indexable": db_connector.is_indexable, # Use existing value + "last_indexed_at": db_connector.last_indexed_at, # Not used by validator + "config": merged_config + } + SearchSourceConnectorBase.model_validate(temp_data_for_validation) + except ValidationError as e: + # Raise specific validation error for the merged config + raise HTTPException( + status_code=422, + detail=f"Validation error for merged config: {str(e)}" + ) + + # If validation passes, update the main update_data dict with the merged config + update_data["config"] = merged_config + + # Apply all updates (including the potentially merged config) + for key, value in update_data.items(): + # Prevent changing connector_type if it causes a duplicate (check moved here) + if key == "connector_type" and value != db_connector.connector_type: result = await session.execute( select(SearchSourceConnector) .filter( SearchSourceConnector.user_id == user.id, - SearchSourceConnector.connector_type == connector_update.connector_type, + SearchSourceConnector.connector_type == value, SearchSourceConnector.id != connector_id ) ) existing_connector = result.scalars().first() - if existing_connector: raise HTTPException( status_code=409, - detail=f"A connector with type {connector_update.connector_type} already exists. Each user can have only one connector of each type." + detail=f"A connector with type {value} already exists. Each user can have only one connector of each type." ) - update_data = connector_update.model_dump(exclude_unset=True) - for key, value in update_data.items(): - setattr(db_connector, key, value) + setattr(db_connector, key, value) + + try: await session.commit() await session.refresh(db_connector) return db_connector - except ValidationError as e: - await session.rollback() - raise HTTPException( - status_code=422, - detail=f"Validation error: {str(e)}" - ) except IntegrityError as e: await session.rollback() + # This might occur if connector_type constraint is violated somehow after the check raise HTTPException( status_code=409, - detail=f"Integrity error: A connector with this type already exists. {str(e)}" + detail=f"Database integrity error during update: {str(e)}" ) - except HTTPException: - await session.rollback() - raise except Exception as e: await session.rollback() + logger.error(f"Failed to update search source connector {connector_id}: {e}", exc_info=True) raise HTTPException( status_code=500, detail=f"Failed to update search source connector: {str(e)}" diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 1005a63..d0c4631 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -1,6 +1,6 @@ from datetime import datetime import uuid -from typing import Dict, Any +from typing import Dict, Any, Optional from pydantic import BaseModel, field_validator from .base import IDModel, TimestampModel from app.db import SearchSourceConnectorType @@ -9,7 +9,7 @@ class SearchSourceConnectorBase(BaseModel): name: str connector_type: SearchSourceConnectorType is_indexable: bool - last_indexed_at: datetime | None + last_indexed_at: Optional[datetime] = None config: Dict[str, Any] @field_validator('config') @@ -77,8 +77,12 @@ class SearchSourceConnectorBase(BaseModel): class SearchSourceConnectorCreate(SearchSourceConnectorBase): pass -class SearchSourceConnectorUpdate(SearchSourceConnectorBase): - pass +class SearchSourceConnectorUpdate(BaseModel): + name: Optional[str] = None + connector_type: Optional[SearchSourceConnectorType] = None + is_indexable: Optional[bool] = None + last_indexed_at: Optional[datetime] = None + config: Optional[Dict[str, Any]] = None class SearchSourceConnectorRead(SearchSourceConnectorBase, IDModel, TimestampModel): user_id: uuid.UUID diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx index 817ca58..bb890eb 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/(manage)/page.tsx @@ -205,7 +205,7 @@ export default function ConnectorsPage() { + + + + + Edit GitHub Connector + + Modify the connector name and repository selections. To change repository selections, you need to re-enter your PAT. + + + + {/* Use editForm for the main form structure */} +
+ + + {/* Connector Name Field */} + ( + + Connector Name + + + + + + )} + /> + +
+ + {/* Repository Selection Section */} +
+

Repository Selection

+ + {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected for indexing)

+ )} + + Click "Change Selection" to re-enter your PAT and update the list. +
+ )} + + {editMode === 'editing_repos' && ( +
+ {/* PAT Input Section (No nested Form provider) */} + {/* We still use patForm fields but trigger validation manually */} +
+ ( + + Re-enter PAT to Fetch Repos + + + + + + )} + /> + +
+ + {/* Fetched Repository List (shown after fetch) */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + + + No Repositories Found + Check the PAT and permissions. + + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} + /> + +
+ ))} +
+
+ ) + )} + +
+ )} +
+ + + + + + +
+ + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx index e841639..ad6ceb7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/page.tsx @@ -70,7 +70,7 @@ export default function EditConnectorPage() { const [connector, setConnector] = useState(null); const [isLoading, setIsLoading] = useState(true); const [isSubmitting, setIsSubmitting] = useState(false); - + console.log("connector", connector); // Initialize the form const form = useForm({ resolver: zodResolver(apiConnectorFormSchema), From 69eea7485b00e673bcc6fb2783a1f2280d7882b1 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 20:38:44 -0700 Subject: [PATCH 3/6] reuse edit page for other connectors --- .../connectors/[connector_id]/edit/page.tsx | 482 +++++++++++------- 1 file changed, 295 insertions(+), 187 deletions(-) diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index e526976..cc348ba 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -37,7 +37,19 @@ import { import { Checkbox } from "@/components/ui/checkbox"; import { Skeleton } from "@/components/ui/skeleton"; -// Schema for PAT input when editing repos +// Helper function to get connector type display name (copied from manage page) +const getConnectorTypeDisplay = (type: string): string => { + const typeMap: Record = { + "SERPER_API": "Serper API", + "TAVILY_API": "Tavily API", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + }; + return typeMap[type] || type; +}; + +// Schema for PAT input when editing GitHub repos (remains separate) const githubPatSchema = z.object({ github_pat: z.string() .min(20, { message: "GitHub Personal Access Token seems too short." }) @@ -47,9 +59,15 @@ const githubPatSchema = z.object({ }); type GithubPatFormValues = z.infer; -// Schema for main edit form (just the name for now) +// Updated schema for main edit form - includes optional fields for other connector configs const editConnectorSchema = z.object({ name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), + // Add optional fields for other connector types' configs + SLACK_BOT_TOKEN: z.string().optional(), + NOTION_INTEGRATION_TOKEN: z.string().optional(), + SERPER_API_KEY: z.string().optional(), + TAVILY_API_KEY: z.string().optional(), + // GITHUB_PAT is handled separately via patForm for repo editing flow }); type EditConnectorFormValues = z.infer; @@ -63,9 +81,9 @@ interface GithubRepo { last_updated: string | null; } -type EditMode = 'viewing' | 'editing_repos'; +type EditMode = 'viewing' | 'editing_repos'; // Only relevant for GitHub -export default function EditGithubConnectorPage() { +export default function EditConnectorPage() { // Renamed for clarity const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; @@ -74,54 +92,74 @@ export default function EditGithubConnectorPage() { const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); const [connector, setConnector] = useState(null); + const [originalConfig, setOriginalConfig] = useState | null>(null); // Store original config object + + // GitHub specific state (only used if connector type is GitHub) const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); - const [originalPat, setOriginalPat] = useState(""); // State to hold the initial PAT - const [editMode, setEditMode] = useState('viewing'); - const [fetchedRepos, setFetchedRepos] = useState(null); // Null indicates not fetched yet for edit - const [newSelectedRepos, setNewSelectedRepos] = useState([]); // Tracks selections *during* edit + const [originalPat, setOriginalPat] = useState(""); + const [editMode, setEditMode] = useState('viewing'); + const [fetchedRepos, setFetchedRepos] = useState(null); + const [newSelectedRepos, setNewSelectedRepos] = useState([]); const [isFetchingRepos, setIsFetchingRepos] = useState(false); + const [isSaving, setIsSaving] = useState(false); - // Form for just the PAT input + // Form for GitHub PAT input (only used for GitHub repo editing) const patForm = useForm({ resolver: zodResolver(githubPatSchema), - defaultValues: { github_pat: "" }, // Default empty, will be reset + defaultValues: { github_pat: "" }, }); - // Form for the main connector details (e.g., name) + // Main form for connector details (name + simple config fields) const editForm = useForm({ resolver: zodResolver(editConnectorSchema), - defaultValues: { name: "" }, + defaultValues: { + name: "", + SLACK_BOT_TOKEN: "", + NOTION_INTEGRATION_TOKEN: "", + SERPER_API_KEY: "", + TAVILY_API_KEY: "", + }, }); - // Effect to find and set the current connector details on load + // Effect to load connector data useEffect(() => { - if (!connectorsLoading && connectors.length > 0 && !connector) { // Added !connector check to prevent loop + if (!connectorsLoading && connectors.length > 0 && !connector) { const currentConnector = connectors.find(c => c.id === connectorId); - if (currentConnector && currentConnector.connector_type === 'GITHUB_CONNECTOR') { + if (currentConnector) { setConnector(currentConnector); - const savedRepos = currentConnector.config?.repo_full_names || []; - const savedPat = currentConnector.config?.GITHUB_PAT || ""; - setCurrentSelectedRepos(savedRepos); - setNewSelectedRepos(savedRepos); - setOriginalPat(savedPat); // Store the original PAT - editForm.reset({ name: currentConnector.name }); - patForm.reset({ github_pat: savedPat }); // Also reset PAT form initially - } else if (currentConnector) { - toast.error("This connector is not a GitHub connector."); - router.push(`/dashboard/${searchSpaceId}/connectors`); + setOriginalConfig(currentConnector.config || {}); // Store original config + + // Reset main form with common and type-specific fields + editForm.reset({ + name: currentConnector.name, + SLACK_BOT_TOKEN: currentConnector.config?.SLACK_BOT_TOKEN || "", + NOTION_INTEGRATION_TOKEN: currentConnector.config?.NOTION_INTEGRATION_TOKEN || "", + SERPER_API_KEY: currentConnector.config?.SERPER_API_KEY || "", + TAVILY_API_KEY: currentConnector.config?.TAVILY_API_KEY || "", + }); + + // If GitHub, set up GitHub-specific state + if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { + const savedRepos = currentConnector.config?.repo_full_names || []; + const savedPat = currentConnector.config?.GITHUB_PAT || ""; + setCurrentSelectedRepos(savedRepos); + setNewSelectedRepos(savedRepos); + setOriginalPat(savedPat); + patForm.reset({ github_pat: savedPat }); + setEditMode('viewing'); // Start in viewing mode for repos + } } else { toast.error("Connector not found."); router.push(`/dashboard/${searchSpaceId}/connectors`); } } - }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector]); // Removed editForm, patForm from dependencies + }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); // Fetch repositories using the entered PAT const handleFetchRepositories = async (values: GithubPatFormValues) => { setIsFetchingRepos(true); setFetchedRepos(null); - // No need for patInputValue state, values.github_pat has the submitted value try { const token = localStorage.getItem('surfsense_bearer_token'); if (!token) throw new Error('No authentication token found'); @@ -137,21 +175,17 @@ export default function EditGithubConnectorPage() { body: JSON.stringify({ github_pat: values.github_pat }) } ); - if (!response.ok) { const errorData = await response.json(); throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); } - const data: GithubRepo[] = await response.json(); setFetchedRepos(data); - // Reset selection based on currently SAVED repos when fetching - setNewSelectedRepos(currentSelectedRepos); + setNewSelectedRepos(currentSelectedRepos); toast.success(`Found ${data.length} repositories. Select which ones to index.`); } catch (error) { console.error("Error fetching GitHub repositories:", error); toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); - // Don't clear PAT form on error, let user fix it } finally { setIsFetchingRepos(false); } @@ -166,62 +200,97 @@ export default function EditGithubConnectorPage() { ); }; - // Save all changes (name and potentially repo selection + PAT) + // Save changes - updated to handle different connector types const handleSaveChanges = async (formData: EditConnectorFormValues) => { - if (!connector) return; + if (!connector || !originalConfig) return; setIsSaving(true); const updatePayload: Partial = {}; let configChanged = false; + let newConfig: Record | null = null; // 1. Check if name changed if (formData.name !== connector.name) { updatePayload.name = formData.name; } - // 2. Check PAT and Repo changes - const currentPatInForm = patForm.getValues('github_pat'); - let patChanged = false; + // 2. Check for config changes based on connector type + switch (connector.connector_type) { + case 'GITHUB_CONNECTOR': + const currentPatInForm = patForm.getValues('github_pat'); + const patChanged = currentPatInForm !== originalPat; + const initialRepoSet = new Set(currentSelectedRepos); + const newRepoSet = new Set(newSelectedRepos); + const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - // Check if PAT input field was actually edited - if (editMode === 'editing_repos' && currentPatInForm !== originalPat) { - patChanged = true; + if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { + if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { + toast.error("Invalid GitHub PAT format. Cannot save."); + setIsSaving(false); return; + } + newConfig = { + GITHUB_PAT: currentPatInForm, + repo_full_names: newSelectedRepos, + }; + if (reposChanged && newSelectedRepos.length === 0) { + toast.warning("Warning: No repositories selected."); + } + } + break; + + case 'SLACK_CONNECTOR': + if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { + if (!formData.SLACK_BOT_TOKEN) { + toast.error("Slack Bot Token cannot be empty."); setIsSaving(false); return; + } + newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; + } + break; + + case 'NOTION_CONNECTOR': + if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { + if (!formData.NOTION_INTEGRATION_TOKEN) { + toast.error("Notion Integration Token cannot be empty."); setIsSaving(false); return; + } + newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; + } + break; + + case 'SERPER_API': + if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { + if (!formData.SERPER_API_KEY) { + toast.error("Serper API Key cannot be empty."); setIsSaving(false); return; + } + newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; + } + break; + + case 'TAVILY_API': + if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { + if (!formData.TAVILY_API_KEY) { + toast.error("Tavily API Key cannot be empty."); setIsSaving(false); return; + } + newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; + } + break; + + // Add cases for other connector types if necessary } - // Check if repo selection was modified - const initialRepoSet = new Set(currentSelectedRepos); - const newRepoSet = new Set(newSelectedRepos); - const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - - // If PAT was changed OR repos were changed (implying PAT was involved) - if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { - // Validate the PAT from the form before including it - if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { - toast.error("Invalid GitHub PAT format in the input field. Cannot save config changes."); - setIsSaving(false); - return; - } - - updatePayload.config = { - // Use the PAT value currently in the form field - GITHUB_PAT: currentPatInForm, - // Use the latest repo selection state - repo_full_names: newSelectedRepos, - }; - configChanged = true; // Mark config as changed - - if (reposChanged && newSelectedRepos.length === 0) { - toast.warning("Warning: You haven't selected any repositories. The connector won't index anything."); - } + // If config was determined to have changed, add it to the payload + if (newConfig !== null) { + updatePayload.config = newConfig; + configChanged = true; } // 3. Check if there are actual changes to save if (Object.keys(updatePayload).length === 0) { toast.info("No changes detected."); setIsSaving(false); - setEditMode('viewing'); - // Reset PAT form to original value if returning to view mode without saving PAT change - patForm.reset({ github_pat: originalPat }); + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + patForm.reset({ github_pat: originalPat }); + } return; } @@ -230,29 +299,38 @@ export default function EditGithubConnectorPage() { await updateConnector(connectorId, updatePayload); toast.success("Connector updated successfully!"); - // Update local state based on what was *actually* saved - if (updatePayload.config) { - setCurrentSelectedRepos(updatePayload.config.repo_full_names || []); - setOriginalPat(updatePayload.config.GITHUB_PAT || ""); - // Reset PAT form with the newly saved PAT - patForm.reset({ github_pat: updatePayload.config.GITHUB_PAT || "" }); - } else { - // If config wasn't in payload, ensure PAT form is reset to original value - patForm.reset({ github_pat: originalPat }); - } - // Update connector name state if it changed (or rely on hook refresh) + // Update local state after successful save + const newlySavedConfig = updatePayload.config || originalConfig; + setOriginalConfig(newlySavedConfig); if (updatePayload.name) { - setConnector(prev => prev ? { ...prev, name: updatePayload.name! } : null); + setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); + editForm.setValue('name', updatePayload.name); + } else { + setConnector(prev => prev ? { ...prev, config: newlySavedConfig } : null); } - // Reset edit state - setEditMode('viewing'); - setFetchedRepos(null); - // Reset working selection to match saved state (use the updated currentSelectedRepos) - setNewSelectedRepos(updatePayload.config?.repo_full_names || currentSelectedRepos); + if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { + const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; + setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); + setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); + setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); + patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); + } else if (connector.connector_type === 'SLACK_CONNECTOR' && configChanged) { + editForm.setValue('SLACK_BOT_TOKEN', newlySavedConfig.SLACK_BOT_TOKEN || ""); + } // Add similar blocks for Notion, Serper, Tavily + else if (connector.connector_type === 'NOTION_CONNECTOR' && configChanged) { + editForm.setValue('NOTION_INTEGRATION_TOKEN', newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""); + } else if (connector.connector_type === 'SERPER_API' && configChanged) { + editForm.setValue('SERPER_API_KEY', newlySavedConfig.SERPER_API_KEY || ""); + } else if (connector.connector_type === 'TAVILY_API' && configChanged) { + editForm.setValue('TAVILY_API_KEY', newlySavedConfig.TAVILY_API_KEY || ""); + } - // Optionally redirect or rely on hook refresh - // router.push(`/dashboard/${searchSpaceId}/connectors`); + // Reset GitHub specific edit state + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + setFetchedRepos(null); + } } catch (error) { console.error("Error updating connector:", error); @@ -298,26 +376,27 @@ export default function EditGithubConnectorPage() { > - Edit GitHub Connector + {/* Title can be dynamic based on type */} + + {/* TODO: Make icon dynamic */} + Edit {getConnectorTypeDisplay(connector.connector_type)} Connector + - Modify the connector name and repository selections. To change repository selections, you need to re-enter your PAT. + Modify the connector name and configuration. - {/* Use editForm for the main form structure */}
- {/* Connector Name Field */} + {/* Name Field (Common) */} ( Connector Name - - - + )} @@ -325,108 +404,137 @@ export default function EditGithubConnectorPage() {
- {/* Repository Selection Section */} -
-

Repository Selection

+ {/* --- Conditional Configuration Section --- */} +

Configuration

- {editMode === 'viewing' && ( -
- Currently Indexed Repositories: - {currentSelectedRepos.length > 0 ? ( -
    - {currentSelectedRepos.map(repo =>
  • {repo}
    • )} -
- ) : ( -

(No repositories currently selected for indexing)

- )} - - Click "Change Selection" to re-enter your PAT and update the list. -
- )} - - {editMode === 'editing_repos' && ( -
- {/* PAT Input Section (No nested Form provider) */} - {/* We still use patForm fields but trigger validation manually */} -
- ( + {/* == GitHub == */} + {connector.connector_type === 'GITHUB_CONNECTOR' && ( +
+

Repository Selection & Access

+ {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected)

+ )} + + To change repo selections or update the PAT, click above. +
+ )} + {editMode === 'editing_repos' && ( +
+ {/* PAT Input */} +
+ ( - Re-enter PAT to Fetch Repos - - - + GitHub PAT + + Enter PAT to fetch/update repos or if you need to update the stored token. - )} - /> - +
+ {/* Repo List */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + No Repositories FoundCheck PAT & permissions. + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} /> + +
+ ))} +
+
+ ) + )} +
+ )} +
+ )} - {/* Fetched Repository List (shown after fetch) */} - {isFetchingRepos && } - {!isFetchingRepos && fetchedRepos !== null && ( - fetchedRepos.length === 0 ? ( - - - No Repositories Found - Check the PAT and permissions. - - ) : ( -
- Select Repositories to Index ({newSelectedRepos.length} selected): -
- {fetchedRepos.map((repo) => ( -
- handleRepoSelectionChange(repo.full_name, !!checked)} - /> - -
- ))} -
-
- ) - )} - -
- )} -
+ {/* == Slack == */} + {connector.connector_type === 'SLACK_CONNECTOR' && ( + ( + + Slack Bot Token + + Update the Slack Bot Token if needed. + + + )} + /> + )} + + {/* == Notion == */} + {connector.connector_type === 'NOTION_CONNECTOR' && ( + ( + + Notion Integration Token + + Update the Notion Integration Token if needed. + + + )} + /> + )} + + {/* == Serper API == */} + {connector.connector_type === 'SERPER_API' && ( + ( + + Serper API Key + + Update the Serper API Key if needed. + + + )} + /> + )} + + {/* == Tavily API == */} + {connector.connector_type === 'TAVILY_API' && ( + ( + + Tavily API Key + + Update the Tavily API Key if needed. + + + )} + /> + )} - - To change repo selections or update the PAT, click above. -
- )} - {editMode === 'editing_repos' && ( -
- {/* PAT Input */} -
- ( - - GitHub PAT - - Enter PAT to fetch/update repos or if you need to update the stored token. - - - )} /> - -
- {/* Repo List */} - {isFetchingRepos && } - {!isFetchingRepos && fetchedRepos !== null && ( - fetchedRepos.length === 0 ? ( - No Repositories FoundCheck PAT & permissions. - ) : ( -
- Select Repositories to Index ({newSelectedRepos.length} selected): -
- {fetchedRepos.map((repo) => ( -
- handleRepoSelectionChange(repo.full_name, !!checked)} /> - -
- ))} -
-
- ) - )} - -
- )} -
+ )} {/* == Slack == */} {connector.connector_type === 'SLACK_CONNECTOR' && ( - ( - - Slack Bot Token - - Update the Slack Bot Token if needed. - - - )} + )} {/* == Notion == */} {connector.connector_type === 'NOTION_CONNECTOR' && ( - ( - - Notion Integration Token - - Update the Notion Integration Token if needed. - - - )} + )} {/* == Serper API == */} {connector.connector_type === 'SERPER_API' && ( - ( - - Serper API Key - - Update the Serper API Key if needed. - - - )} + )} {/* == Tavily API == */} {connector.connector_type === 'TAVILY_API' && ( - ( - - Tavily API Key - - Update the Tavily API Key if needed. - - - )} + )} diff --git a/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx new file mode 100644 index 0000000..dc1320f --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorLoadingSkeleton.tsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { Skeleton } from "@/components/ui/skeleton"; +import { Card, CardContent, CardHeader } from "@/components/ui/card"; + +export function EditConnectorLoadingSkeleton() { + return ( +
+ + + + + + + + + + + +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditConnectorNameForm.tsx b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx new file mode 100644 index 0000000..3f18820 --- /dev/null +++ b/surfsense_web/components/editConnector/EditConnectorNameForm.tsx @@ -0,0 +1,25 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditConnectorNameFormProps { + control: Control; // Use Control if type is available +} + +export function EditConnectorNameForm({ control }: EditConnectorNameFormProps) { + return ( + ( + + Connector Name + + + + )} + /> + ); +} diff --git a/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx new file mode 100644 index 0000000..17f83f7 --- /dev/null +++ b/surfsense_web/components/editConnector/EditGitHubConnectorConfig.tsx @@ -0,0 +1,160 @@ +import React from 'react'; +import { UseFormReturn } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Button } from "@/components/ui/button"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Edit, KeyRound, Loader2, CircleAlert } from 'lucide-react'; + +// Types needed from parent +interface GithubRepo { + id: number; + name: string; + full_name: string; + private: boolean; + url: string; + description: string | null; + last_updated: string | null; +} +type GithubPatFormValues = { github_pat: string; }; +type EditMode = 'viewing' | 'editing_repos'; + +interface EditGitHubConnectorConfigProps { + // State from parent + editMode: EditMode; + originalPat: string; + currentSelectedRepos: string[]; + fetchedRepos: GithubRepo[] | null; + newSelectedRepos: string[]; + isFetchingRepos: boolean; + // Forms from parent + patForm: UseFormReturn; + // Handlers from parent + setEditMode: (mode: EditMode) => void; + handleFetchRepositories: (values: GithubPatFormValues) => Promise; + handleRepoSelectionChange: (repoFullName: string, checked: boolean) => void; + setNewSelectedRepos: React.Dispatch>; + setFetchedRepos: React.Dispatch>; +} + +export function EditGitHubConnectorConfig({ + editMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + newSelectedRepos, + isFetchingRepos, + patForm, + setEditMode, + handleFetchRepositories, + handleRepoSelectionChange, + setNewSelectedRepos, + setFetchedRepos +}: EditGitHubConnectorConfigProps) { + + return ( +
+

Repository Selection & Access

+ + {/* Viewing Mode */} + {editMode === 'viewing' && ( +
+ Currently Indexed Repositories: + {currentSelectedRepos.length > 0 ? ( +
    + {currentSelectedRepos.map(repo =>
  • {repo}
    • )} +
+ ) : ( +

(No repositories currently selected)

+ )} + + To change repo selections or update the PAT, click above. +
+ )} + + {/* Editing Mode */} + {editMode === 'editing_repos' && ( +
+ {/* PAT Input */} +
+ ( + + GitHub PAT + + Enter PAT to fetch/update repos or if you need to update the stored token. + + + )} + /> + +
+ + {/* Repo List */} + {isFetchingRepos && } + {!isFetchingRepos && fetchedRepos !== null && ( + fetchedRepos.length === 0 ? ( + + + No Repositories Found + Check PAT & permissions. + + ) : ( +
+ Select Repositories to Index ({newSelectedRepos.length} selected): +
+ {fetchedRepos.map((repo) => ( +
+ handleRepoSelectionChange(repo.full_name, !!checked)} + /> + +
+ ))} +
+
+ ) + )} + +
+ )} +
+ ); +} diff --git a/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx new file mode 100644 index 0000000..c0c8032 --- /dev/null +++ b/surfsense_web/components/editConnector/EditSimpleTokenForm.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { Control } from 'react-hook-form'; +import { FormField, FormItem, FormLabel, FormControl, FormDescription, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { KeyRound } from 'lucide-react'; + +// Assuming EditConnectorFormValues is defined elsewhere or passed as generic +interface EditSimpleTokenFormProps { + control: Control; + fieldName: string; // e.g., "SLACK_BOT_TOKEN" + fieldLabel: string; // e.g., "Slack Bot Token" + fieldDescription: string; + placeholder?: string; +} + +export function EditSimpleTokenForm({ + control, + fieldName, + fieldLabel, + fieldDescription, + placeholder +}: EditSimpleTokenFormProps) { + return ( + ( + + {fieldLabel} + + {fieldDescription} + + + )} + /> + ); +} From 53436370d6fdbb707e37ab86614a535a7a3ea2f8 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 21:25:29 -0700 Subject: [PATCH 5/6] refactor edit connector page for smaller file size --- .../connectors/[connector_id]/edit/page.tsx | 400 +++--------------- .../components/editConnector/types.ts | 33 ++ surfsense_web/hooks/useConnectorEditPage.ts | 210 +++++++++ surfsense_web/lib/connectors/utils.ts | 11 + 4 files changed, 312 insertions(+), 342 deletions(-) create mode 100644 surfsense_web/components/editConnector/types.ts create mode 100644 surfsense_web/hooks/useConnectorEditPage.ts create mode 100644 surfsense_web/lib/connectors/utils.ts diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 2032aab..00824bc 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -1,372 +1,90 @@ "use client"; -import { useState, useEffect } from "react"; +import React, { useEffect } from 'react'; import { useRouter, useParams } from "next/navigation"; import { motion } from "framer-motion"; -import { zodResolver } from "@hookform/resolvers/zod"; -import { useForm } from "react-hook-form"; -import * as z from "zod"; import { toast } from "sonner"; -import { ArrowLeft, Check, Loader2, Github, } from "lucide-react"; +import { ArrowLeft, Check, Loader2, Github } from "lucide-react"; -import { useSearchSourceConnectors, SearchSourceConnector } from "@/hooks/useSearchSourceConnectors"; -import { - Form, -} from "@/components/ui/form"; +import { Form } from "@/components/ui/form"; import { Button } from "@/components/ui/button"; -import { - Card, - CardContent, - CardDescription, - CardFooter, - CardHeader, - CardTitle, -} from "@/components/ui/card"; +import { Card, CardContent, CardDescription, CardFooter, CardHeader, CardTitle } from "@/components/ui/card"; + +// Import Utils, Types, Hook, and Components +import { getConnectorTypeDisplay } from '@/lib/connectors/utils'; +import { useConnectorEditPage } from '@/hooks/useConnectorEditPage'; import { EditConnectorLoadingSkeleton } from "@/components/editConnector/EditConnectorLoadingSkeleton"; import { EditConnectorNameForm } from "@/components/editConnector/EditConnectorNameForm"; import { EditGitHubConnectorConfig } from "@/components/editConnector/EditGitHubConnectorConfig"; import { EditSimpleTokenForm } from "@/components/editConnector/EditSimpleTokenForm"; - -// Helper function to get connector type display name (copied from manage page) -const getConnectorTypeDisplay = (type: string): string => { - const typeMap: Record = { - "SERPER_API": "Serper API", - "TAVILY_API": "Tavily API", - "SLACK_CONNECTOR": "Slack", - "NOTION_CONNECTOR": "Notion", - "GITHUB_CONNECTOR": "GitHub", - }; - return typeMap[type] || type; -}; - -// Schema for PAT input when editing GitHub repos (remains separate) -const githubPatSchema = z.object({ - github_pat: z.string() - .min(20, { message: "GitHub Personal Access Token seems too short." }) - .refine(pat => pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { - message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", - }), -}); -type GithubPatFormValues = z.infer; - -// Updated schema for main edit form - includes optional fields for other connector configs -const editConnectorSchema = z.object({ - name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), - // Add optional fields for other connector types' configs - SLACK_BOT_TOKEN: z.string().optional(), - NOTION_INTEGRATION_TOKEN: z.string().optional(), - SERPER_API_KEY: z.string().optional(), - TAVILY_API_KEY: z.string().optional(), - // GITHUB_PAT is handled separately via patForm for repo editing flow -}); -type EditConnectorFormValues = z.infer; - -interface GithubRepo { - id: number; - name: string; - full_name: string; - private: boolean; - url: string; - description: string | null; - last_updated: string | null; -} - -type EditMode = 'viewing' | 'editing_repos'; // Only relevant for GitHub - -export default function EditConnectorPage() { // Renamed for clarity +export default function EditConnectorPage() { const router = useRouter(); const params = useParams(); const searchSpaceId = params.search_space_id as string; - const connectorId = parseInt(params.connector_id as string, 10); + // Ensure connectorId is parsed safely + const connectorIdParam = params.connector_id as string; + const connectorId = connectorIdParam ? parseInt(connectorIdParam, 10) : NaN; - const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + // Use the custom hook to manage state and logic + const { + connectorsLoading, + connector, + isSaving, + editForm, + patForm, // Needed for GitHub child component + handleSaveChanges, + // GitHub specific props for the child component + editMode, + setEditMode, // Pass down if needed by GitHub component + originalPat, + currentSelectedRepos, + fetchedRepos, + setFetchedRepos, + newSelectedRepos, + setNewSelectedRepos, + isFetchingRepos, + handleFetchRepositories, + handleRepoSelectionChange, + } = useConnectorEditPage(connectorId, searchSpaceId); - const [connector, setConnector] = useState(null); - const [originalConfig, setOriginalConfig] = useState | null>(null); // Store original config object - - // GitHub specific state (only used if connector type is GitHub) - const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); - const [originalPat, setOriginalPat] = useState(""); - const [editMode, setEditMode] = useState('viewing'); - const [fetchedRepos, setFetchedRepos] = useState(null); - const [newSelectedRepos, setNewSelectedRepos] = useState([]); - const [isFetchingRepos, setIsFetchingRepos] = useState(false); - - const [isSaving, setIsSaving] = useState(false); - - // Form for GitHub PAT input (only used for GitHub repo editing) - const patForm = useForm({ - resolver: zodResolver(githubPatSchema), - defaultValues: { github_pat: "" }, - }); - - // Main form for connector details (name + simple config fields) - const editForm = useForm({ - resolver: zodResolver(editConnectorSchema), - defaultValues: { - name: "", - SLACK_BOT_TOKEN: "", - NOTION_INTEGRATION_TOKEN: "", - SERPER_API_KEY: "", - TAVILY_API_KEY: "", - }, - }); - - // Effect to load connector data + // Redirect if connectorId is not a valid number after parsing useEffect(() => { - if (!connectorsLoading && connectors.length > 0 && !connector) { - const currentConnector = connectors.find(c => c.id === connectorId); - if (currentConnector) { - setConnector(currentConnector); - setOriginalConfig(currentConnector.config || {}); // Store original config - - // Reset main form with common and type-specific fields - editForm.reset({ - name: currentConnector.name, - SLACK_BOT_TOKEN: currentConnector.config?.SLACK_BOT_TOKEN || "", - NOTION_INTEGRATION_TOKEN: currentConnector.config?.NOTION_INTEGRATION_TOKEN || "", - SERPER_API_KEY: currentConnector.config?.SERPER_API_KEY || "", - TAVILY_API_KEY: currentConnector.config?.TAVILY_API_KEY || "", - }); - - // If GitHub, set up GitHub-specific state - if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { - const savedRepos = currentConnector.config?.repo_full_names || []; - const savedPat = currentConnector.config?.GITHUB_PAT || ""; - setCurrentSelectedRepos(savedRepos); - setNewSelectedRepos(savedRepos); - setOriginalPat(savedPat); - patForm.reset({ github_pat: savedPat }); - setEditMode('viewing'); // Start in viewing mode for repos - } - } else { - toast.error("Connector not found."); - router.push(`/dashboard/${searchSpaceId}/connectors`); - } + if (isNaN(connectorId)) { + toast.error("Invalid Connector ID."); + router.push(`/dashboard/${searchSpaceId}/connectors`); } - }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); - - // Fetch repositories using the entered PAT - const handleFetchRepositories = async (values: GithubPatFormValues) => { - setIsFetchingRepos(true); - setFetchedRepos(null); - try { - const token = localStorage.getItem('surfsense_bearer_token'); - if (!token) throw new Error('No authentication token found'); - - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${token}` - }, - body: JSON.stringify({ github_pat: values.github_pat }) - } - ); - if (!response.ok) { - const errorData = await response.json(); - throw new Error(errorData.detail || `Failed to fetch repositories: ${response.statusText}`); - } - const data: GithubRepo[] = await response.json(); - setFetchedRepos(data); - setNewSelectedRepos(currentSelectedRepos); - toast.success(`Found ${data.length} repositories. Select which ones to index.`); - } catch (error) { - console.error("Error fetching GitHub repositories:", error); - toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); - } finally { - setIsFetchingRepos(false); - } - }; - - // Handle checkbox changes during editing - const handleRepoSelectionChange = (repoFullName: string, checked: boolean) => { - setNewSelectedRepos(prev => - checked - ? [...prev, repoFullName] - : prev.filter(name => name !== repoFullName) - ); - }; - - // Save changes - updated to handle different connector types - const handleSaveChanges = async (formData: EditConnectorFormValues) => { - if (!connector || !originalConfig) return; - - setIsSaving(true); - const updatePayload: Partial = {}; - let configChanged = false; - let newConfig: Record | null = null; - - // 1. Check if name changed - if (formData.name !== connector.name) { - updatePayload.name = formData.name; - } - - // 2. Check for config changes based on connector type - switch (connector.connector_type) { - case 'GITHUB_CONNECTOR': - const currentPatInForm = patForm.getValues('github_pat'); - const patChanged = currentPatInForm !== originalPat; - const initialRepoSet = new Set(currentSelectedRepos); - const newRepoSet = new Set(newSelectedRepos); - const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); - - if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { - if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { - toast.error("Invalid GitHub PAT format. Cannot save."); - setIsSaving(false); return; - } - newConfig = { - GITHUB_PAT: currentPatInForm, - repo_full_names: newSelectedRepos, - }; - if (reposChanged && newSelectedRepos.length === 0) { - toast.warning("Warning: No repositories selected."); - } - } - break; - - case 'SLACK_CONNECTOR': - if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { - if (!formData.SLACK_BOT_TOKEN) { - toast.error("Slack Bot Token cannot be empty."); setIsSaving(false); return; - } - newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; - } - break; - - case 'NOTION_CONNECTOR': - if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { - if (!formData.NOTION_INTEGRATION_TOKEN) { - toast.error("Notion Integration Token cannot be empty."); setIsSaving(false); return; - } - newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; - } - break; - - case 'SERPER_API': - if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { - if (!formData.SERPER_API_KEY) { - toast.error("Serper API Key cannot be empty."); setIsSaving(false); return; - } - newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; - } - break; - - case 'TAVILY_API': - if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { - if (!formData.TAVILY_API_KEY) { - toast.error("Tavily API Key cannot be empty."); setIsSaving(false); return; - } - newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; - } - break; - - // Add cases for other connector types if necessary - } - - // If config was determined to have changed, add it to the payload - if (newConfig !== null) { - updatePayload.config = newConfig; - configChanged = true; - } - - // 3. Check if there are actual changes to save - if (Object.keys(updatePayload).length === 0) { - toast.info("No changes detected."); - setIsSaving(false); - if (connector.connector_type === 'GITHUB_CONNECTOR') { - setEditMode('viewing'); - patForm.reset({ github_pat: originalPat }); - } - return; - } - - // 4. Proceed with update API call - try { - await updateConnector(connectorId, updatePayload); - toast.success("Connector updated successfully!"); - - // Update local state after successful save - const newlySavedConfig = updatePayload.config || originalConfig; - setOriginalConfig(newlySavedConfig); - if (updatePayload.name) { - setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); - editForm.setValue('name', updatePayload.name); - } else { - setConnector(prev => prev ? { ...prev, config: newlySavedConfig } : null); - } - - if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { - const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; - setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); - setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); - setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); - patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); - } else if (connector.connector_type === 'SLACK_CONNECTOR' && configChanged) { - editForm.setValue('SLACK_BOT_TOKEN', newlySavedConfig.SLACK_BOT_TOKEN || ""); - } // Add similar blocks for Notion, Serper, Tavily - else if (connector.connector_type === 'NOTION_CONNECTOR' && configChanged) { - editForm.setValue('NOTION_INTEGRATION_TOKEN', newlySavedConfig.NOTION_INTEGRATION_TOKEN || ""); - } else if (connector.connector_type === 'SERPER_API' && configChanged) { - editForm.setValue('SERPER_API_KEY', newlySavedConfig.SERPER_API_KEY || ""); - } else if (connector.connector_type === 'TAVILY_API' && configChanged) { - editForm.setValue('TAVILY_API_KEY', newlySavedConfig.TAVILY_API_KEY || ""); - } - - // Reset GitHub specific edit state - if (connector.connector_type === 'GITHUB_CONNECTOR') { - setEditMode('viewing'); - setFetchedRepos(null); - } - - } catch (error) { - console.error("Error updating connector:", error); - toast.error(error instanceof Error ? error.message : "Failed to update connector."); - } finally { - setIsSaving(false); - } - }; + }, [connectorId, router, searchSpaceId]); + // Loading State if (connectorsLoading || !connector) { + // Handle NaN case before showing skeleton + if (isNaN(connectorId)) return null; return ; } + // Main Render using data/handlers from the hook return (
- - + - {/* Title can be dynamic based on type */} - {/* TODO: Make icon dynamic */} + {/* TODO: Dynamic icon */} Edit {getConnectorTypeDisplay(connector.connector_type)} Connector - - Modify the connector name and configuration. - + Modify connector name and configuration. - + + {/* Pass hook's handleSaveChanges */} - {/* Name Component */} + {/* Pass form control from hook */}
@@ -376,14 +94,15 @@ export default function EditConnectorPage() { // Renamed for clarity {/* == GitHub == */} {connector.connector_type === 'GITHUB_CONNECTOR' && ( )} - {/* == Notion == */} {connector.connector_type === 'NOTION_CONNECTOR' && ( )} - - {/* == Serper API == */} + {/* == Serper == */} {connector.connector_type === 'SERPER_API' && ( )} - - {/* == Tavily API == */} + {/* == Tavily == */} {connector.connector_type === 'TAVILY_API' && ( pat.startsWith('ghp_') || pat.startsWith('github_pat_'), { + message: "GitHub PAT should start with 'ghp_' or 'github_pat_'", + }), +}); +export type GithubPatFormValues = z.infer; + +export const editConnectorSchema = z.object({ + name: z.string().min(3, { message: "Connector name must be at least 3 characters." }), + SLACK_BOT_TOKEN: z.string().optional(), + NOTION_INTEGRATION_TOKEN: z.string().optional(), + SERPER_API_KEY: z.string().optional(), + TAVILY_API_KEY: z.string().optional(), +}); +export type EditConnectorFormValues = z.infer; diff --git a/surfsense_web/hooks/useConnectorEditPage.ts b/surfsense_web/hooks/useConnectorEditPage.ts new file mode 100644 index 0000000..e75998e --- /dev/null +++ b/surfsense_web/hooks/useConnectorEditPage.ts @@ -0,0 +1,210 @@ +import { useState, useEffect, useCallback } from 'react'; +import { useRouter } from 'next/navigation'; +import { useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { toast } from 'sonner'; +import { useSearchSourceConnectors, SearchSourceConnector } from '@/hooks/useSearchSourceConnectors'; +import { + GithubRepo, + EditMode, + githubPatSchema, + editConnectorSchema, + GithubPatFormValues, + EditConnectorFormValues +} from '@/components/editConnector/types'; + +export function useConnectorEditPage(connectorId: number, searchSpaceId: string) { + const router = useRouter(); + const { connectors, updateConnector, isLoading: connectorsLoading } = useSearchSourceConnectors(); + + // State managed by the hook + const [connector, setConnector] = useState(null); + const [originalConfig, setOriginalConfig] = useState | null>(null); + const [isSaving, setIsSaving] = useState(false); + const [currentSelectedRepos, setCurrentSelectedRepos] = useState([]); + const [originalPat, setOriginalPat] = useState(""); + const [editMode, setEditMode] = useState('viewing'); + const [fetchedRepos, setFetchedRepos] = useState(null); + const [newSelectedRepos, setNewSelectedRepos] = useState([]); + const [isFetchingRepos, setIsFetchingRepos] = useState(false); + + // Forms managed by the hook + const patForm = useForm({ + resolver: zodResolver(githubPatSchema), + defaultValues: { github_pat: "" }, + }); + const editForm = useForm({ + resolver: zodResolver(editConnectorSchema), + defaultValues: { name: "", SLACK_BOT_TOKEN: "", NOTION_INTEGRATION_TOKEN: "", SERPER_API_KEY: "", TAVILY_API_KEY: "" }, + }); + + // Effect to load initial data + useEffect(() => { + if (!connectorsLoading && connectors.length > 0 && !connector) { + const currentConnector = connectors.find(c => c.id === connectorId); + if (currentConnector) { + setConnector(currentConnector); + const config = currentConnector.config || {}; + setOriginalConfig(config); + editForm.reset({ + name: currentConnector.name, + SLACK_BOT_TOKEN: config.SLACK_BOT_TOKEN || "", + NOTION_INTEGRATION_TOKEN: config.NOTION_INTEGRATION_TOKEN || "", + SERPER_API_KEY: config.SERPER_API_KEY || "", + TAVILY_API_KEY: config.TAVILY_API_KEY || "", + }); + if (currentConnector.connector_type === 'GITHUB_CONNECTOR') { + const savedRepos = config.repo_full_names || []; + const savedPat = config.GITHUB_PAT || ""; + setCurrentSelectedRepos(savedRepos); + setNewSelectedRepos(savedRepos); + setOriginalPat(savedPat); + patForm.reset({ github_pat: savedPat }); + setEditMode('viewing'); + } + } else { + toast.error("Connector not found."); + router.push(`/dashboard/${searchSpaceId}/connectors`); + } + } + }, [connectorId, connectors, connectorsLoading, router, searchSpaceId, connector, editForm, patForm]); + + // Handlers managed by the hook + const handleFetchRepositories = useCallback(async (values: GithubPatFormValues) => { + setIsFetchingRepos(true); + setFetchedRepos(null); + try { + const token = localStorage.getItem('surfsense_bearer_token'); + if (!token) throw new Error('No auth token'); + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/github/repositories/`, + { method: 'POST', headers: { 'Content-Type': 'application/json', 'Authorization': `Bearer ${token}` }, body: JSON.stringify({ github_pat: values.github_pat }) } + ); + if (!response.ok) { const err = await response.json(); throw new Error(err.detail || 'Fetch failed'); } + const data: GithubRepo[] = await response.json(); + setFetchedRepos(data); + setNewSelectedRepos(currentSelectedRepos); + toast.success(`Found ${data.length} repos.`); + } catch (error) { + console.error("Error fetching GitHub repositories:", error); + toast.error(error instanceof Error ? error.message : "Failed to fetch repositories."); + } finally { setIsFetchingRepos(false); } + }, [currentSelectedRepos]); // Added dependency + + const handleRepoSelectionChange = useCallback((repoFullName: string, checked: boolean) => { + setNewSelectedRepos(prev => checked ? [...prev, repoFullName] : prev.filter(name => name !== repoFullName)); + }, []); + + const handleSaveChanges = useCallback(async (formData: EditConnectorFormValues) => { + if (!connector || !originalConfig) return; + setIsSaving(true); + const updatePayload: Partial = {}; + let configChanged = false; + let newConfig: Record | null = null; + + if (formData.name !== connector.name) { + updatePayload.name = formData.name; + } + + switch (connector.connector_type) { + case 'GITHUB_CONNECTOR': + const currentPatInForm = patForm.getValues('github_pat'); + const patChanged = currentPatInForm !== originalPat; + const initialRepoSet = new Set(currentSelectedRepos); + const newRepoSet = new Set(newSelectedRepos); + const reposChanged = initialRepoSet.size !== newRepoSet.size || ![...initialRepoSet].every(repo => newRepoSet.has(repo)); + if (patChanged || (editMode === 'editing_repos' && reposChanged && fetchedRepos !== null)) { + if (!currentPatInForm || !(currentPatInForm.startsWith('ghp_') || currentPatInForm.startsWith('github_pat_'))) { + toast.error("Invalid GitHub PAT format. Cannot save."); setIsSaving(false); return; + } + newConfig = { GITHUB_PAT: currentPatInForm, repo_full_names: newSelectedRepos }; + if (reposChanged && newSelectedRepos.length === 0) { toast.warning("Warning: No repositories selected."); } + } + break; + case 'SLACK_CONNECTOR': + if (formData.SLACK_BOT_TOKEN !== originalConfig.SLACK_BOT_TOKEN) { + if (!formData.SLACK_BOT_TOKEN) { toast.error("Slack Token empty."); setIsSaving(false); return; } + newConfig = { SLACK_BOT_TOKEN: formData.SLACK_BOT_TOKEN }; + } + break; + // ... other cases ... + case 'NOTION_CONNECTOR': + if (formData.NOTION_INTEGRATION_TOKEN !== originalConfig.NOTION_INTEGRATION_TOKEN) { + if (!formData.NOTION_INTEGRATION_TOKEN) { toast.error("Notion Token empty."); setIsSaving(false); return; } + newConfig = { NOTION_INTEGRATION_TOKEN: formData.NOTION_INTEGRATION_TOKEN }; + } + break; + case 'SERPER_API': + if (formData.SERPER_API_KEY !== originalConfig.SERPER_API_KEY) { + if (!formData.SERPER_API_KEY) { toast.error("Serper Key empty."); setIsSaving(false); return; } + newConfig = { SERPER_API_KEY: formData.SERPER_API_KEY }; + } + break; + case 'TAVILY_API': + if (formData.TAVILY_API_KEY !== originalConfig.TAVILY_API_KEY) { + if (!formData.TAVILY_API_KEY) { toast.error("Tavily Key empty."); setIsSaving(false); return; } + newConfig = { TAVILY_API_KEY: formData.TAVILY_API_KEY }; + } + break; + } + + if (newConfig !== null) { + updatePayload.config = newConfig; + configChanged = true; + } + + if (Object.keys(updatePayload).length === 0) { + toast.info("No changes detected."); + setIsSaving(false); + if (connector.connector_type === 'GITHUB_CONNECTOR') { setEditMode('viewing'); patForm.reset({ github_pat: originalPat }); } + return; + } + + try { + await updateConnector(connectorId, updatePayload); + toast.success("Connector updated!"); + const newlySavedConfig = updatePayload.config || originalConfig; + setOriginalConfig(newlySavedConfig); + if (updatePayload.name) { + setConnector(prev => prev ? { ...prev, name: updatePayload.name!, config: newlySavedConfig } : null); + } + if (connector.connector_type === 'GITHUB_CONNECTOR' && configChanged) { + const savedGitHubConfig = newlySavedConfig as { GITHUB_PAT?: string; repo_full_names?: string[] }; + setCurrentSelectedRepos(savedGitHubConfig.repo_full_names || []); + setOriginalPat(savedGitHubConfig.GITHUB_PAT || ""); + setNewSelectedRepos(savedGitHubConfig.repo_full_names || []); + patForm.reset({ github_pat: savedGitHubConfig.GITHUB_PAT || "" }); + } + if (connector.connector_type === 'GITHUB_CONNECTOR') { + setEditMode('viewing'); + setFetchedRepos(null); + } + // Resetting simple form values is handled by useEffect if connector state updates + } catch (error) { + console.error("Error updating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to update connector."); + } finally { setIsSaving(false); } + }, [connector, originalConfig, updateConnector, connectorId, patForm, originalPat, currentSelectedRepos, newSelectedRepos, editMode, fetchedRepos]); // Added dependencies + + // Return values needed by the component + return { + connectorsLoading, + connector, + isSaving, + editForm, + patForm, + handleSaveChanges, + // GitHub specific props + editMode, + setEditMode, + originalPat, + currentSelectedRepos, + fetchedRepos, + setFetchedRepos, + newSelectedRepos, + setNewSelectedRepos, + isFetchingRepos, + handleFetchRepositories, + handleRepoSelectionChange, + }; +} diff --git a/surfsense_web/lib/connectors/utils.ts b/surfsense_web/lib/connectors/utils.ts new file mode 100644 index 0000000..f0a6dab --- /dev/null +++ b/surfsense_web/lib/connectors/utils.ts @@ -0,0 +1,11 @@ +// Helper function to get connector type display name +export const getConnectorTypeDisplay = (type: string): string => { + const typeMap: Record = { + "SERPER_API": "Serper API", + "TAVILY_API": "Tavily API", + "SLACK_CONNECTOR": "Slack", + "NOTION_CONNECTOR": "Notion", + "GITHUB_CONNECTOR": "GitHub", + }; + return typeMap[type] || type; +}; From 32c721113cda58763ec3e28f6b4809014952f416 Mon Sep 17 00:00:00 2001 From: Adamsmith6300 Date: Wed, 16 Apr 2025 22:06:50 -0700 Subject: [PATCH 6/6] update edit connectors page to support linear connector --- .../routes/search_source_connectors_routes.py | 6 +- .../connectors/[connector_id]/edit/page.tsx | 11 +++ .../components/editConnector/types.ts | 1 + surfsense_web/hooks/useConnectorEditPage.ts | 72 +++++++++++++------ surfsense_web/lib/connectors/utils.ts | 1 + 5 files changed, 68 insertions(+), 23 deletions(-) diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index 6c03496..786c19d 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -18,8 +18,9 @@ from app.db import get_async_session, User, SearchSourceConnector, SearchSourceC from app.schemas import SearchSourceConnectorCreate, SearchSourceConnectorUpdate, SearchSourceConnectorRead, SearchSourceConnectorBase from app.users import current_active_user from app.utils.check_ownership import check_ownership -from pydantic import ValidationError +from pydantic import BaseModel, Field, ValidationError from app.tasks.connectors_indexing_tasks import index_slack_messages, index_notion_pages, index_github_repos, index_linear_issues +from app.connectors.github_connector import GitHubConnector from datetime import datetime, timezone, timedelta import logging @@ -28,7 +29,7 @@ logger = logging.getLogger(__name__) router = APIRouter() -# --- New Schema for GitHub PAT --- +# Use Pydantic's BaseModel here class GitHubPATRequest(BaseModel): github_pat: str = Field(..., description="GitHub Personal Access Token") @@ -104,6 +105,7 @@ async def create_search_source_connector( await session.rollback() raise except Exception as e: + logger.error(f"Failed to create search source connector: {str(e)}") await session.rollback() raise HTTPException( status_code=500, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx index 00824bc..d41295f 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/[connector_id]/edit/page.tsx @@ -149,6 +149,17 @@ export default function EditConnectorPage() { /> )} + {/* == Linear == */} + {connector.connector_type === 'LINEAR_CONNECTOR' && ( + + )} +