From 9abaf4fd2a8c2370f9a820d369c15d6e5b8842ce Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 00:05:55 +0200 Subject: [PATCH 1/9] add (backend) google calendar connection flow --- surfsense_backend/app/db.py | 29 +++++ .../routes/google_calendar_connector_route.py | 121 ++++++++++++++++++ .../app/schemas/google_calendar_accounts.py | 26 ++++ 3 files changed, 176 insertions(+) create mode 100644 surfsense_backend/app/routes/google_calendar_connector_route.py create mode 100644 surfsense_backend/app/schemas/google_calendar_accounts.py diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index d749b3b..33a4057 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -46,6 +46,7 @@ class DocumentType(str, Enum): JIRA_CONNECTOR = "JIRA_CONNECTOR" CONFLUENCE_CONNECTOR = "CONFLUENCE_CONNECTOR" CLICKUP_CONNECTOR = "CLICKUP_CONNECTOR" + GOOGLE_CALENDAR_CONNECTOR = "GOOGLE_CALENDAR_CONNECTOR" class SearchSourceConnectorType(str, Enum): @@ -60,6 +61,7 @@ class SearchSourceConnectorType(str, Enum): JIRA_CONNECTOR = "JIRA_CONNECTOR" CONFLUENCE_CONNECTOR = "CONFLUENCE_CONNECTOR" CLICKUP_CONNECTOR = "CLICKUP_CONNECTOR" + GOOGLE_CALENDAR_CONNECTOR = "GOOGLE_CALENDAR_CONNECTOR" class ChatType(str, Enum): @@ -244,6 +246,21 @@ class SearchSourceConnector(BaseModel, TimestampMixin): user = relationship("User", back_populates="search_source_connectors") +class GoogleCalendarAccount(BaseModel): + __tablename__ = "google_calendar_accounts" + + user_id = Column( + UUID(as_uuid=True), + ForeignKey("user.id", ondelete="CASCADE"), + nullable=False, + unique=True, + ) + + access_token = Column(String, nullable=False) + refresh_token = Column(String, nullable=False) + user = relationship("User", back_populates="calendar_account") + + class LLMConfig(BaseModel, TimestampMixin): __tablename__ = "llm_configs" @@ -297,6 +314,12 @@ if config.AUTH_TYPE == "GOOGLE": search_source_connectors = relationship( "SearchSourceConnector", back_populates="user" ) + calendar_account = relationship( + "GoogleCalendarAccount", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) llm_configs = relationship( "LLMConfig", back_populates="user", @@ -331,6 +354,12 @@ else: search_source_connectors = relationship( "SearchSourceConnector", back_populates="user" ) + calendar_account = relationship( + "GoogleCalendarAccount", + back_populates="user", + uselist=False, + cascade="all, delete-orphan", + ) llm_configs = relationship( "LLMConfig", back_populates="user", diff --git a/surfsense_backend/app/routes/google_calendar_connector_route.py b/surfsense_backend/app/routes/google_calendar_connector_route.py new file mode 100644 index 0000000..d4eee5d --- /dev/null +++ b/surfsense_backend/app/routes/google_calendar_connector_route.py @@ -0,0 +1,121 @@ +# app/routes/google_calendar.py + +import base64 +import json +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi.responses import RedirectResponse +from google_auth_oauthlib.flow import Flow +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.future import select + +from app.config import config +from app.db import GoogleCalendarAccount, User, get_async_session +from app.users import current_active_user + +router = APIRouter() + +SCOPES = ["https://www.googleapis.com/auth/calendar.readonly"] +REDIRECT_URI = config.GOOGLE_CALENDAR_REDIRECT_URI + + +def get_google_flow(): + try: + return Flow.from_client_config( + { + "web": { + "client_id": config.GOOGLE_OAUTH_CLIENT_ID, + "client_secret": config.GOOGLE_OAUTH_CLIENT_SECRET, + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "redirect_uris": [REDIRECT_URI], + } + }, + scopes=SCOPES, + redirect_uri=REDIRECT_URI, + ) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to create Google flow: {e!s}" + ) from e + + +@router.get("/auth/google/calendar/connector/init/") +async def connect_calendar(space_id: int, user: User = Depends(current_active_user)): + try: + if not space_id: + raise HTTPException(status_code=400, detail="space_id is required") + + flow = get_google_flow() + + # Encode space_id and user_id in state + state_payload = json.dumps( + { + "space_id": space_id, + "user_id": str(user.id), + } + ) + state_encoded = base64.urlsafe_b64encode(state_payload.encode()).decode() + + auth_url, _ = flow.authorization_url( + access_type="offline", + prompt="consent", + include_granted_scopes="true", + state=state_encoded, + ) + return {"auth_url": auth_url} + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to initiate Google OAuth: {e!s}" + ) from e + + +@router.get("/auth/google/calendar/connector/callback/") +async def calendar_callback( + request: Request, + code: str, + state: str, + session: AsyncSession = Depends(get_async_session), +): + try: + # Decode and parse the state + decoded_state = base64.urlsafe_b64decode(state.encode()).decode() + data = json.loads(decoded_state) + + user_id = UUID(data["user_id"]) + space_id = data["space_id"] + + flow = get_google_flow() + flow.fetch_token(code=code) + + creds = flow.credentials + token = creds.token + refresh_token = creds.refresh_token + + existing = await session.scalar( + select(GoogleCalendarAccount).where( + GoogleCalendarAccount.user_id == user_id + ) + ) + if existing: + existing.access_token = token + existing.refresh_token = refresh_token or existing.refresh_token + else: + session.add( + GoogleCalendarAccount( + user_id=user_id, + access_token=token, + refresh_token=refresh_token, + ) + ) + + await session.commit() + + return RedirectResponse( + f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-calendar-connector?success=true" + ) + except Exception as e: + raise HTTPException( + status_code=500, detail=f"Failed to complete Google OAuth: {e!s}" + ) from e diff --git a/surfsense_backend/app/schemas/google_calendar_accounts.py b/surfsense_backend/app/schemas/google_calendar_accounts.py new file mode 100644 index 0000000..32e8355 --- /dev/null +++ b/surfsense_backend/app/schemas/google_calendar_accounts.py @@ -0,0 +1,26 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict + + +class GoogleCalendarAccountBase(BaseModel): + user_id: UUID + access_token: str + refresh_token: str + + +class GoogleCalendarAccountCreate(GoogleCalendarAccountBase): + pass + + +class GoogleCalendarAccountUpdate(BaseModel): + access_token: str + refresh_token: str + + +class GoogleCalendarAccountRead(BaseModel): + user_id: UUID + access_token: str + refresh_token: str + + model_config = ConfigDict(from_attributes=True) From f96afe0e2a2ea236817fecdbfb1322cb5586d573 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 01:36:36 +0200 Subject: [PATCH 2/9] update seach source connector schema --- .../app/schemas/google_auth_credentials.py | 13 ++++++++++ .../app/schemas/google_calendar_accounts.py | 26 ------------------- .../app/schemas/search_source_connector.py | 9 +++++++ 3 files changed, 22 insertions(+), 26 deletions(-) create mode 100644 surfsense_backend/app/schemas/google_auth_credentials.py delete mode 100644 surfsense_backend/app/schemas/google_calendar_accounts.py diff --git a/surfsense_backend/app/schemas/google_auth_credentials.py b/surfsense_backend/app/schemas/google_auth_credentials.py new file mode 100644 index 0000000..2d5a048 --- /dev/null +++ b/surfsense_backend/app/schemas/google_auth_credentials.py @@ -0,0 +1,13 @@ +from datetime import datetime + +from pydantic import BaseModel + + +class GoogleAuthCredentialsBase(BaseModel): + token: str + refresh_token: str + token_uri: str + client_id: str + expiry: datetime + scopes: list[str] + client_secret: str diff --git a/surfsense_backend/app/schemas/google_calendar_accounts.py b/surfsense_backend/app/schemas/google_calendar_accounts.py deleted file mode 100644 index 32e8355..0000000 --- a/surfsense_backend/app/schemas/google_calendar_accounts.py +++ /dev/null @@ -1,26 +0,0 @@ -from uuid import UUID - -from pydantic import BaseModel, ConfigDict - - -class GoogleCalendarAccountBase(BaseModel): - user_id: UUID - access_token: str - refresh_token: str - - -class GoogleCalendarAccountCreate(GoogleCalendarAccountBase): - pass - - -class GoogleCalendarAccountUpdate(BaseModel): - access_token: str - refresh_token: str - - -class GoogleCalendarAccountRead(BaseModel): - user_id: UUID - access_token: str - refresh_token: str - - model_config = ConfigDict(from_attributes=True) diff --git a/surfsense_backend/app/schemas/search_source_connector.py b/surfsense_backend/app/schemas/search_source_connector.py index 28d5425..4c36893 100644 --- a/surfsense_backend/app/schemas/search_source_connector.py +++ b/surfsense_backend/app/schemas/search_source_connector.py @@ -5,6 +5,7 @@ from typing import Any from pydantic import BaseModel, ConfigDict, field_validator from app.db import SearchSourceConnectorType +from app.schemas.google_auth_credentials import GoogleAuthCredentialsBase from .base import IDModel, TimestampModel @@ -179,6 +180,14 @@ class SearchSourceConnectorBase(BaseModel): if not config.get("CLICKUP_API_TOKEN"): raise ValueError("CLICKUP_API_TOKEN cannot be empty") + elif connector_type == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR: + # Required fields + required_keys = list(GoogleAuthCredentialsBase.model_fields.keys()) + + for key in required_keys: + if key not in config or config[key] in (None, ""): + raise ValueError(f"{key} is required and cannot be empty") + return config From 1c9f857d004864e95481c025de2f17c4afe78fbb Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 01:41:41 +0200 Subject: [PATCH 3/9] add google calendar connector route --- surfsense_backend/app/routes/__init__.py | 2 + ...=> google_calendar_add_connector_route.py} | 75 ++++++++++++------ .../routes/search_source_connectors_routes.py | 79 +++++++++++++++++++ 3 files changed, 133 insertions(+), 23 deletions(-) rename surfsense_backend/app/routes/{google_calendar_connector_route.py => google_calendar_add_connector_route.py} (55%) diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 91c41ee..00b8f63 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -2,6 +2,7 @@ from fastapi import APIRouter from .chats_routes import router as chats_router from .documents_routes import router as documents_router +from .google_calendar_add_connector_route import router as google_oauth_router from .llm_config_routes import router as llm_config_router from .logs_routes import router as logs_router from .podcasts_routes import router as podcasts_router @@ -15,5 +16,6 @@ router.include_router(documents_router) router.include_router(podcasts_router) router.include_router(chats_router) router.include_router(search_source_connectors_router) +router.include_router(google_oauth_router) router.include_router(llm_config_router) router.include_router(logs_router) diff --git a/surfsense_backend/app/routes/google_calendar_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py similarity index 55% rename from surfsense_backend/app/routes/google_calendar_connector_route.py rename to surfsense_backend/app/routes/google_calendar_add_connector_route.py index d4eee5d..d78d6dd 100644 --- a/surfsense_backend/app/routes/google_calendar_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -2,16 +2,19 @@ import base64 import json +from sqlite3 import IntegrityError from uuid import UUID +from venv import logger from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import RedirectResponse from google_auth_oauthlib.flow import Flow +from jsonschema import ValidationError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from app.config import config -from app.db import GoogleCalendarAccount, User, get_async_session +from app.db import SearchSourceConnector, User, get_async_session from app.users import current_active_user router = APIRouter() @@ -41,7 +44,7 @@ def get_google_flow(): ) from e -@router.get("/auth/google/calendar/connector/init/") +@router.get("/auth/google/calendar/connector/add/") async def connect_calendar(space_id: int, user: User = Depends(current_active_user)): try: if not space_id: @@ -90,31 +93,57 @@ async def calendar_callback( flow.fetch_token(code=code) creds = flow.credentials - token = creds.token - refresh_token = creds.refresh_token + creds_dict = json.loads(creds.to_json()) - existing = await session.scalar( - select(GoogleCalendarAccount).where( - GoogleCalendarAccount.user_id == user_id - ) - ) - if existing: - existing.access_token = token - existing.refresh_token = refresh_token or existing.refresh_token - else: - session.add( - GoogleCalendarAccount( - user_id=user_id, - access_token=token, - refresh_token=refresh_token, + try: + # Check if a connector with the same type already exists for this user + result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.user_id == user_id, + SearchSourceConnector.connector_type == "GOOGLE_CALENDAR_CONNECTOR", ) ) + existing_connector = result.scalars().first() + if existing_connector: + raise HTTPException( + status_code=409, + detail="A GOOGLE_CALENDAR_CONNECTOR connector already exists. Each user can have only one connector of each type.", + ) + db_connector = SearchSourceConnector( + name="Google Calendar Connector", + connector_type="GOOGLE_CALENDAR_CONNECTOR", + config=creds_dict, + user_id=user_id, + is_indexable=True, + ) + session.add(db_connector) + await session.commit() + await session.refresh(db_connector) + return RedirectResponse( + f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-calendar-connector?success=true" + ) + except ValidationError as e: + await session.rollback() + raise HTTPException( + status_code=422, detail=f"Validation error: {e!s}" + ) from e + except IntegrityError as e: + await session.rollback() + raise HTTPException( + status_code=409, + detail=f"Integrity error: A connector with this type already exists. {e!s}", + ) from e + except HTTPException: + await session.rollback() + raise + except Exception as e: + logger.error(f"Failed to create search source connector: {e!s}") + await session.rollback() + raise HTTPException( + status_code=500, + detail=f"Failed to create search source connector: {e!s}", + ) from e - await session.commit() - - return RedirectResponse( - f"{config.NEXT_FRONTEND_URL}/dashboard/{space_id}/connectors/add/google-calendar-connector?success=true" - ) except Exception as e: raise HTTPException( status_code=500, detail=f"Failed to complete Google OAuth: {e!s}" diff --git a/surfsense_backend/app/routes/search_source_connectors_routes.py b/surfsense_backend/app/routes/search_source_connectors_routes.py index f91bd9f..a65f1cf 100644 --- a/surfsense_backend/app/routes/search_source_connectors_routes.py +++ b/surfsense_backend/app/routes/search_source_connectors_routes.py @@ -40,6 +40,7 @@ from app.tasks.connectors_indexing_tasks import ( index_confluence_pages, index_discord_messages, index_github_repos, + index_google_calendar_events, index_jira_issues, index_linear_issues, index_notion_pages, @@ -489,6 +490,24 @@ async def index_connector_content( ) response_message = "ClickUp indexing started in the background." + elif ( + connector.connector_type + == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR + ): + # Run indexing in background + logger.info( + f"Triggering Google Calendar indexing for connector {connector_id} into search space {search_space_id} from {indexing_from} to {indexing_to}" + ) + background_tasks.add_task( + run_google_calendar_indexing_with_new_session, + connector_id, + search_space_id, + str(user.id), + indexing_from, + indexing_to, + ) + response_message = "Google Calendar indexing started in the background." + elif connector.connector_type == SearchSourceConnectorType.DISCORD_CONNECTOR: # Run indexing in background logger.info( @@ -1034,3 +1053,63 @@ async def run_clickup_indexing( exc_info=True, ) # Optionally update status in DB to indicate failure + + +# Add new helper functions for Google Calendar indexing +async def run_google_calendar_indexing_with_new_session( + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """Wrapper to run Google Calendar indexing with its own database session.""" + logger.info( + f"Background task started: Indexing Google Calendar connector {connector_id} into space {search_space_id} from {start_date} to {end_date}" + ) + async with async_session_maker() as session: + await run_google_calendar_indexing( + session, connector_id, search_space_id, user_id, start_date, end_date + ) + logger.info( + f"Background task finished: Indexing Google Calendar connector {connector_id}" + ) + + +async def run_google_calendar_indexing( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str, + end_date: str, +): + """Runs the Google Calendar indexing task and updates the timestamp.""" + try: + indexed_count, error_message = await index_google_calendar_events( + session, + connector_id, + search_space_id, + user_id, + start_date, + end_date, + update_last_indexed=False, + ) + if error_message: + logger.error( + f"Google Calendar indexing failed for connector {connector_id}: {error_message}" + ) + # Optionally update status in DB to indicate failure + else: + logger.info( + f"Google Calendar indexing successful for connector {connector_id}. Indexed {indexed_count} documents." + ) + # Update the last indexed timestamp only on success + await update_connector_last_indexed(session, connector_id) + await session.commit() # Commit timestamp update + except Exception as e: + logger.error( + f"Critical error in run_google_calendar_indexing for connector {connector_id}: {e}", + exc_info=True, + ) + # Optionally update status in DB to indicate failure From 44d233866395217a406a95f1cf919e91dec3f303 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 02:22:53 +0200 Subject: [PATCH 4/9] get all primary calendar event by default --- .../connectors/google_calendar_connector.py | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) create mode 100644 surfsense_backend/app/connectors/google_calendar_connector.py diff --git a/surfsense_backend/app/connectors/google_calendar_connector.py b/surfsense_backend/app/connectors/google_calendar_connector.py new file mode 100644 index 0000000..d481491 --- /dev/null +++ b/surfsense_backend/app/connectors/google_calendar_connector.py @@ -0,0 +1,261 @@ +""" +Google Calendar Connector Module | Google OAuth Credentials | Google Calendar API + +A module for retrieving calendar events from Google Calendar using Google OAuth credentials. +Allows fetching events from specified calendars within date ranges using Google OAuth credentials. +""" + +from datetime import datetime +from typing import Any + +from google.auth.transport.requests import Request +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build + + +class GoogleCalendarConnector: + """Class for retrieving data from Google Calendar using Google OAuth credentials.""" + + def __init__( + self, + credentials: Credentials, + ): + """ + Initialize the GoogleCalendarConnector class. + + Args: + credentials: Google OAuth Credentials object + """ + self._credentials = credentials + self.service = None + + def _get_credentials(self) -> Credentials: + """ + Get valid Google OAuth credentials. + + Returns: + Google OAuth credentials + + Raises: + ValueError: If credentials have not been set + Exception: If credential refresh fails + """ + if not all( + [ + self._credentials.client_id, + self._credentials.client_secret, + self._credentials.refresh_token, + ] + ): + raise ValueError( + "Google OAuth credentials (client_id, client_secret, refresh_token) must be set" + ) + + if self._credentials and not self._credentials.expired: + return self._credentials + + # Create credentials from refresh token + self._credentials = Credentials( + token=self._credentials.token, + refresh_token=self._credentials.refresh_token, + token_uri=self._credentials.token_uri, + client_id=self._credentials.client_id, + client_secret=self._credentials.client_secret, + scopes=self._credentials.scopes, + ) + + # Refresh the token if needed + if self._credentials.expired or not self._credentials.valid: + try: + self._credentials.refresh(Request()) + except Exception as e: + raise Exception( + f"Failed to refresh Google OAuth credentials: {e!s}" + ) from e + + return self._credentials + + def _get_service(self): + """ + Get the Google Calendar service instance using Google OAuth credentials. + + Returns: + Google Calendar service instance + + Raises: + ValueError: If credentials have not been set + Exception: If service creation fails + """ + if self.service: + return self.service + + try: + credentials = self._get_credentials() + self.service = build("calendar", "v3", credentials=credentials) + return self.service + except Exception as e: + raise Exception(f"Failed to create Google Calendar service: {e!s}") from e + + def get_calendars(self) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch list of user's calendars using Google OAuth credentials. + + Returns: + Tuple containing (calendars list, error message or None) + """ + try: + service = self._get_service() + calendars_result = service.calendarList().list().execute() + calendars = calendars_result.get("items", []) + + # Format calendar data + formatted_calendars = [] + for calendar in calendars: + formatted_calendars.append( + { + "id": calendar.get("id"), + "summary": calendar.get("summary"), + "description": calendar.get("description", ""), + "primary": calendar.get("primary", False), + "accessRole": calendar.get("accessRole"), + "timeZone": calendar.get("timeZone"), + } + ) + + return formatted_calendars, None + + except Exception as e: + return [], f"Error fetching calendars: {e!s}" + + def get_all_primary_calendar_events( + self, + max_results: int = 2500, + ) -> tuple[list[dict[str, Any]], str | None]: + """ + Fetch events from the primary calendar using Google OAuth credentials. + + Args: + max_results: Maximum number of events to fetch (default: 2500) + + Returns: + Tuple containing (events list, error message or None) + """ + try: + service = self._get_service() + + # Fetch events + events_result = ( + service.events() + .list( + calendarId="primary", + maxResults=max_results, + singleEvents=True, + orderBy="startTime", + ) + .execute() + ) + + events = events_result.get("items", []) + + if not events: + return [], "No events found in the specified date range." + + return events, None + + except Exception as e: + return [], f"Error fetching events: {e!s}" + + def format_event_to_markdown(self, event: dict[str, Any]) -> str: + """ + Format a Google Calendar event to markdown. + + Args: + event: Event object from Google Calendar API + + Returns: + Formatted markdown string + """ + # Extract basic event information + summary = event.get("summary", "No Title") + description = event.get("description", "") + location = event.get("location", "") + calendar_id = event.get("calendarId", "") + + # Extract start and end times + start = event.get("start", {}) + end = event.get("end", {}) + + start_time = start.get("dateTime") or start.get("date", "") + end_time = end.get("dateTime") or end.get("date", "") + + # Format times for display + if start_time: + try: + if "T" in start_time: # DateTime format + start_dt = datetime.fromisoformat(start_time.replace("Z", "+00:00")) + start_formatted = start_dt.strftime("%Y-%m-%d %H:%M") + else: # Date format (all-day event) + start_formatted = start_time + except Exception: + start_formatted = start_time + else: + start_formatted = "Unknown" + + if end_time: + try: + if "T" in end_time: # DateTime format + end_dt = datetime.fromisoformat(end_time.replace("Z", "+00:00")) + end_formatted = end_dt.strftime("%Y-%m-%d %H:%M") + else: # Date format (all-day event) + end_formatted = end_time + except Exception: + end_formatted = end_time + else: + end_formatted = "Unknown" + + # Extract attendees + attendees = event.get("attendees", []) + attendee_list = [] + for attendee in attendees: + email = attendee.get("email", "") + display_name = attendee.get("displayName", email) + response_status = attendee.get("responseStatus", "") + attendee_list.append(f"- {display_name} ({response_status})") + + # Build markdown content + markdown_content = f"# {summary}\n\n" + + # Add event details + markdown_content += f"**Start:** {start_formatted}\n" + markdown_content += f"**End:** {end_formatted}\n" + + if location: + markdown_content += f"**Location:** {location}\n" + + if calendar_id: + markdown_content += f"**Calendar:** {calendar_id}\n" + + markdown_content += "\n" + + # Add description if available + if description: + markdown_content += f"## Description\n\n{description}\n\n" + + # Add attendees if available + if attendee_list: + markdown_content += "## Attendees\n\n" + markdown_content += "\n".join(attendee_list) + markdown_content += "\n\n" + + # Add event metadata + markdown_content += "## Event Details\n\n" + markdown_content += f"- **Event ID:** {event.get('id', 'Unknown')}\n" + markdown_content += f"- **Created:** {event.get('created', 'Unknown')}\n" + markdown_content += f"- **Updated:** {event.get('updated', 'Unknown')}\n" + + if event.get("recurringEventId"): + markdown_content += ( + f"- **Recurring Event ID:** {event.get('recurringEventId')}\n" + ) + + return markdown_content From edf46e4de13106c0990b4c1453addb06ec95b78e Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 04:39:48 +0200 Subject: [PATCH 5/9] update seach source connector schema --- surfsense_backend/.env.example | 1 + .../15_add_google_calendar_connector_enums.py | 65 +++ .../app/agents/researcher/nodes.py | 69 +++ .../agents/researcher/qna_agent/prompts.py | 1 + .../researcher/sub_section_writer/prompts.py | 3 + .../app/agents/researcher/utils.py | 2 + surfsense_backend/app/config/__init__.py | 7 +- .../connectors/google_calendar_connector.py | 41 +- surfsense_backend/app/db.py | 2 +- surfsense_backend/app/routes/__init__.py | 6 +- .../app/schemas/google_auth_credentials.py | 7 +- .../app/services/connector_service.py | 135 ++++++ .../app/tasks/connectors_indexing_tasks.py | 361 ++++++++++++++ surfsense_backend/pyproject.toml | 2 + surfsense_backend/uv.lock | 58 +++ .../add/google-calendar-connector/page.tsx | 452 ++++++++++++++++++ .../[search_space_id]/connectors/add/page.tsx | 9 + .../documents/(manage)/page.tsx | 2 + .../components/chat/ChatInputGroup.tsx | 2 +- .../components/chat/ConnectorComponents.tsx | 3 + .../components/editConnector/types.ts | 4 + 21 files changed, 1213 insertions(+), 19 deletions(-) create mode 100644 surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py create mode 100644 surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx diff --git a/surfsense_backend/.env.example b/surfsense_backend/.env.example index 0b4e2cf..58e3f73 100644 --- a/surfsense_backend/.env.example +++ b/surfsense_backend/.env.example @@ -8,6 +8,7 @@ AUTH_TYPE=GOOGLE or LOCAL # For Google Auth Only GOOGLE_OAUTH_CLIENT_ID=924507538m GOOGLE_OAUTH_CLIENT_SECRET=GOCSV +GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback # Embedding Model EMBEDDING_MODEL=mixedbread-ai/mxbai-embed-large-v1 diff --git a/surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py b/surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py new file mode 100644 index 0000000..c4843ac --- /dev/null +++ b/surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py @@ -0,0 +1,65 @@ +"""Add Google Calendar connector enums + +Revision ID: 15 +Revises: 14 +Create Date: 2024-02-01 12:00:00.000000 + +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "15" +down_revision: str | None = "14" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Safely add 'GOOGLE_CALENDAR_CONNECTOR' to enum types if missing.""" + + # Add to searchsourceconnectortype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'searchsourceconnectortype' AND e.enumlabel = 'GOOGLE_CALENDAR_CONNECTOR' + ) THEN + ALTER TYPE searchsourceconnectortype ADD VALUE 'GOOGLE_CALENDAR_CONNECTOR'; + END IF; + END + $$; + """ + ) + + # Add to documenttype enum + op.execute( + """ + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_type t + JOIN pg_enum e ON t.oid = e.enumtypid + WHERE t.typname = 'documenttype' AND e.enumlabel = 'GOOGLE_CALENDAR_CONNECTOR' + ) THEN + ALTER TYPE documenttype ADD VALUE 'GOOGLE_CALENDAR_CONNECTOR'; + END IF; + END + $$; + """ + ) + + +def downgrade() -> None: + """Remove 'GOOGLE_CALENDAR_CONNECTOR' from enum types.""" + + # Note: PostgreSQL doesn't support removing enum values directly + # This would require recreating the enum type, which is complex + # For now, we'll leave the enum values in place + # In a production environment, you might want to implement a more sophisticated downgrade + pass diff --git a/surfsense_backend/app/agents/researcher/nodes.py b/surfsense_backend/app/agents/researcher/nodes.py index 7882fed..6e27aa3 100644 --- a/surfsense_backend/app/agents/researcher/nodes.py +++ b/surfsense_backend/app/agents/researcher/nodes.py @@ -271,6 +271,49 @@ async def fetch_documents_by_ids( else: url = "" + elif doc_type == "GOOGLE_CALENDAR_CONNECTOR": + # Extract Google Calendar-specific metadata + event_id = metadata.get("event_id", "Unknown Event") + event_summary = metadata.get("event_summary", "Untitled Event") + calendar_id = metadata.get("calendar_id", "") + start_time = metadata.get("start_time", "") + location = metadata.get("location", "") + + title = f"Calendar: {event_summary}" + if start_time: + # Format the start time for display + try: + if "T" in start_time: + from datetime import datetime + + start_dt = datetime.fromisoformat( + start_time.replace("Z", "+00:00") + ) + formatted_time = start_dt.strftime("%Y-%m-%d %H:%M") + title += f" ({formatted_time})" + else: + title += f" ({start_time})" + except Exception: + title += f" ({start_time})" + + description = ( + doc.content[:100] + "..." + if len(doc.content) > 100 + else doc.content + ) + if location: + description += f" | Location: {location}" + if calendar_id and calendar_id != "primary": + description += f" | Calendar: {calendar_id}" + + # Construct Google Calendar URL + if event_id: + url = ( + f"https://calendar.google.com/calendar/event?eid={event_id}" + ) + else: + url = "" + elif doc_type == "EXTENSION": # Extract Extension-specific metadata webpage_title = metadata.get("VisitedWebPageTitle", doc.title) @@ -919,6 +962,32 @@ async def fetch_relevant_documents( ) } ) + elif connector == "GOOGLE_CALENDAR_CONNECTOR": + ( + source_object, + calendar_chunks, + ) = await connector_service.search_google_calendar( + user_query=reformulated_query, + user_id=user_id, + search_space_id=search_space_id, + top_k=top_k, + search_mode=search_mode, + ) + + # Add to sources and raw documents + if source_object: + all_sources.append(source_object) + all_raw_documents.extend(calendar_chunks) + + # Stream found document count + if streaming_service and writer: + writer( + { + "yield_value": streaming_service.format_terminal_info_delta( + f"📅 Found {len(calendar_chunks)} calendar events related to your query" + ) + } + ) elif connector == "CONFLUENCE_CONNECTOR": ( source_object, diff --git a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py index 608c165..a7554aa 100644 --- a/surfsense_backend/app/agents/researcher/qna_agent/prompts.py +++ b/surfsense_backend/app/agents/researcher/qna_agent/prompts.py @@ -18,6 +18,7 @@ You are SurfSense, an advanced AI research assistant that provides detailed, wel - JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking) - CONFLUENCE_CONNECTOR: "Confluence pages and comments" (personal project documentation) - CLICKUP_CONNECTOR: "ClickUp tasks and project data" (personal task management) +- GOOGLE_CALENDAR_CONNECTOR: "Google Calendar events, meetings, and schedules" (personal calendar and time management) - DISCORD_CONNECTOR: "Discord server conversations and shared content" (personal community communications) - TAVILY_API: "Tavily search API results" (personalized search results) - LINKUP_API: "Linkup search API results" (personalized search results) diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index 83125e6..cb57ea1 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -15,6 +15,9 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio - YOUTUBE_VIDEO: "YouTube video transcripts and metadata" (personally saved videos) - GITHUB_CONNECTOR: "GitHub repository content and issues" (personal repositories and interactions) - LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management) +- JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking) +- CONFLUENCE_CONNECTOR: "Confluence pages and comments" (personal project documentation) +- GOOGLE_CALENDAR_CONNECTOR: "Google Calendar events, meetings, and schedules" (personal calendar and time management) - DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions) - TAVILY_API: "Tavily search API results" (personalized search results) - LINKUP_API: "Linkup search API results" (personalized search results) diff --git a/surfsense_backend/app/agents/researcher/utils.py b/surfsense_backend/app/agents/researcher/utils.py index e26788c..b2d3dcd 100644 --- a/surfsense_backend/app/agents/researcher/utils.py +++ b/surfsense_backend/app/agents/researcher/utils.py @@ -63,6 +63,8 @@ def get_connector_friendly_name(connector_name: str) -> str: "GITHUB_CONNECTOR": "GitHub", "LINEAR_CONNECTOR": "Linear", "JIRA_CONNECTOR": "Jira", + "CONFLUENCE_CONNECTOR": "Confluence", + "GOOGLE_CALENDAR_CONNECTOR": "Google Calendar", "DISCORD_CONNECTOR": "Discord", "TAVILY_API": "Tavily Search", "LINKUP_API": "Linkup Search", diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index 48de86d..b71b113 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -43,9 +43,10 @@ class Config: # AUTH: Google OAuth AUTH_TYPE = os.getenv("AUTH_TYPE") - if AUTH_TYPE == "GOOGLE": - GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID") - GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET") + + GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID") + GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET") + GOOGLE_CALENDAR_REDIRECT_URI = os.getenv("GOOGLE_CALENDAR_REDIRECT_URI") # LLM instances are now managed per-user through the LLMConfig system # Legacy environment variables removed in favor of user-specific configurations diff --git a/surfsense_backend/app/connectors/google_calendar_connector.py b/surfsense_backend/app/connectors/google_calendar_connector.py index d481491..3d7ca2e 100644 --- a/surfsense_backend/app/connectors/google_calendar_connector.py +++ b/surfsense_backend/app/connectors/google_calendar_connector.py @@ -1,6 +1,5 @@ """ Google Calendar Connector Module | Google OAuth Credentials | Google Calendar API - A module for retrieving calendar events from Google Calendar using Google OAuth credentials. Allows fetching events from specified calendars within date ranges using Google OAuth credentials. """ @@ -8,6 +7,8 @@ Allows fetching events from specified calendars within date ranges using Google from datetime import datetime from typing import Any +import pytz +from dateutil.parser import isoparse from google.auth.transport.requests import Request from google.oauth2.credentials import Credentials from googleapiclient.discovery import build @@ -22,7 +23,6 @@ class GoogleCalendarConnector: ): """ Initialize the GoogleCalendarConnector class. - Args: credentials: Google OAuth Credentials object """ @@ -32,10 +32,8 @@ class GoogleCalendarConnector: def _get_credentials(self) -> Credentials: """ Get valid Google OAuth credentials. - Returns: Google OAuth credentials - Raises: ValueError: If credentials have not been set Exception: If credential refresh fails @@ -78,10 +76,8 @@ class GoogleCalendarConnector: def _get_service(self): """ Get the Google Calendar service instance using Google OAuth credentials. - Returns: Google Calendar service instance - Raises: ValueError: If credentials have not been set Exception: If service creation fails @@ -99,7 +95,6 @@ class GoogleCalendarConnector: def get_calendars(self) -> tuple[list[dict[str, Any]], str | None]: """ Fetch list of user's calendars using Google OAuth credentials. - Returns: Tuple containing (calendars list, error message or None) """ @@ -129,20 +124,44 @@ class GoogleCalendarConnector: def get_all_primary_calendar_events( self, + start_date: str, + end_date: str, max_results: int = 2500, ) -> tuple[list[dict[str, Any]], str | None]: """ Fetch events from the primary calendar using Google OAuth credentials. - Args: max_results: Maximum number of events to fetch (default: 2500) - Returns: Tuple containing (events list, error message or None) """ try: service = self._get_service() + # Parse both dates + dt_start = isoparse(start_date) + dt_end = isoparse(end_date) + + if dt_start.tzinfo is None: + dt_start = dt_start.replace(tzinfo=pytz.UTC) + else: + dt_start = dt_start.astimezone(pytz.UTC) + + if dt_end.tzinfo is None: + dt_end = dt_end.replace(tzinfo=pytz.UTC) + else: + dt_end = dt_end.astimezone(pytz.UTC) + + if dt_start >= dt_end: + return [], ( + f"start_date ({dt_start.isoformat()}) must be strictly before " + f"end_date ({dt_end.isoformat()})." + ) + + # RFC3339 with 'Z' for UTC + time_min = dt_start.isoformat().replace("+00:00", "Z") + time_max = dt_end.isoformat().replace("+00:00", "Z") + # Fetch events events_result = ( service.events() @@ -151,6 +170,8 @@ class GoogleCalendarConnector: maxResults=max_results, singleEvents=True, orderBy="startTime", + timeMin=time_min, + timeMax=time_max, ) .execute() ) @@ -168,10 +189,8 @@ class GoogleCalendarConnector: def format_event_to_markdown(self, event: dict[str, Any]) -> str: """ Format a Google Calendar event to markdown. - Args: event: Event object from Google Calendar API - Returns: Formatted markdown string """ diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 33a4057..59cd6df 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -257,7 +257,7 @@ class GoogleCalendarAccount(BaseModel): ) access_token = Column(String, nullable=False) - refresh_token = Column(String, nullable=False) + refresh_token = Column(String, nullable=True) user = relationship("User", back_populates="calendar_account") diff --git a/surfsense_backend/app/routes/__init__.py b/surfsense_backend/app/routes/__init__.py index 00b8f63..3e9c6ba 100644 --- a/surfsense_backend/app/routes/__init__.py +++ b/surfsense_backend/app/routes/__init__.py @@ -2,7 +2,9 @@ from fastapi import APIRouter from .chats_routes import router as chats_router from .documents_routes import router as documents_router -from .google_calendar_add_connector_route import router as google_oauth_router +from .google_calendar_add_connector_route import ( + router as google_calendar_add_connector_router, +) from .llm_config_routes import router as llm_config_router from .logs_routes import router as logs_router from .podcasts_routes import router as podcasts_router @@ -16,6 +18,6 @@ router.include_router(documents_router) router.include_router(podcasts_router) router.include_router(chats_router) router.include_router(search_source_connectors_router) -router.include_router(google_oauth_router) +router.include_router(google_calendar_add_connector_router) router.include_router(llm_config_router) router.include_router(logs_router) diff --git a/surfsense_backend/app/schemas/google_auth_credentials.py b/surfsense_backend/app/schemas/google_auth_credentials.py index 2d5a048..16e112e 100644 --- a/surfsense_backend/app/schemas/google_auth_credentials.py +++ b/surfsense_backend/app/schemas/google_auth_credentials.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import UTC, datetime from pydantic import BaseModel @@ -11,3 +11,8 @@ class GoogleAuthCredentialsBase(BaseModel): expiry: datetime scopes: list[str] client_secret: str + + @property + def is_expired(self) -> bool: + """Check if the credentials have expired.""" + return self.expiry <= datetime.now(UTC) diff --git a/surfsense_backend/app/services/connector_service.py b/surfsense_backend/app/services/connector_service.py index 3b3cce7..d063a86 100644 --- a/surfsense_backend/app/services/connector_service.py +++ b/surfsense_backend/app/services/connector_service.py @@ -1073,6 +1073,141 @@ class ConnectorService: return result_object, jira_chunks + async def search_google_calendar( + self, + user_query: str, + user_id: str, + search_space_id: int, + top_k: int = 20, + search_mode: SearchMode = SearchMode.CHUNKS, + ) -> tuple: + """ + Search for Google Calendar events and return both the source information and langchain documents + + Args: + user_query: The user's query + user_id: The user's ID + search_space_id: The search space ID to search in + top_k: Maximum number of results to return + search_mode: Search mode (CHUNKS or DOCUMENTS) + + Returns: + tuple: (sources_info, langchain_documents) + """ + if search_mode == SearchMode.CHUNKS: + calendar_chunks = await self.chunk_retriever.hybrid_search( + query_text=user_query, + top_k=top_k, + user_id=user_id, + search_space_id=search_space_id, + document_type="GOOGLE_CALENDAR_CONNECTOR", + ) + elif search_mode == SearchMode.DOCUMENTS: + calendar_chunks = await self.document_retriever.hybrid_search( + query_text=user_query, + top_k=top_k, + user_id=user_id, + search_space_id=search_space_id, + document_type="GOOGLE_CALENDAR_CONNECTOR", + ) + # Transform document retriever results to match expected format + calendar_chunks = self._transform_document_results(calendar_chunks) + + # Early return if no results + if not calendar_chunks: + return { + "id": 31, + "name": "Google Calendar Events", + "type": "GOOGLE_CALENDAR_CONNECTOR", + "sources": [], + }, [] + + # Process each chunk and create sources directly without deduplication + sources_list = [] + async with self.counter_lock: + for _i, chunk in enumerate(calendar_chunks): + # Extract document metadata + document = chunk.get("document", {}) + metadata = document.get("metadata", {}) + + # Extract Google Calendar-specific metadata + event_id = metadata.get("event_id", "") + event_summary = metadata.get("event_summary", "Untitled Event") + calendar_id = metadata.get("calendar_id", "") + start_time = metadata.get("start_time", "") + end_time = metadata.get("end_time", "") + location = metadata.get("location", "") + + # Create a more descriptive title for calendar events + title = f"Calendar: {event_summary}" + if start_time: + # Format the start time for display + try: + if "T" in start_time: + from datetime import datetime + + start_dt = datetime.fromisoformat( + start_time.replace("Z", "+00:00") + ) + formatted_time = start_dt.strftime("%Y-%m-%d %H:%M") + title += f" ({formatted_time})" + else: + title += f" ({start_time})" + except Exception: + title += f" ({start_time})" + + # Create a more descriptive description for calendar events + description = chunk.get("content", "")[:100] + if len(description) == 100: + description += "..." + + # Add event info to description + info_parts = [] + if location: + info_parts.append(f"Location: {location}") + if calendar_id and calendar_id != "primary": + info_parts.append(f"Calendar: {calendar_id}") + if end_time: + info_parts.append(f"End: {end_time}") + + if info_parts: + if description: + description += f" | {' | '.join(info_parts)}" + else: + description = " | ".join(info_parts) + + # For URL, we could construct a URL to the Google Calendar event + url = "" + if event_id and calendar_id: + # Google Calendar event URL format + url = f"https://calendar.google.com/calendar/event?eid={event_id}" + + source = { + "id": document.get("id", self.source_id_counter), + "title": title, + "description": description, + "url": url, + "event_id": event_id, + "event_summary": event_summary, + "calendar_id": calendar_id, + "start_time": start_time, + "end_time": end_time, + "location": location, + } + + self.source_id_counter += 1 + sources_list.append(source) + + # Create result object + result_object = { + "id": 31, # Assign a unique ID for the Google Calendar connector + "name": "Google Calendar Events", + "type": "GOOGLE_CALENDAR_CONNECTOR", + "sources": sources_list, + } + + return result_object, calendar_chunks + async def search_confluence( self, user_query: str, diff --git a/surfsense_backend/app/tasks/connectors_indexing_tasks.py b/surfsense_backend/app/tasks/connectors_indexing_tasks.py index 5f257fb..72bc18c 100644 --- a/surfsense_backend/app/tasks/connectors_indexing_tasks.py +++ b/surfsense_backend/app/tasks/connectors_indexing_tasks.py @@ -2,6 +2,7 @@ import asyncio import logging from datetime import UTC, datetime, timedelta +from google.oauth2.credentials import Credentials from slack_sdk.errors import SlackApiError from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.ext.asyncio import AsyncSession @@ -12,6 +13,7 @@ from app.connectors.clickup_connector import ClickUpConnector from app.connectors.confluence_connector import ConfluenceConnector from app.connectors.discord_connector import DiscordConnector from app.connectors.github_connector import GitHubConnector +from app.connectors.google_calendar_connector import GoogleCalendarConnector from app.connectors.jira_connector import JiraConnector from app.connectors.linear_connector import LinearConnector from app.connectors.notion_history import NotionHistoryConnector @@ -3012,3 +3014,362 @@ async def index_clickup_tasks( ) logger.error(f"Failed to index ClickUp tasks: {e!s}", exc_info=True) return 0, f"Failed to index ClickUp tasks: {e!s}" + + +async def index_google_calendar_events( + session: AsyncSession, + connector_id: int, + search_space_id: int, + user_id: str, + start_date: str | None = None, + end_date: str | None = None, + update_last_indexed: bool = True, +) -> tuple[int, str | None]: + """ + Index Google Calendar events. + + Args: + session: Database session + connector_id: ID of the Google Calendar connector + search_space_id: ID of the search space to store documents in + user_id: User ID + start_date: Start date for indexing (YYYY-MM-DD format) + end_date: End date for indexing (YYYY-MM-DD format) + update_last_indexed: Whether to update the last_indexed_at timestamp (default: True) + + Returns: + Tuple containing (number of documents indexed, error message or None) + """ + task_logger = TaskLoggingService(session, search_space_id) + + # Log task start + log_entry = await task_logger.log_task_start( + task_name="google_calendar_events_indexing", + source="connector_indexing_task", + message=f"Starting Google Calendar events indexing for connector {connector_id}", + metadata={ + "connector_id": connector_id, + "user_id": str(user_id), + "start_date": start_date, + "end_date": end_date, + }, + ) + + try: + # Get the connector from the database + result = await session.execute( + select(SearchSourceConnector).filter( + SearchSourceConnector.id == connector_id, + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, + ) + ) + connector = result.scalars().first() + + if not connector: + await task_logger.log_task_failure( + log_entry, + f"Connector with ID {connector_id} not found", + "Connector not found", + {"error_type": "ConnectorNotFound"}, + ) + return 0, f"Connector with ID {connector_id} not found" + + # Get the Google Calendar credentials from the connector config + credentials = Credentials( + token=connector.config.get("token"), + refresh_token=connector.config.get("refresh_token"), + token_uri=connector.config.get("token_uri"), + client_id=connector.config.get("client_id"), + client_secret=connector.config.get("client_secret"), + scopes=connector.config.get("scopes"), + ) + + if ( + not credentials.client_id + or not credentials.client_secret + or not credentials.refresh_token + ): + await task_logger.log_task_failure( + log_entry, + f"Google Calendar credentials not found in connector config for connector {connector_id}", + "Missing Google Calendar credentials", + {"error_type": "MissingCredentials"}, + ) + return 0, "Google Calendar credentials not found in connector config" + + # Initialize Google Calendar client + await task_logger.log_task_progress( + log_entry, + f"Initializing Google Calendar client for connector {connector_id}", + {"stage": "client_initialization"}, + ) + + calendar_client = GoogleCalendarConnector(credentials=credentials) + + # Calculate date range + if start_date is None or end_date is None: + # Fall back to calculating dates based on last_indexed_at + calculated_end_date = datetime.now() + + # Use last_indexed_at as start date if available, otherwise use 30 days ago + if connector.last_indexed_at: + # Convert dates to be comparable (both timezone-naive) + last_indexed_naive = ( + connector.last_indexed_at.replace(tzinfo=None) + if connector.last_indexed_at.tzinfo + else connector.last_indexed_at + ) + + # Check if last_indexed_at is in the future or after end_date + if last_indexed_naive > calculated_end_date: + logger.warning( + f"Last indexed date ({last_indexed_naive.strftime('%Y-%m-%d')}) is in the future. Using 30 days ago instead." + ) + calculated_start_date = calculated_end_date - timedelta(days=30) + else: + calculated_start_date = last_indexed_naive + logger.info( + f"Using last_indexed_at ({calculated_start_date.strftime('%Y-%m-%d')}) as start date" + ) + else: + calculated_start_date = calculated_end_date - timedelta( + days=30 + ) # Use 30 days as default for calendar events + logger.info( + f"No last_indexed_at found, using {calculated_start_date.strftime('%Y-%m-%d')} (30 days ago) as start date" + ) + + # Use calculated dates if not provided + start_date_str = ( + start_date if start_date else calculated_start_date.strftime("%Y-%m-%d") + ) + end_date_str = ( + end_date if end_date else calculated_end_date.strftime("%Y-%m-%d") + ) + else: + # Use provided dates + start_date_str = start_date + end_date_str = end_date + + await task_logger.log_task_progress( + log_entry, + f"Fetching Google Calendar events from {start_date_str} to {end_date_str}", + { + "stage": "fetching_events", + "start_date": start_date_str, + "end_date": end_date_str, + }, + ) + + # Get events within date range from primary calendar + try: + events, error = calendar_client.get_all_primary_calendar_events( + start_date=start_date_str, end_date=end_date_str + ) + + if error: + logger.error(f"Failed to get Google Calendar events: {error}") + + # Don't treat "No events found" as an error that should stop indexing + if "No events found" in error: + logger.info( + "No events found is not a critical error, continuing with update" + ) + if update_last_indexed: + connector.last_indexed_at = datetime.now() + await session.commit() + logger.info( + f"Updated last_indexed_at to {connector.last_indexed_at} despite no events found" + ) + + await task_logger.log_task_success( + log_entry, + f"No Google Calendar events found in date range {start_date_str} to {end_date_str}", + {"events_found": 0}, + ) + return 0, None + else: + await task_logger.log_task_failure( + log_entry, + f"Failed to get Google Calendar events: {error}", + "API Error", + {"error_type": "APIError"}, + ) + return 0, f"Failed to get Google Calendar events: {error}" + + logger.info(f"Retrieved {len(events)} events from Google Calendar API") + + except Exception as e: + logger.error(f"Error fetching Google Calendar events: {e!s}", exc_info=True) + return 0, f"Error fetching Google Calendar events: {e!s}" + + # Process and index each event + documents_indexed = 0 + skipped_events = [] + documents_skipped = 0 + + for event in events: + try: + event_id = event.get("id") + event_summary = event.get("summary", "No Title") + calendar_id = event.get("calendarId", "") + + if not event_id: + logger.warning(f"Skipping event with missing ID: {event_summary}") + skipped_events.append(f"{event_summary} (missing ID)") + documents_skipped += 1 + continue + + # Format event as markdown + event_markdown = calendar_client.format_event_to_markdown(event) + + if not event_markdown.strip(): + logger.warning(f"Skipping event with no content: {event_summary}") + skipped_events.append(f"{event_summary} (no content)") + documents_skipped += 1 + continue + + # Create a simple summary for the document + start = event.get("start", {}) + end = event.get("end", {}) + start_time = start.get("dateTime") or start.get("date", "") + end_time = end.get("dateTime") or end.get("date", "") + location = event.get("location", "") + description = event.get("description", "") + + summary_content = f"Google Calendar Event: {event_summary}\n\n" + summary_content += f"Calendar: {calendar_id}\n" + summary_content += f"Start: {start_time}\n" + summary_content += f"End: {end_time}\n" + + if location: + summary_content += f"Location: {location}\n" + + if description: + # Take first 300 characters of description for summary + desc_preview = description[:300] + if len(description) > 300: + desc_preview += "..." + summary_content += f"Description: {desc_preview}\n" + + # Generate content hash + content_hash = generate_content_hash(event_markdown, search_space_id) + + # Check if document already exists + existing_doc_by_hash_result = await session.execute( + select(Document).where(Document.content_hash == content_hash) + ) + existing_document_by_hash = ( + existing_doc_by_hash_result.scalars().first() + ) + + if existing_document_by_hash: + logger.info( + f"Document with content hash {content_hash} already exists for event {event_summary}. Skipping processing." + ) + documents_skipped += 1 + continue + + # Generate embedding for the summary + summary_embedding = config.embedding_model_instance.embed( + summary_content + ) + + # Process chunks - using the full event markdown + chunks = [ + Chunk( + content=chunk.text, + embedding=config.embedding_model_instance.embed(chunk.text), + ) + for chunk in config.chunker_instance.chunk(event_markdown) + ] + + # Create and store new document + logger.info(f"Creating new document for event {event_summary}") + document = Document( + search_space_id=search_space_id, + title=f"Calendar Event - {event_summary}", + document_type=DocumentType.GOOGLE_CALENDAR_CONNECTOR, + document_metadata={ + "event_id": event_id, + "event_summary": event_summary, + "calendar_id": calendar_id, + "start_time": start_time, + "end_time": end_time, + "location": location, + "indexed_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + }, + content=summary_content, + content_hash=content_hash, + embedding=summary_embedding, + chunks=chunks, + ) + + session.add(document) + documents_indexed += 1 + logger.info(f"Successfully indexed new event {event_summary}") + + except Exception as e: + logger.error( + f"Error processing event {event.get('summary', 'Unknown')}: {e!s}", + exc_info=True, + ) + skipped_events.append( + f"{event.get('summary', 'Unknown')} (processing error)" + ) + documents_skipped += 1 + continue # Skip this event and continue with others + + # Update the last_indexed_at timestamp for the connector only if requested + total_processed = documents_indexed + if update_last_indexed: + connector.last_indexed_at = datetime.now() + logger.info(f"Updated last_indexed_at to {connector.last_indexed_at}") + + # Commit all changes + await session.commit() + logger.info( + "Successfully committed all Google Calendar document changes to database" + ) + + # Log success + await task_logger.log_task_success( + log_entry, + f"Successfully completed Google Calendar indexing for connector {connector_id}", + { + "events_processed": total_processed, + "documents_indexed": documents_indexed, + "documents_skipped": documents_skipped, + "skipped_events_count": len(skipped_events), + }, + ) + + logger.info( + f"Google Calendar indexing completed: {documents_indexed} new events, {documents_skipped} skipped" + ) + return ( + total_processed, + None, + ) # Return None as the error message to indicate success + + except SQLAlchemyError as db_error: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Database error during Google Calendar indexing for connector {connector_id}", + str(db_error), + {"error_type": "SQLAlchemyError"}, + ) + logger.error(f"Database error: {db_error!s}", exc_info=True) + return 0, f"Database error: {db_error!s}" + except Exception as e: + await session.rollback() + await task_logger.log_task_failure( + log_entry, + f"Failed to index Google Calendar events for connector {connector_id}", + str(e), + {"error_type": type(e).__name__}, + ) + logger.error(f"Failed to index Google Calendar events: {e!s}", exc_info=True) + return 0, f"Failed to index Google Calendar events: {e!s}" diff --git a/surfsense_backend/pyproject.toml b/surfsense_backend/pyproject.toml index 9ab56b7..3d1701e 100644 --- a/surfsense_backend/pyproject.toml +++ b/surfsense_backend/pyproject.toml @@ -14,6 +14,8 @@ dependencies = [ "fastapi-users[oauth,sqlalchemy]>=14.0.1", "firecrawl-py>=1.12.0", "github3.py==4.0.1", + "google-api-python-client>=2.156.0", + "google-auth-oauthlib>=1.2.1", "langchain-community>=0.3.17", "langchain-unstructured>=0.1.6", "langgraph>=0.3.29", diff --git a/surfsense_backend/uv.lock b/surfsense_backend/uv.lock index f3bc42c..3386961 100644 --- a/surfsense_backend/uv.lock +++ b/surfsense_backend/uv.lock @@ -1252,6 +1252,22 @@ grpc = [ { name = "grpcio-status" }, ] +[[package]] +name = "google-api-python-client" +version = "2.177.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core" }, + { name = "google-auth" }, + { name = "google-auth-httplib2" }, + { name = "httplib2" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/75/a89cad519fa8910132e3b08571d0e682ae1163643da6f963f1930f3dc788/google_api_python_client-2.177.0.tar.gz", hash = "sha256:9ffd2b57d68f5afa7e6ac64e2c440534eaa056cbb394812a62ff94723c31b50e", size = 13184405, upload-time = "2025-07-23T16:22:46.321Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/f5/121248e18ca605a11720c81ae1b52a5a8cb690af9f01887c56de23cd9a5a/google_api_python_client-2.177.0-py3-none-any.whl", hash = "sha256:f2f50f11105ab883eb9b6cf38ec54ea5fd4b429249f76444bec90deba5be79b3", size = 13709470, upload-time = "2025-07-23T16:22:44.081Z" }, +] + [[package]] name = "google-auth" version = "2.40.3" @@ -1266,6 +1282,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/17/63/b19553b658a1692443c62bd07e5868adaa0ad746a0751ba62c59568cd45b/google_auth-2.40.3-py2.py3-none-any.whl", hash = "sha256:1370d4593e86213563547f97a92752fc658456fe4514c809544f330fed45a7ca", size = 216137, upload-time = "2025-06-04T18:04:55.573Z" }, ] +[[package]] +name = "google-auth-httplib2" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "httplib2" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/56/be/217a598a818567b28e859ff087f347475c807a5649296fb5a817c58dacef/google-auth-httplib2-0.2.0.tar.gz", hash = "sha256:38aa7badf48f974f1eb9861794e9c0cb2a0511a4ec0679b1f886d108f5640e05", size = 10842, upload-time = "2023-12-12T17:40:30.722Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/8a/fe34d2f3f9470a27b01c9e76226965863f153d5fbe276f83608562e49c04/google_auth_httplib2-0.2.0-py2.py3-none-any.whl", hash = "sha256:b65a0a2123300dd71281a7bf6e64d65a0759287df52729bdd1ae2e47dc311a3d", size = 9253, upload-time = "2023-12-12T17:40:13.055Z" }, +] + +[[package]] +name = "google-auth-oauthlib" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "requests-oauthlib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fb/87/e10bf24f7bcffc1421b84d6f9c3377c30ec305d082cd737ddaa6d8f77f7c/google_auth_oauthlib-1.2.2.tar.gz", hash = "sha256:11046fb8d3348b296302dd939ace8af0a724042e8029c1b872d87fabc9f41684", size = 20955, upload-time = "2025-04-22T16:40:29.172Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/84/40ee070be95771acd2f4418981edb834979424565c3eec3cd88b6aa09d24/google_auth_oauthlib-1.2.2-py3-none-any.whl", hash = "sha256:fd619506f4b3908b5df17b65f39ca8d66ea56986e5472eb5978fd8f3786f00a2", size = 19072, upload-time = "2025-04-22T16:40:28.174Z" }, +] + [[package]] name = "google-cloud-vision" version = "3.10.2" @@ -1470,6 +1512,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httplib2" +version = "0.22.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81", size = 351116, upload-time = "2023-03-21T22:29:37.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc", size = 96854, upload-time = "2023-03-21T22:29:35.683Z" }, +] + [[package]] name = "httptools" version = "0.6.4" @@ -4724,6 +4778,8 @@ dependencies = [ { name = "fastapi-users", extra = ["oauth", "sqlalchemy"] }, { name = "firecrawl-py" }, { name = "github3-py" }, + { name = "google-api-python-client" }, + { name = "google-auth-oauthlib" }, { name = "langchain-community" }, { name = "langchain-unstructured" }, { name = "langgraph" }, @@ -4763,6 +4819,8 @@ requires-dist = [ { name = "fastapi-users", extras = ["oauth", "sqlalchemy"], specifier = ">=14.0.1" }, { name = "firecrawl-py", specifier = ">=1.12.0" }, { name = "github3-py", specifier = "==4.0.1" }, + { name = "google-api-python-client", specifier = ">=2.156.0" }, + { name = "google-auth-oauthlib", specifier = ">=1.2.1" }, { name = "langchain-community", specifier = ">=0.3.17" }, { name = "langchain-unstructured", specifier = ">=0.1.6" }, { name = "langgraph", specifier = ">=0.3.29" }, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx new file mode 100644 index 0000000..64c32ef --- /dev/null +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -0,0 +1,452 @@ +"use client"; + +import { zodResolver } from "@hookform/resolvers/zod"; +import { IconCalendar } from "@tabler/icons-react"; +import { motion } from "framer-motion"; +import { ArrowLeft, Check, ExternalLink, Loader2 } from "lucide-react"; +import Link from "next/link"; +import { useParams, useRouter, useSearchParams } from "next/navigation"; +import { useEffect, useState } from "react"; +import { useForm } from "react-hook-form"; +import { toast } from "sonner"; +import { z } from "zod"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardFooter, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Form, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, +} from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; + +// Define the form schema with Zod +const googleCalendarConnectorFormSchema = z.object({ + name: z.string().min(3, { + message: "Connector name must be at least 3 characters.", + }), + calendar_ids: z.array(z.string()).min(1, { + message: "At least one calendar must be selected.", + }), +}); + +// Define the type for the form values +type GoogleCalendarConnectorFormValues = z.infer; + +// Interface for calendar data +interface Calendar { + id: string; + summary: string; + description?: string; + primary?: boolean; + access_role: string; + time_zone?: string; +} + +// Interface for OAuth credentials +interface OAuthCredentials { + client_id: string; + client_secret: string; + refresh_token: string; + access_token: string; +} + +export default function GoogleCalendarConnectorPage() { + const router = useRouter(); + const params = useParams(); + const searchParams = useSearchParams(); + const searchSpaceId = params.search_space_id as string; + const isSuccess = searchParams.get("success") === "true"; + + const { createConnector } = useSearchSourceConnectors(); + const [isSubmitting, setIsSubmitting] = useState(false); + const [isConnecting, setIsConnecting] = useState(false); + const [isConnected, setIsConnected] = useState(false); + const [calendars, setCalendars] = useState([]); + const [credentials, setCredentials] = useState(null); + + // Initialize the form + const form = useForm({ + resolver: zodResolver(googleCalendarConnectorFormSchema), + defaultValues: { + name: "", + calendar_ids: [], + }, + }); + + useEffect(() => { + if (isSuccess) { + toast.success("Google Calendar connector created successfully!"); + } + }, [isSuccess]); + + // Check for OAuth callback parameters + useEffect(() => { + const success = searchParams.get("success"); + const error = searchParams.get("error"); + const message = searchParams.get("message"); + const sessionKey = searchParams.get("session_key"); + + if (success === "true" && sessionKey) { + // Fetch OAuth data from backend + fetchOAuthData(sessionKey); + } else if (error) { + toast.error(message || "Failed to connect to Google Calendar"); + } + }, [searchParams]); + + // Fetch OAuth data from backend + const fetchOAuthData = async (sessionKey: string) => { + try { + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/auth/google/session?session_key=${sessionKey}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, + }, + } + ); + + if (!response.ok) { + throw new Error("Failed to fetch OAuth data"); + } + + const data = await response.json(); + + setCredentials(data.credentials); + setCalendars(data.calendars); + setIsConnected(true); + toast.success("Successfully connected to Google Calendar!"); + } catch (error) { + console.error("Error fetching OAuth data:", error); + toast.error("Failed to retrieve Google Calendar data"); + } + }; + + // Handle Google OAuth connection + const handleConnectGoogle = async () => { + setIsConnecting(true); + try { + // Call backend to initiate OAuth flow + const response = await fetch( + `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/auth/google/calendar/connector/add/?space_id=${searchSpaceId}`, + { + method: "GET", + headers: { + Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, + }, + } + ); + + if (!response.ok) { + throw new Error("Failed to initiate Google OAuth"); + } + + const data = await response.json(); + + // Redirect to Google for authentication + window.location.href = data.auth_url; + } catch (error) { + console.error("Error connecting to Google:", error); + toast.error("Failed to connect to Google Calendar"); + setIsConnecting(false); + } + }; + + // Handle form submission + const onSubmit = async (values: GoogleCalendarConnectorFormValues) => { + if (!isConnected || !credentials) { + toast.error("Please connect your Google account first"); + return; + } + + if (values.calendar_ids.length === 0) { + toast.error("Please select at least one calendar"); + return; + } + + setIsSubmitting(true); + try { + await createConnector({ + name: values.name, + connector_type: "GOOGLE_CALENDAR_CONNECTOR", + config: { + GOOGLE_CALENDAR_CLIENT_ID: credentials.client_id, + GOOGLE_CALENDAR_CLIENT_SECRET: credentials.client_secret, + GOOGLE_CALENDAR_REFRESH_TOKEN: credentials.refresh_token, + GOOGLE_CALENDAR_CALENDAR_IDS: values.calendar_ids, + }, + is_indexable: true, + last_indexed_at: null, + }); + + toast.success("Google Calendar connector created successfully!"); + + // Navigate back to connectors page + router.push(`/dashboard/${searchSpaceId}/connectors`); + } catch (error) { + console.error("Error creating connector:", error); + toast.error(error instanceof Error ? error.message : "Failed to create connector"); + } finally { + setIsSubmitting(false); + } + }; + + return ( +
+ + {/* Header */} +
+ + + Back to connectors + +
+
+ +
+
+

Connect Google Calendar

+

+ Connect your Google Calendar to search events, meetings and schedules. +

+
+
+
+ + {/* OAuth Connection Card */} + {!isConnected ? ( + + + Connect Your Google Account + + Connect your Google account to access your calendar events. We'll only request + read-only access to your calendars. + + + +
+ + Read-only access to your calendar events +
+
+ + Access works even when you're offline +
+
+ + You can disconnect anytime +
+ + + + + + + ) : ( + /* Configuration Form Card */ + + + Configure Google Calendar Connector + + Your Google account is connected! Now select which calendars to include and give + your connector a name. + + +
+ + + {/* Connector Name */} + ( + + Connector Name + + + + + A friendly name to identify this connector. + + + + )} + /> + + {/* Calendar Selection */} + ( + +
+ Select Calendars + + Choose which calendars you want to include in your search results. + +
+ {calendars.map((calendar) => ( + { + return ( + + + { + return checked + ? field.onChange([...field.value, calendar.id]) + : field.onChange( + field.value?.filter((value) => value !== calendar.id) + ); + }} + /> + +
+ + {calendar.summary} + {calendar.primary && ( + + Primary + + )} + + {calendar.description && ( + + {calendar.description} + + )} +
+ + ); + }} + /> + ))} + + + )} + /> + + + + + +
+ + + )} + + {/* Help Section */} + + + How It Works + + +
+

1. Connect Your Account

+

+ Click "Connect Your Google Account" to start the secure OAuth process. You'll be + redirected to Google to sign in. +

+
+
+

2. Grant Permissions

+

+ Google will ask for permission to read your calendar events. We only request + read-only access to keep your data safe. +

+
+
+

3. Select Calendars

+

+ Choose which calendars you want to include in your search results. You can select + multiple calendars. +

+
+
+

4. Start Searching

+

+ Once connected, your calendar events will be indexed and searchable alongside your + other content. +

+
+ {isConnected && ( +
+

+ ✅ Your Google account is successfully connected! You can now configure your + connector above. +

+
+ )} + + + +
+ ); +} diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index 7f79bf8..2a7855d 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -9,6 +9,7 @@ import { IconBrandWindows, IconBrandZoom, IconChecklist, + IconCalendar, IconChevronDown, IconChevronRight, IconLayoutKanban, @@ -148,6 +149,13 @@ const connectorCategories: ConnectorCategory[] = [ id: "communication", title: "Communication", connectors: [ + { + id: "google-calendar-connector", + title: "Google Calendar", + description: "Connect to Google Calendar to search events, meetings and schedules.", + icon: , + status: "available", + }, { id: "gmail", title: "Gmail", @@ -212,6 +220,7 @@ export default function ConnectorsPage() { "knowledge-bases", "project-management", "team-chats", + "communication", ]); const toggleCategory = (categoryId: string) => { diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index b7eb2ec..5ba8cbd 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -8,6 +8,7 @@ import { IconBrandSlack, IconBrandYoutube, IconChecklist, + IconCalendar, IconLayoutKanban, IconTicket, } from "@tabler/icons-react"; @@ -148,6 +149,7 @@ const documentTypeIcons = { DISCORD_CONNECTOR: IconBrandDiscord, CONFLUENCE_CONNECTOR: IconBook, CLICKUP_CONNECTOR: IconChecklist, + GOOGLE_CALENDAR_CONNECTOR: IconCalendar, } as const; const columns: ColumnDef[] = [ diff --git a/surfsense_web/components/chat/ChatInputGroup.tsx b/surfsense_web/components/chat/ChatInputGroup.tsx index 80d42b0..6e54a8f 100644 --- a/surfsense_web/components/chat/ChatInputGroup.tsx +++ b/surfsense_web/components/chat/ChatInputGroup.tsx @@ -196,7 +196,7 @@ const ConnectorSelector = React.memo( type="button" > {getConnectorIcon(connector.type)} - {connector.name} + {connector.name} ); }) diff --git a/surfsense_web/components/chat/ConnectorComponents.tsx b/surfsense_web/components/chat/ConnectorComponents.tsx index f946b88..2a3cdb4 100644 --- a/surfsense_web/components/chat/ConnectorComponents.tsx +++ b/surfsense_web/components/chat/ConnectorComponents.tsx @@ -4,6 +4,7 @@ import { IconBrandNotion, IconBrandSlack, IconBrandYoutube, + IconCalendar, IconLayoutKanban, IconLinkPlus, IconTicket, @@ -56,6 +57,8 @@ export const getConnectorIcon = (connectorType: string) => { return ; case "JIRA_CONNECTOR": return ; + case "GOOGLE_CALENDAR_CONNECTOR": + return ; case "DEEP": return ; case "DEEPER": diff --git a/surfsense_web/components/editConnector/types.ts b/surfsense_web/components/editConnector/types.ts index 4523593..38ffc36 100644 --- a/surfsense_web/components/editConnector/types.ts +++ b/surfsense_web/components/editConnector/types.ts @@ -39,5 +39,9 @@ export const editConnectorSchema = z.object({ JIRA_BASE_URL: z.string().optional(), JIRA_EMAIL: z.string().optional(), JIRA_API_TOKEN: z.string().optional(), + GOOGLE_CALENDAR_CLIENT_ID: z.string().optional(), + GOOGLE_CALENDAR_CLIENT_SECRET: z.string().optional(), + GOOGLE_CALENDAR_REFRESH_TOKEN: z.string().optional(), + GOOGLE_CALENDAR_CALENDAR_IDS: z.string().optional(), }); export type EditConnectorFormValues = z.infer; From ad0a1e5c97fb53395b1b83d9582b812df6fc010a Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sat, 2 Aug 2025 05:36:43 +0200 Subject: [PATCH 6/9] check if a google calendar exixst before adding it , in the add page --- .../app/agents/researcher/utils.py | 1 + surfsense_backend/app/config/__init__.py | 5 +- .../google_calendar_add_connector_route.py | 21 +- .../add/google-calendar-connector/page.tsx | 340 ++---------------- .../[search_space_id]/connectors/add/page.tsx | 2 +- .../documents/(manage)/page.tsx | 2 +- .../hooks/useSearchSourceConnectors.ts | 2 + 7 files changed, 62 insertions(+), 311 deletions(-) diff --git a/surfsense_backend/app/agents/researcher/utils.py b/surfsense_backend/app/agents/researcher/utils.py index b2d3dcd..21969e6 100644 --- a/surfsense_backend/app/agents/researcher/utils.py +++ b/surfsense_backend/app/agents/researcher/utils.py @@ -47,6 +47,7 @@ def get_connector_emoji(connector_name: str) -> str: "DISCORD_CONNECTOR": "🗨️", "TAVILY_API": "🔍", "LINKUP_API": "🔗", + "GOOGLE_CALENDAR_CONNECTOR": "📅", } return connector_emojis.get(connector_name, "🔎") diff --git a/surfsense_backend/app/config/__init__.py b/surfsense_backend/app/config/__init__.py index b71b113..38a1f3e 100644 --- a/surfsense_backend/app/config/__init__.py +++ b/surfsense_backend/app/config/__init__.py @@ -41,11 +41,14 @@ class Config: NEXT_FRONTEND_URL = os.getenv("NEXT_FRONTEND_URL") - # AUTH: Google OAuth + # Auth AUTH_TYPE = os.getenv("AUTH_TYPE") + # Google OAuth GOOGLE_OAUTH_CLIENT_ID = os.getenv("GOOGLE_OAUTH_CLIENT_ID") GOOGLE_OAUTH_CLIENT_SECRET = os.getenv("GOOGLE_OAUTH_CLIENT_SECRET") + + # Google Calendar redirect URI GOOGLE_CALENDAR_REDIRECT_URI = os.getenv("GOOGLE_CALENDAR_REDIRECT_URI") # LLM instances are now managed per-user through the LLMConfig system diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index d78d6dd..fea69d1 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -1,22 +1,28 @@ # app/routes/google_calendar.py - import base64 import json -from sqlite3 import IntegrityError +import logging from uuid import UUID -from venv import logger from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import RedirectResponse from google_auth_oauthlib.flow import Flow -from jsonschema import ValidationError +from pydantic import ValidationError +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select from app.config import config -from app.db import SearchSourceConnector, User, get_async_session +from app.db import ( + SearchSourceConnector, + SearchSourceConnectorType, + User, + get_async_session, +) from app.users import current_active_user +logger = logging.getLogger(__name__) + router = APIRouter() SCOPES = ["https://www.googleapis.com/auth/calendar.readonly"] @@ -100,7 +106,8 @@ async def calendar_callback( result = await session.execute( select(SearchSourceConnector).filter( SearchSourceConnector.user_id == user_id, - SearchSourceConnector.connector_type == "GOOGLE_CALENDAR_CONNECTOR", + SearchSourceConnector.connector_type + == SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, ) ) existing_connector = result.scalars().first() @@ -111,7 +118,7 @@ async def calendar_callback( ) db_connector = SearchSourceConnector( name="Google Calendar Connector", - connector_type="GOOGLE_CALENDAR_CONNECTOR", + connector_type=SearchSourceConnectorType.GOOGLE_CALENDAR_CONNECTOR, config=creds_dict, user_id=user_id, is_indexable=True, diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx index 64c32ef..fb04a27 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/google-calendar-connector/page.tsx @@ -19,128 +19,36 @@ import { CardHeader, CardTitle, } from "@/components/ui/card"; -import { Checkbox } from "@/components/ui/checkbox"; import { - Form, - FormControl, - FormDescription, - FormField, - FormItem, - FormLabel, - FormMessage, -} from "@/components/ui/form"; -import { Input } from "@/components/ui/input"; -import { useSearchSourceConnectors } from "@/hooks/useSearchSourceConnectors"; - -// Define the form schema with Zod -const googleCalendarConnectorFormSchema = z.object({ - name: z.string().min(3, { - message: "Connector name must be at least 3 characters.", - }), - calendar_ids: z.array(z.string()).min(1, { - message: "At least one calendar must be selected.", - }), -}); - -// Define the type for the form values -type GoogleCalendarConnectorFormValues = z.infer; - -// Interface for calendar data -interface Calendar { - id: string; - summary: string; - description?: string; - primary?: boolean; - access_role: string; - time_zone?: string; -} - -// Interface for OAuth credentials -interface OAuthCredentials { - client_id: string; - client_secret: string; - refresh_token: string; - access_token: string; -} + type SearchSourceConnector, + useSearchSourceConnectors, +} from "@/hooks/useSearchSourceConnectors"; export default function GoogleCalendarConnectorPage() { const router = useRouter(); const params = useParams(); - const searchParams = useSearchParams(); const searchSpaceId = params.search_space_id as string; - const isSuccess = searchParams.get("success") === "true"; - - const { createConnector } = useSearchSourceConnectors(); - const [isSubmitting, setIsSubmitting] = useState(false); const [isConnecting, setIsConnecting] = useState(false); - const [isConnected, setIsConnected] = useState(false); - const [calendars, setCalendars] = useState([]); - const [credentials, setCredentials] = useState(null); + const [doesConnectorExist, setDoesConnectorExist] = useState(false); - // Initialize the form - const form = useForm({ - resolver: zodResolver(googleCalendarConnectorFormSchema), - defaultValues: { - name: "", - calendar_ids: [], - }, - }); + const { fetchConnectors } = useSearchSourceConnectors(); useEffect(() => { - if (isSuccess) { - toast.success("Google Calendar connector created successfully!"); - } - }, [isSuccess]); - - // Check for OAuth callback parameters - useEffect(() => { - const success = searchParams.get("success"); - const error = searchParams.get("error"); - const message = searchParams.get("message"); - const sessionKey = searchParams.get("session_key"); - - if (success === "true" && sessionKey) { - // Fetch OAuth data from backend - fetchOAuthData(sessionKey); - } else if (error) { - toast.error(message || "Failed to connect to Google Calendar"); - } - }, [searchParams]); - - // Fetch OAuth data from backend - const fetchOAuthData = async (sessionKey: string) => { - try { - const response = await fetch( - `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/auth/google/session?session_key=${sessionKey}`, - { - method: "GET", - headers: { - Authorization: `Bearer ${localStorage.getItem("surfsense_bearer_token")}`, - }, - } + fetchConnectors().then((data) => { + const connector = data.find( + (c: SearchSourceConnector) => c.connector_type === "GOOGLE_CALENDAR_CONNECTOR" ); - - if (!response.ok) { - throw new Error("Failed to fetch OAuth data"); + if (connector) { + setDoesConnectorExist(true); } - - const data = await response.json(); - - setCredentials(data.credentials); - setCalendars(data.calendars); - setIsConnected(true); - toast.success("Successfully connected to Google Calendar!"); - } catch (error) { - console.error("Error fetching OAuth data:", error); - toast.error("Failed to retrieve Google Calendar data"); - } - }; + }); + }, []); // Handle Google OAuth connection const handleConnectGoogle = async () => { - setIsConnecting(true); try { - // Call backend to initiate OAuth flow + setIsConnecting(true); + // Call backend to initiate authorization flow const response = await fetch( `${process.env.NEXT_PUBLIC_FASTAPI_BACKEND_URL}/api/v1/auth/google/calendar/connector/add/?space_id=${searchSpaceId}`, { @@ -162,46 +70,8 @@ export default function GoogleCalendarConnectorPage() { } catch (error) { console.error("Error connecting to Google:", error); toast.error("Failed to connect to Google Calendar"); - setIsConnecting(false); - } - }; - - // Handle form submission - const onSubmit = async (values: GoogleCalendarConnectorFormValues) => { - if (!isConnected || !credentials) { - toast.error("Please connect your Google account first"); - return; - } - - if (values.calendar_ids.length === 0) { - toast.error("Please select at least one calendar"); - return; - } - - setIsSubmitting(true); - try { - await createConnector({ - name: values.name, - connector_type: "GOOGLE_CALENDAR_CONNECTOR", - config: { - GOOGLE_CALENDAR_CLIENT_ID: credentials.client_id, - GOOGLE_CALENDAR_CLIENT_SECRET: credentials.client_secret, - GOOGLE_CALENDAR_REFRESH_TOKEN: credentials.refresh_token, - GOOGLE_CALENDAR_CALENDAR_IDS: values.calendar_ids, - }, - is_indexable: true, - last_indexed_at: null, - }); - - toast.success("Google Calendar connector created successfully!"); - - // Navigate back to connectors page - router.push(`/dashboard/${searchSpaceId}/connectors`); - } catch (error) { - console.error("Error creating connector:", error); - toast.error(error instanceof Error ? error.message : "Failed to create connector"); } finally { - setIsSubmitting(false); + setIsConnecting(false); } }; @@ -228,14 +98,14 @@ export default function GoogleCalendarConnectorPage() {

Connect Google Calendar

- Connect your Google Calendar to search events, meetings and schedules. + Connect your Google Calendar to search events.

{/* OAuth Connection Card */} - {!isConnected ? ( + {!doesConnectorExist ? ( Connect Your Google Account @@ -285,167 +155,35 @@ export default function GoogleCalendarConnectorPage() { /* Configuration Form Card */ - Configure Google Calendar Connector - - Your Google account is connected! Now select which calendars to include and give - your connector a name. - + ✅ Your Google calendar is successfully connected! -
- - - {/* Connector Name */} - ( - - Connector Name - - - - - A friendly name to identify this connector. - - - - )} - /> - - {/* Calendar Selection */} - ( - -
- Select Calendars - - Choose which calendars you want to include in your search results. - -
- {calendars.map((calendar) => ( - { - return ( - - - { - return checked - ? field.onChange([...field.value, calendar.id]) - : field.onChange( - field.value?.filter((value) => value !== calendar.id) - ); - }} - /> - -
- - {calendar.summary} - {calendar.primary && ( - - Primary - - )} - - {calendar.description && ( - - {calendar.description} - - )} -
- - ); - }} - /> - ))} - - - )} - /> - - - - - -
- )} {/* Help Section */} - - - How It Works - - -
-

1. Connect Your Account

-

- Click "Connect Your Google Account" to start the secure OAuth process. You'll be - redirected to Google to sign in. -

-
-
-

2. Grant Permissions

-

- Google will ask for permission to read your calendar events. We only request - read-only access to keep your data safe. -

-
-
-

3. Select Calendars

-

- Choose which calendars you want to include in your search results. You can select - multiple calendars. -

-
-
-

4. Start Searching

-

- Once connected, your calendar events will be indexed and searchable alongside your - other content. -

-
- {isConnected && ( -
-

- ✅ Your Google account is successfully connected! You can now configure your - connector above. + {!doesConnectorExist && ( + + + How It Works + + +

+

1. Connect Your Account

+

+ Click "Connect Your Google Account" to start the secure OAuth process. You'll be + redirected to Google to sign in.

- )} - - +
+

2. Grant Permissions

+

+ Google will ask for permission to read your calendar events. We only request + read-only access to keep your data safe. +

+
+ + + )}
); diff --git a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx index 2a7855d..c6ec629 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/connectors/add/page.tsx @@ -8,8 +8,8 @@ import { IconBrandSlack, IconBrandWindows, IconBrandZoom, - IconChecklist, IconCalendar, + IconChecklist, IconChevronDown, IconChevronRight, IconLayoutKanban, diff --git a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx index 5ba8cbd..dd226c7 100644 --- a/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx +++ b/surfsense_web/app/dashboard/[search_space_id]/documents/(manage)/page.tsx @@ -7,8 +7,8 @@ import { IconBrandNotion, IconBrandSlack, IconBrandYoutube, - IconChecklist, IconCalendar, + IconChecklist, IconLayoutKanban, IconTicket, } from "@tabler/icons-react"; diff --git a/surfsense_web/hooks/useSearchSourceConnectors.ts b/surfsense_web/hooks/useSearchSourceConnectors.ts index 5785abd..0c59a3b 100644 --- a/surfsense_web/hooks/useSearchSourceConnectors.ts +++ b/surfsense_web/hooks/useSearchSourceConnectors.ts @@ -86,6 +86,8 @@ export const useSearchSourceConnectors = (lazy: boolean = false) => { // Update connector source items when connectors change updateConnectorSourceItems(data); + + return data; } catch (err) { setError(err instanceof Error ? err : new Error("An unknown error occurred")); console.error("Error fetching search source connectors:", err); From 7cc8cd6127da3645e56c15b73a9ad0ba990b85d3 Mon Sep 17 00:00:00 2001 From: Vincenzo Incutti Date: Mon, 4 Aug 2025 14:02:13 +0100 Subject: [PATCH 7/9] Recurse rules --- .recurseml.yaml | 1 + .rules/avoid_source_deduplication.mdc | 28 +++++++++++++++++++ .rules/consistent_container_image_sources.mdc | 28 +++++++++++++++++++ .rules/no_env_files_in_repo.mdc | 22 +++++++++++++++ .rules/require_unique_id_props.mdc | 28 +++++++++++++++++++ 5 files changed, 107 insertions(+) create mode 100644 .recurseml.yaml create mode 100644 .rules/avoid_source_deduplication.mdc create mode 100644 .rules/consistent_container_image_sources.mdc create mode 100644 .rules/no_env_files_in_repo.mdc create mode 100644 .rules/require_unique_id_props.mdc diff --git a/.recurseml.yaml b/.recurseml.yaml new file mode 100644 index 0000000..225cc5b --- /dev/null +++ b/.recurseml.yaml @@ -0,0 +1 @@ +rules: .rules/ \ No newline at end of file diff --git a/.rules/avoid_source_deduplication.mdc b/.rules/avoid_source_deduplication.mdc new file mode 100644 index 0000000..e5fa19b --- /dev/null +++ b/.rules/avoid_source_deduplication.mdc @@ -0,0 +1,28 @@ +```yaml +name: avoid-source-deduplication +description: Preserve unique source entries in search results to maintain proper citation tracking +globs: ['**/connector_service.py', '**/search_service.py'] +alwaysApply: true +``` + +Search result processing should preserve all source entries to maintain accurate citation tracking, rather than deduplicating sources. + +❌ Bad - Deduplicating sources: +```python +mapped_sources = {} +for chunk in chunks: + source_key = chunk.get('url') or chunk.get('title') + if source_key not in mapped_sources: + mapped_sources[source_key] = create_source(chunk) +sources_list = list(mapped_sources.values()) +``` + +✅ Good - Preserving unique sources: +```python +sources_list = [] +for chunk in chunks: + source = create_source(chunk) + sources_list.append(source) +``` + +Each chunk should maintain its unique source reference for proper citation tracking. \ No newline at end of file diff --git a/.rules/consistent_container_image_sources.mdc b/.rules/consistent_container_image_sources.mdc new file mode 100644 index 0000000..ff3115b --- /dev/null +++ b/.rules/consistent_container_image_sources.mdc @@ -0,0 +1,28 @@ +```yaml +name: consistent-container-image-sources +description: Maintain consistent image sources in Docker compose files using authorized registries +globs: ['**/docker-compose.yml', '**/docker-compose.*.yml'] +alwaysApply: true +``` + +Docker compose files should use consistent image sources from authorized registries rather than local builds in production configurations. + +❌ Bad - Mixing build and image sources: +```yaml +services: + frontend: + build: ./frontend + backend: + image: ghcr.io/org/backend:latest +``` + +✅ Good - Consistent image sources: +```yaml +services: + frontend: + image: ghcr.io/org/frontend:latest + backend: + image: ghcr.io/org/backend:latest +``` + +Use build contexts only in development compose files. \ No newline at end of file diff --git a/.rules/no_env_files_in_repo.mdc b/.rules/no_env_files_in_repo.mdc new file mode 100644 index 0000000..baac119 --- /dev/null +++ b/.rules/no_env_files_in_repo.mdc @@ -0,0 +1,22 @@ +```yaml +name: no-env-files-in-repo +description: Prevent committing environment and configuration files containing sensitive credentials +globs: ['**/.env', '**/.env.*', '**/config/*.yml', '**/config/*.yaml'] +alwaysApply: true +``` + +Configuration files like `.env` should never be committed to version control as they often contain sensitive information like API keys, passwords, and tokens. + +❌ Bad - Committing .env files: +``` +POSTGRES_DATABASE_URL=postgresql+psycopg2://user:password@localhost:5432/db +API_KEY=sk-1234567890abcdef +``` + +✅ Good - Use .env.example instead: +``` +POSTGRES_DATABASE_URL=postgresql+psycopg2://user:password@host:5432/dbname +API_KEY=your-api-key-here +``` + +Add `.env` and similar config files to .gitignore and provide example templates instead. \ No newline at end of file diff --git a/.rules/require_unique_id_props.mdc b/.rules/require_unique_id_props.mdc new file mode 100644 index 0000000..33adbc5 --- /dev/null +++ b/.rules/require_unique_id_props.mdc @@ -0,0 +1,28 @@ +```yaml +name: require-unique-id-props +description: Ensure unique key props are provided when mapping arrays to React elements +globs: ['**/*.tsx', '**/*.jsx'] +alwaysApply: true +``` + +When mapping arrays to React elements, each element must have a unique key prop to help React efficiently update the DOM. + +❌ Bad - Missing key prop: +```jsx +{items.map((item) => ( + + {item.name} + +))} +``` + +✅ Good - With key prop: +```jsx +{items.map((item) => ( + + {item.name} + +))} +``` + +Keys should be stable, predictable, and unique among siblings. \ No newline at end of file From 605d0f71fd0596c65ea0fd34b54199d567f55041 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Sun, 3 Aug 2025 12:21:34 +0200 Subject: [PATCH 8/9] add requested changes --- ...17_add_google_calendar_connector_enums.py} | 10 +++---- .../researcher/sub_section_writer/prompts.py | 1 + surfsense_backend/app/db.py | 27 ------------------- 3 files changed, 6 insertions(+), 32 deletions(-) rename surfsense_backend/alembic/versions/{15_add_google_calendar_connector_enums.py => 17_add_google_calendar_connector_enums.py} (91%) diff --git a/surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py b/surfsense_backend/alembic/versions/17_add_google_calendar_connector_enums.py similarity index 91% rename from surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py rename to surfsense_backend/alembic/versions/17_add_google_calendar_connector_enums.py index c4843ac..5fe5bf6 100644 --- a/surfsense_backend/alembic/versions/15_add_google_calendar_connector_enums.py +++ b/surfsense_backend/alembic/versions/17_add_google_calendar_connector_enums.py @@ -1,7 +1,7 @@ """Add Google Calendar connector enums -Revision ID: 15 -Revises: 14 +Revision ID: 17 +Revises: 16 Create Date: 2024-02-01 12:00:00.000000 """ @@ -11,8 +11,8 @@ from collections.abc import Sequence from alembic import op # revision identifiers, used by Alembic. -revision: str = "15" -down_revision: str | None = "14" +revision: str = "17" +down_revision: str | None = "16" branch_labels: str | Sequence[str] | None = None depends_on: str | Sequence[str] | None = None @@ -59,7 +59,7 @@ def downgrade() -> None: """Remove 'GOOGLE_CALENDAR_CONNECTOR' from enum types.""" # Note: PostgreSQL doesn't support removing enum values directly - # This would require recreating the enum type, which is complex + # This would require recreating the enrelum type, which is complex # For now, we'll leave the enum values in place # In a production environment, you might want to implement a more sophisticated downgrade pass diff --git a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py index cb57ea1..5080c1b 100644 --- a/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py +++ b/surfsense_backend/app/agents/researcher/sub_section_writer/prompts.py @@ -17,6 +17,7 @@ You are SurfSense, an advanced AI research assistant that synthesizes informatio - LINEAR_CONNECTOR: "Linear project issues and discussions" (personal project management) - JIRA_CONNECTOR: "Jira project issues, tickets, and comments" (personal project tracking) - CONFLUENCE_CONNECTOR: "Confluence pages and comments" (personal project documentation) +- CLICKUP_CONNECTOR: "ClickUp tasks and project data" (personal task management) - GOOGLE_CALENDAR_CONNECTOR: "Google Calendar events, meetings, and schedules" (personal calendar and time management) - DISCORD_CONNECTOR: "Discord server messages and channels" (personal community interactions) - TAVILY_API: "Tavily search API results" (personalized search results) diff --git a/surfsense_backend/app/db.py b/surfsense_backend/app/db.py index 59cd6df..f572438 100644 --- a/surfsense_backend/app/db.py +++ b/surfsense_backend/app/db.py @@ -246,21 +246,6 @@ class SearchSourceConnector(BaseModel, TimestampMixin): user = relationship("User", back_populates="search_source_connectors") -class GoogleCalendarAccount(BaseModel): - __tablename__ = "google_calendar_accounts" - - user_id = Column( - UUID(as_uuid=True), - ForeignKey("user.id", ondelete="CASCADE"), - nullable=False, - unique=True, - ) - - access_token = Column(String, nullable=False) - refresh_token = Column(String, nullable=True) - user = relationship("User", back_populates="calendar_account") - - class LLMConfig(BaseModel, TimestampMixin): __tablename__ = "llm_configs" @@ -314,12 +299,6 @@ if config.AUTH_TYPE == "GOOGLE": search_source_connectors = relationship( "SearchSourceConnector", back_populates="user" ) - calendar_account = relationship( - "GoogleCalendarAccount", - back_populates="user", - uselist=False, - cascade="all, delete-orphan", - ) llm_configs = relationship( "LLMConfig", back_populates="user", @@ -354,12 +333,6 @@ else: search_source_connectors = relationship( "SearchSourceConnector", back_populates="user" ) - calendar_account = relationship( - "GoogleCalendarAccount", - back_populates="user", - uselist=False, - cascade="all, delete-orphan", - ) llm_configs = relationship( "LLMConfig", back_populates="user", From c1c749426d0e17501b4ea4b33841a59eddeee2f9 Mon Sep 17 00:00:00 2001 From: CREDO23 Date: Thu, 7 Aug 2025 21:43:48 +0200 Subject: [PATCH 9/9] fix merge conflits --- .../app/routes/google_calendar_add_connector_route.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/surfsense_backend/app/routes/google_calendar_add_connector_route.py b/surfsense_backend/app/routes/google_calendar_add_connector_route.py index fea69d1..a319c87 100644 --- a/surfsense_backend/app/routes/google_calendar_add_connector_route.py +++ b/surfsense_backend/app/routes/google_calendar_add_connector_route.py @@ -1,4 +1,7 @@ -# app/routes/google_calendar.py +import os + +os.environ["OAUTHLIB_RELAX_TOKEN_SCOPE"] = "1" + import base64 import json import logging