From 271a21aee65bf091c9dbc2ec8a6f17a7756125cc Mon Sep 17 00:00:00 2001 From: "DESKTOP-RTLN3BA\\$punk" Date: Mon, 6 Jul 2026 19:26:35 -0700 Subject: [PATCH] feat: docs and ui tweaks --- .../multi_agent_chat/shared/tools/mcp/tool.py | 33 +- .../builtins/knowledge_base/agent.py | 9 +- .../subagents/shared/subagent_builder.py | 15 + .../app/capabilities/core/access/rest.py | 41 +- .../app/capabilities/core/billing.py | 47 ++ .../app/services/mcp_oauth/registry.py | 26 +- surfsense_backend/app/utils/validators.py | 6 + surfsense_backend/test_scraper_api_smoke.py | 106 ++++ .../subagents/shared/test_subagent_builder.py | 30 + .../test_mcp_allowlist_fallback.py | 52 ++ .../capabilities/access/test_rest_router.py | 35 ++ .../tests/unit/utils/test_validators.py | 3 + .../[workspace_id]/client-layout.tsx | 6 +- .../components/playground-index.tsx | 16 + .../components/playground-runner.tsx | 8 +- .../connector-dialog.atoms.ts | 18 + .../assistant-ui/connector-popup.tsx | 2 +- .../components/connector-dialog-header.tsx | 2 +- .../components/github-connect-form.tsx | 2 +- .../components/obsidian-config.tsx | 2 +- .../constants/connector-constants.ts | 17 + .../hooks/use-connector-dialog.ts | 48 +- .../tabs/active-connectors-tab.tsx | 8 +- .../tabs/all-connectors-tab.tsx | 56 +- .../components/assistant-ui/thread.tsx | 72 ++- .../components/documents/FolderTreeView.tsx | 2 +- .../layout/ui/sidebar/DocumentsSidebar.tsx | 172 +++++- .../content/docs/connectors/airtable.mdx | 103 ---- .../content/docs/connectors/clickup.mdx | 59 -- .../content/docs/connectors/confluence.mdx | 114 ---- .../{ => deprecated}/baidu-search.mdx | 0 .../connectors/{ => deprecated}/discord.mdx | 6 +- .../{ => deprecated}/elasticsearch.mdx | 4 + .../docs/connectors/{ => deprecated}/luma.mdx | 4 + .../docs/connectors/deprecated/meta.json | 5 + .../{ => deprecated}/microsoft-teams.mdx | 8 +- .../connectors/deprecated/web-crawler.mdx | 10 + .../content/docs/connectors/dropbox.mdx | 79 --- .../docs/connectors/external/airtable.mdx | 69 +++ .../connectors/{ => external}/bookstack.mdx | 4 + .../connectors/{ => external}/circleback.mdx | 0 .../docs/connectors/external/dropbox.mdx | 71 +++ .../docs/connectors/{ => external}/github.mdx | 4 + .../docs/connectors/external/google.mdx | 97 ++++ .../docs/connectors/external/index.mdx | 101 ++++ .../docs/connectors/external/meta.json | 15 + .../connectors/{ => external}/obsidian.mdx | 0 .../docs/connectors/external/onedrive.mdx | 78 +++ .../docs/connectors/external/slack.mdx | 98 ++++ .../content/docs/connectors/gmail.mdx | 91 --- .../docs/connectors/google-calendar.mdx | 88 --- .../content/docs/connectors/google-drive.mdx | 91 --- .../content/docs/connectors/index.mdx | 131 +---- .../content/docs/connectors/jira.mdx | 100 ---- .../content/docs/connectors/linear.mdx | 69 --- .../content/docs/connectors/meta.json | 24 +- .../docs/connectors/microsoft-onedrive.mdx | 104 ---- .../docs/connectors/native/google-maps.mdx | 64 +++ .../docs/connectors/native/google-search.mdx | 39 ++ .../content/docs/connectors/native/index.mdx | 91 +++ .../content/docs/connectors/native/meta.json | 5 + .../content/docs/connectors/native/reddit.mdx | 49 ++ .../docs/connectors/native/web-crawl.mdx | 58 ++ .../docs/connectors/native/youtube.mdx | 54 ++ .../content/docs/connectors/notion.mdx | 106 ---- .../content/docs/connectors/slack.mdx | 95 ---- .../content/docs/connectors/web-crawler.mdx | 6 - .../docs/docker-installation/dev-compose.mdx | 39 -- .../docker-installation/docker-compose.mdx | 419 -------------- .../docs/docker-installation/index.mdx | 205 ++++++- .../docker-installation/install-script.mdx | 100 ---- .../docs/docker-installation/meta.json | 2 +- .../migrate-from-allinone.mdx | 114 ---- .../docs/docker-installation/updating.mdx | 54 +- .../content/docs/how-to/zero-sync.mdx | 6 +- surfsense_web/content/docs/index.mdx | 38 +- .../content/docs/manual-installation.mdx | 536 +++--------------- surfsense_web/content/docs/meta.json | 1 - surfsense_web/content/docs/prerequisites.mdx | 86 --- .../contracts/types/scraper.types.ts | 9 + surfsense_web/lib/playground/catalog.ts | 24 +- surfsense_web/lib/playground/format.ts | 17 + .../lib/playground/platform-icons.tsx | 29 + surfsense_web/messages/en.json | 2 +- .../public/connectors/google-maps.svg | 1 + .../public/connectors/google-search.svg | 1 + surfsense_web/public/connectors/reddit.svg | 1 + surfsense_web/public/connectors/web.svg | 1 + .../tests/connectors/clickup/journey.spec.ts | 2 +- .../composio/calendar/journey.spec.ts | 2 +- .../connectors/composio/drive/journey.spec.ts | 6 +- .../connectors/composio/gmail/journey.spec.ts | 2 +- .../connectors/confluence/journey.spec.ts | 2 +- .../tests/connectors/dropbox/journey.spec.ts | 6 +- .../google/calendar/journey.spec.ts | 2 +- .../connectors/google/drive/journey.spec.ts | 6 +- .../connectors/google/gmail/journey.spec.ts | 2 +- .../tests/connectors/jira/journey.spec.ts | 2 +- .../tests/connectors/linear/journey.spec.ts | 2 +- .../tests/connectors/notion/journey.spec.ts | 2 +- .../tests/connectors/onedrive/journey.spec.ts | 6 +- .../tests/connectors/slack/journey.spec.ts | 2 +- .../tests/helpers/ui/connector-popup.ts | 29 +- 103 files changed, 2161 insertions(+), 2625 deletions(-) create mode 100644 surfsense_backend/test_scraper_api_smoke.py create mode 100644 surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py delete mode 100644 surfsense_web/content/docs/connectors/airtable.mdx delete mode 100644 surfsense_web/content/docs/connectors/clickup.mdx delete mode 100644 surfsense_web/content/docs/connectors/confluence.mdx rename surfsense_web/content/docs/connectors/{ => deprecated}/baidu-search.mdx (100%) rename surfsense_web/content/docs/connectors/{ => deprecated}/discord.mdx (88%) rename surfsense_web/content/docs/connectors/{ => deprecated}/elasticsearch.mdx (94%) rename surfsense_web/content/docs/connectors/{ => deprecated}/luma.mdx (90%) create mode 100644 surfsense_web/content/docs/connectors/deprecated/meta.json rename surfsense_web/content/docs/connectors/{ => deprecated}/microsoft-teams.mdx (87%) create mode 100644 surfsense_web/content/docs/connectors/deprecated/web-crawler.mdx delete mode 100644 surfsense_web/content/docs/connectors/dropbox.mdx create mode 100644 surfsense_web/content/docs/connectors/external/airtable.mdx rename surfsense_web/content/docs/connectors/{ => external}/bookstack.mdx (90%) rename surfsense_web/content/docs/connectors/{ => external}/circleback.mdx (100%) create mode 100644 surfsense_web/content/docs/connectors/external/dropbox.mdx rename surfsense_web/content/docs/connectors/{ => external}/github.mdx (90%) create mode 100644 surfsense_web/content/docs/connectors/external/google.mdx create mode 100644 surfsense_web/content/docs/connectors/external/index.mdx create mode 100644 surfsense_web/content/docs/connectors/external/meta.json rename surfsense_web/content/docs/connectors/{ => external}/obsidian.mdx (100%) create mode 100644 surfsense_web/content/docs/connectors/external/onedrive.mdx create mode 100644 surfsense_web/content/docs/connectors/external/slack.mdx delete mode 100644 surfsense_web/content/docs/connectors/gmail.mdx delete mode 100644 surfsense_web/content/docs/connectors/google-calendar.mdx delete mode 100644 surfsense_web/content/docs/connectors/google-drive.mdx delete mode 100644 surfsense_web/content/docs/connectors/jira.mdx delete mode 100644 surfsense_web/content/docs/connectors/linear.mdx delete mode 100644 surfsense_web/content/docs/connectors/microsoft-onedrive.mdx create mode 100644 surfsense_web/content/docs/connectors/native/google-maps.mdx create mode 100644 surfsense_web/content/docs/connectors/native/google-search.mdx create mode 100644 surfsense_web/content/docs/connectors/native/index.mdx create mode 100644 surfsense_web/content/docs/connectors/native/meta.json create mode 100644 surfsense_web/content/docs/connectors/native/reddit.mdx create mode 100644 surfsense_web/content/docs/connectors/native/web-crawl.mdx create mode 100644 surfsense_web/content/docs/connectors/native/youtube.mdx delete mode 100644 surfsense_web/content/docs/connectors/notion.mdx delete mode 100644 surfsense_web/content/docs/connectors/slack.mdx delete mode 100644 surfsense_web/content/docs/connectors/web-crawler.mdx delete mode 100644 surfsense_web/content/docs/docker-installation/dev-compose.mdx delete mode 100644 surfsense_web/content/docs/docker-installation/docker-compose.mdx delete mode 100644 surfsense_web/content/docs/docker-installation/install-script.mdx delete mode 100644 surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx delete mode 100644 surfsense_web/content/docs/prerequisites.mdx create mode 100644 surfsense_web/lib/playground/platform-icons.tsx create mode 100644 surfsense_web/public/connectors/google-maps.svg create mode 100644 surfsense_web/public/connectors/google-search.svg create mode 100644 surfsense_web/public/connectors/reddit.svg create mode 100644 surfsense_web/public/connectors/web.svg diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py index 2d9d0cf06..fc86dc001 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/shared/tools/mcp/tool.py @@ -649,14 +649,31 @@ async def _load_http_mcp_tools( total_discovered = len(tool_definitions) if allowed_set: - tool_definitions = [td for td in tool_definitions if td["name"] in allowed_set] - logger.info( - "HTTP MCP server '%s' (connector %d): %d/%d tools after allowlist filter", - url, - connector_id, - len(tool_definitions), - total_discovered, - ) + filtered = [td for td in tool_definitions if td["name"] in allowed_set] + if not filtered and total_discovered: + # The server renamed its tools out from under our allowlist + # (e.g. Notion's "notion-" prefix rename) — a fully-dead + # allowlist would silently disable the connector. Load + # everything instead: renamed tools won't match + # ``readonly_tools`` either, so every tool stays HITL-gated. + logger.warning( + "HTTP MCP server '%s' (connector %d): allowlist matched 0/%d " + "advertised tools — server likely renamed its tools. " + "Loading all tools (HITL-gated). Update the registry allowlist: %s", + url, + connector_id, + total_discovered, + sorted(td["name"] for td in tool_definitions), + ) + else: + tool_definitions = filtered + logger.info( + "HTTP MCP server '%s' (connector %d): %d/%d tools after allowlist filter", + url, + connector_id, + len(tool_definitions), + total_discovered, + ) else: logger.info( "Discovered %d tools from HTTP MCP server '%s' (connector %d) — no allowlist, loading all", diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py index 876593a47..b4128524a 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/builtins/knowledge_base/agent.py @@ -16,6 +16,9 @@ from langchain_core.tools import BaseTool from app.agents.chat.multi_agent_chat.shared.filesystem_selection import FilesystemMode from app.agents.chat.multi_agent_chat.shared.permissions import Rule, Ruleset from app.agents.chat.multi_agent_chat.subagents.shared.spec import SurfSenseSubagentSpec +from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + append_today_utc, +) from .middleware_stack import build_kb_middleware from .prompts import load_description, load_readonly_system_prompt, load_system_prompt @@ -57,7 +60,7 @@ def build_subagent( { "name": NAME, "description": load_description(), - "system_prompt": load_system_prompt(filesystem_mode), + "system_prompt": append_today_utc(load_system_prompt(filesystem_mode)), "model": llm, "tools": [_build_search_knowledge_base_tool(dependencies)], "middleware": build_kb_middleware( @@ -86,7 +89,9 @@ def build_readonly_subagent( { "name": READONLY_NAME, "description": "Read-only knowledge_base specialist (invoked via ask_knowledge_base).", - "system_prompt": load_readonly_system_prompt(filesystem_mode), + "system_prompt": append_today_utc( + load_readonly_system_prompt(filesystem_mode) + ), "model": llm, "tools": [_build_search_knowledge_base_tool(dependencies)], "middleware": build_kb_middleware( diff --git a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py index b8182ef24..099ef62d8 100644 --- a/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py +++ b/surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/subagent_builder.py @@ -5,6 +5,7 @@ from __future__ import annotations import logging import re import time as _perf_time +from datetime import UTC, datetime from typing import Any, cast from deepagents import SubAgent @@ -60,6 +61,19 @@ def _resolve_includes(prompt: str, *, subagent_name: str) -> str: return _INCLUDE_DIRECTIVE_RE.sub(_replace, prompt) +def append_today_utc(prompt: str) -> str: + """Append the current UTC date so subagents share the main agent's clock. + + The main agent injects ``Today (UTC): ...`` into its identity prompt, but + delegated subagents (calendar/gmail/drive, date-filtered searches, etc.) + never saw it and would guess the date. Appended at build time; the compiled + graph is cache-keyed on the main prompt hash (which carries the same date), + so a day rollover rebuilds both together and they stay consistent. + """ + today = datetime.now(UTC).date().isoformat() + return f"{prompt}\n\nToday (UTC): {today}\n" + + def _user_allowlist_for( dependencies: dict[str, Any], subagent_name: str ) -> Ruleset | None: @@ -115,6 +129,7 @@ def pack_subagent( _t0 = _perf_time.perf_counter() system_prompt = _resolve_includes(system_prompt, subagent_name=name) + system_prompt = append_today_utc(system_prompt) _t_resolve = _perf_time.perf_counter() - _t0 flags = dependencies["flags"] diff --git a/surfsense_backend/app/capabilities/core/access/rest.py b/surfsense_backend/app/capabilities/core/access/rest.py index d421103c6..59e8bc8ca 100644 --- a/surfsense_backend/app/capabilities/core/access/rest.py +++ b/surfsense_backend/app/capabilities/core/access/rest.py @@ -28,7 +28,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth.context import AuthContext from app.capabilities.core.access.rate_limit import enforce_capability_rate_limit -from app.capabilities.core.billing import charge_capability, gate_capability +from app.capabilities.core.billing import ( + charge_capability, + gate_capability, + pricing_meters, +) from app.capabilities.core.events import run_event_bus from app.capabilities.core.progress import progress_scope from app.capabilities.core.runs import ( @@ -55,13 +59,22 @@ _SSE_HEADERS = { } +class PricingMeter(BaseModel): + """One live per-item rate a verb charges on, e.g. 3500 micro-USD per place.""" + + unit: str + micros_per_unit: int + + class CapabilitySummary(BaseModel): - """A verb's identity + input/output JSON schemas, for the playground UI.""" + """A verb's identity + input/output JSON schemas + pricing, for the playground UI.""" name: str description: str input_schema: dict output_schema: dict + # Empty list = free (billing disabled or an unmetered verb). + pricing: list[PricingMeter] = [] class RunSummary(BaseModel): @@ -125,12 +138,17 @@ def _register_capabilities_list( ) -> None: """Register the ``GET`` that lists verbs + their input schemas for the UI.""" - summaries = [ - CapabilitySummary( - name=capability.name, - description=capability.description, - input_schema=capability.input_schema.model_json_schema(), - output_schema=capability.output_schema.model_json_schema(), + # Schemas are static; pricing is attached per request because rates are + # read live from config (env retune + restart, no rebuild). + base_summaries = [ + ( + CapabilitySummary( + name=capability.name, + description=capability.description, + input_schema=capability.input_schema.model_json_schema(), + output_schema=capability.output_schema.model_json_schema(), + ), + capability.billing_unit, ) for capability in capabilities ] @@ -141,7 +159,12 @@ def _register_capabilities_list( auth: AuthContext = Depends(get_auth_context), ) -> list[CapabilitySummary]: await check_workspace_access(session, auth, workspace_id) - return summaries + return [ + summary.model_copy( + update={"pricing": [PricingMeter(**m) for m in pricing_meters(unit)]} + ) + for summary, unit in base_summaries + ] router.add_api_route( "/workspaces/{workspace_id}/scrapers/capabilities", diff --git a/surfsense_backend/app/capabilities/core/billing.py b/surfsense_backend/app/capabilities/core/billing.py index 7445e4c18..71aa45c58 100644 --- a/surfsense_backend/app/capabilities/core/billing.py +++ b/surfsense_backend/app/capabilities/core/billing.py @@ -43,6 +43,53 @@ def _platform_rate(unit: BillingUnit) -> int: return int(getattr(config, _PLATFORM_RATE_KEYS[unit])) +# Display noun for each platform meter, e.g. "$3.50 / 1k places". +_UNIT_NOUNS: dict[BillingUnit, str] = { + BillingUnit.REDDIT_ITEM: "item", + BillingUnit.GOOGLE_SEARCH_SERP: "SERP", + BillingUnit.GOOGLE_MAPS_PLACE: "place", + BillingUnit.GOOGLE_MAPS_REVIEW: "review", + BillingUnit.YOUTUBE_VIDEO: "video", + BillingUnit.YOUTUBE_COMMENT: "comment", +} + + +def pricing_meters(unit: BillingUnit | None) -> list[dict]: + """The live per-item rates a verb charges, for UI display. Empty = free. + + Mirrors the gate/charge logic exactly: meters whose billing flag is off are + omitted, so a self-hosted install with billing disabled reads as free. + """ + if unit is None: + return [] + if unit is BillingUnit.WEB_CRAWL: + meters = [] + if WebCrawlCreditService.billing_enabled(): + meters.append( + {"unit": "page", "micros_per_unit": config.WEB_CRAWL_MICROS_PER_SUCCESS} + ) + if WebCrawlCreditService.captcha_billing_enabled() and captcha_enabled(): + meters.append( + { + "unit": "captcha solve", + "micros_per_unit": config.WEB_CRAWL_CAPTCHA_MICROS_PER_SOLVE, + } + ) + return meters + if not config.PLATFORM_SCRAPE_BILLING_ENABLED: + return [] + meters = [{"unit": _UNIT_NOUNS[unit], "micros_per_unit": _platform_rate(unit)}] + if unit is BillingUnit.GOOGLE_MAPS_PLACE: + # Dual-metered: attached reviews bill on their own meter. + meters.append( + { + "unit": _UNIT_NOUNS[BillingUnit.GOOGLE_MAPS_REVIEW], + "micros_per_unit": _platform_rate(BillingUnit.GOOGLE_MAPS_REVIEW), + } + ) + return meters + + async def gate_capability( payload: BillableInput, unit: BillingUnit | None, ctx: CapabilityContext ) -> None: diff --git a/surfsense_backend/app/services/mcp_oauth/registry.py b/surfsense_backend/app/services/mcp_oauth/registry.py index 2a8819e0c..f4cfabfd0 100644 --- a/surfsense_backend/app/services/mcp_oauth/registry.py +++ b/surfsense_backend/app/services/mcp_oauth/registry.py @@ -153,13 +153,26 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { "mpim:history", "im:history", ], + # Both prefixed and unprefixed variants: sources disagree on which + # names mcp.slack.com currently advertises. Unmatched entries are + # ignored at discovery time. allowed_tools=[ "slack_search_channels", "slack_read_channel", "slack_read_thread", + "search_channels", + "read_channel", + "read_thread", ], readonly_tools=frozenset( - {"slack_search_channels", "slack_read_channel", "slack_read_thread"} + { + "slack_search_channels", + "slack_read_channel", + "slack_read_thread", + "search_channels", + "read_channel", + "read_thread", + } ), # TODO: oauth.v2.user.access only returns team.id, not team.name. # To populate team_name, either add "team:read" scope and call @@ -192,13 +205,22 @@ MCP_SERVICES: dict[str, MCPServiceConfig] = { # DCR (RFC 7591): Notion issues its own client credentials. It expires # DCR registrations, but refresh reuses the original persisted # ``mcp_oauth.client_id`` (see _refresh_connector_token). + # Notion renamed its MCP tools with a "notion-" prefix (late 2025). + # Unprefixed names kept for servers still advertising the old names — + # allowlist entries the server doesn't advertise are simply ignored. allowed_tools=[ + "notion-search", + "notion-fetch", + "notion-create-pages", + "notion-update-page", "search", "fetch", "create-pages", "update-page", ], - readonly_tools=frozenset({"search", "fetch"}), + readonly_tools=frozenset( + {"notion-search", "notion-fetch", "search", "fetch"} + ), account_metadata_keys=["workspace_name"], ), "confluence": MCPServiceConfig( diff --git a/surfsense_backend/app/utils/validators.py b/surfsense_backend/app/utils/validators.py index aec59cf7b..98bf8aa85 100644 --- a/surfsense_backend/app/utils/validators.py +++ b/surfsense_backend/app/utils/validators.py @@ -30,6 +30,12 @@ DEPRECATED_CONNECTOR_TYPES: frozenset[str] = frozenset( "SEARXNG_API", "LINKUP_API", "BAIDU_SEARCH_API", + # Legacy content crawlers/search superseded by the file Import menu and + # hosted MCP tooling. Created via the generic connector /add route, which + # is the single choke point enforcing this deprecation. + "YOUTUBE_CONNECTOR", + "WEBCRAWLER_CONNECTOR", + "ELASTICSEARCH_CONNECTOR", } ) diff --git a/surfsense_backend/test_scraper_api_smoke.py b/surfsense_backend/test_scraper_api_smoke.py new file mode 100644 index 000000000..8b6177d76 --- /dev/null +++ b/surfsense_backend/test_scraper_api_smoke.py @@ -0,0 +1,106 @@ +"""One-shot smoke test: hit every scraper API endpoint with a PAT, print PASS/FAIL.""" + +import json +import os +import sys +import time + +import httpx + +BASE = "http://localhost:8000/api/v1/workspaces/12/scrapers" +PAT = os.environ["SURFSENSE_PAT"] # export a ss_pat_... key before running +HEADERS = {"Authorization": f"Bearer {PAT}", "Content-Type": "application/json"} + +# Minimal payloads: 1-3 items each to keep credit spend tiny. +VERBS = [ + ("google_search/scrape", {"queries": ["surfsense github"], "max_pages_per_query": 1}), + ("web/crawl", {"startUrls": ["https://example.com"], "maxCrawlDepth": 0}), + ("reddit/scrape", {"urls": ["https://www.reddit.com/r/Python/"], "max_items": 3}), + ("youtube/scrape", {"search_queries": ["python tutorial"], "max_results": 1}), + ( + "youtube/comments", + {"urls": ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"], "max_comments": 3}, + ), + ( + "google_maps/scrape", + {"search_queries": ["coffee shop"], "location": "New York, USA", "max_places": 1}, + ), + ( + "google_maps/reviews", + # Google Sydney office, a stable well-known place id. + {"place_ids": ["ChIJN1t_tDeuEmsRUsoyG83frY4"], "max_reviews": 3}, + ), +] + +results = [] +client = httpx.Client(headers=HEADERS, timeout=300) +sync_run_id = None + +for verb, payload in VERBS: + t0 = time.time() + try: + r = client.post(f"{BASE}/{verb}", json=payload) + dur = time.time() - t0 + run_id = r.headers.get("x-run-id") + if r.status_code == 200: + body = r.json() + # Count items in whatever list field the output has. + items = next((len(v) for v in body.values() if isinstance(v, list)), "?") + results.append((verb, "PASS", f"{r.status_code} items={items} run={run_id} {dur:.1f}s")) + if sync_run_id is None and run_id: + sync_run_id = run_id + else: + results.append((verb, "FAIL", f"{r.status_code} {r.text[:200]} {dur:.1f}s")) + except Exception as e: + results.append((verb, "FAIL", f"{type(e).__name__}: {e}")) + print(f"[{results[-1][1]}] {verb}: {results[-1][2]}", flush=True) + +# --- Run history endpoints --- +r = client.get(f"{BASE}/runs", params={"limit": 5}) +ok = r.status_code == 200 and isinstance(r.json(), list) +results.append(("GET runs (list)", "PASS" if ok else "FAIL", f"{r.status_code} rows={len(r.json()) if ok else '?'}")) +print(f"[{results[-1][1]}] GET runs: {results[-1][2]}", flush=True) + +if sync_run_id: + r = client.get(f"{BASE}/runs/{sync_run_id}") + ok = r.status_code == 200 and r.json().get("id") == sync_run_id + results.append(("GET runs/{id} (detail)", "PASS" if ok else "FAIL", f"{r.status_code} status={r.json().get('status') if ok else r.text[:100]}")) + print(f"[{results[-1][1]}] GET run detail: {results[-1][2]}", flush=True) + +# --- Async mode + SSE events --- +r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"]}) +if r.status_code == 202: + async_id = r.json()["run_id"] + seen, finished = [], None + with client.stream("GET", f"{BASE}/runs/{async_id}/events", timeout=120) as s: + for line in s.iter_lines(): + if line.startswith("data: "): + ev = json.loads(line[6:]) + seen.append(ev["type"]) + if ev["type"] == "run.finished": + finished = ev.get("status") + break + ok = finished == "success" + results.append(("async + SSE events", "PASS" if ok else "FAIL", f"202 events={seen} final={finished}")) +else: + results.append(("async + SSE events", "FAIL", f"{r.status_code} {r.text[:200]}")) +print(f"[{results[-1][1]}] async+SSE: {results[-1][2]}", flush=True) + +# --- Cancel endpoint --- +r = client.post(f"{BASE}/web/crawl?mode=async", json={"startUrls": ["https://example.com"], "maxCrawlDepth": 2, "maxCrawlPages": 50}) +if r.status_code == 202: + cancel_id = r.json()["run_id"] + time.sleep(1) + r2 = client.post(f"{BASE}/runs/{cancel_id}/cancel") + ok = r2.status_code == 200 and r2.json().get("status") == "cancelled" + results.append(("POST runs/{id}/cancel", "PASS" if ok else "FAIL", f"{r2.status_code} {r2.text[:150]}")) +else: + results.append(("POST runs/{id}/cancel", "FAIL", f"setup {r.status_code}")) +print(f"[{results[-1][1]}] cancel: {results[-1][2]}", flush=True) + +print("\n===== SUMMARY =====") +for name, status_, detail in results: + print(f"{status_:4} {name}: {detail}") +failed = [r for r in results if r[1] == "FAIL"] +print(f"\n{len(results) - len(failed)}/{len(results)} passed") +sys.exit(1 if failed else 0) diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py index 364569ae8..fa5faea5c 100644 --- a/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/subagents/shared/test_subagent_builder.py @@ -25,6 +25,7 @@ from app.agents.chat.multi_agent_chat.shared.permissions.middleware.core import PermissionMiddleware, ) from app.agents.chat.multi_agent_chat.subagents.shared.subagent_builder import ( + append_today_utc, pack_subagent, ) @@ -104,6 +105,35 @@ async def test_subagent_recovers_when_primary_llm_fails(): assert final.content == "recovered via fallback" +def test_packed_subagent_prompt_carries_todays_utc_date(): + """Subagents must inherit the main agent's clock, not guess the date.""" + from datetime import UTC, datetime + + today = datetime.now(UTC).date().isoformat() + + result = pack_subagent( + name="date_test", + description="test", + system_prompt="be helpful", + tools=[], + ruleset=Ruleset(origin="date_test", rules=[]), + dependencies={"flags": AgentFeatureFlags()}, + ) + + prompt = result.spec["system_prompt"] + assert prompt.startswith("be helpful") + assert f"Today (UTC): {today}" in prompt + + +def test_append_today_utc_is_idempotent_shape(): + """Helper appends exactly one dated line and preserves the original body.""" + from datetime import UTC, datetime + + today = datetime.now(UTC).date().isoformat() + out = append_today_utc("body") + assert out == f"body\n\nToday (UTC): {today}\n" + + def _extract_permission_mw(spec) -> PermissionMiddleware: """Find the lone PermissionMiddleware in a subagent's middleware list.""" matches = [m for m in spec["middleware"] if isinstance(m, PermissionMiddleware)] diff --git a/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py new file mode 100644 index 000000000..eb6a03ccb --- /dev/null +++ b/surfsense_backend/tests/unit/agents/multi_agent_chat/test_mcp_allowlist_fallback.py @@ -0,0 +1,52 @@ +"""Allowlist filtering in ``_load_http_mcp_tools``. + +Covers the fallback added after Notion renamed its MCP tools ("notion-" +prefix) and a stale allowlist silently disabled the connector: when the +allowlist matches zero advertised tools, load everything (HITL-gated) +instead of nothing. +""" + +from datetime import UTC, datetime + +from app.agents.chat.multi_agent_chat.shared.tools.mcp.cache import ( + CachedMCPTools, +) +from app.agents.chat.multi_agent_chat.shared.tools.mcp.tool import ( + _load_http_mcp_tools, +) + +_CACHED = CachedMCPTools( + discovered_at=datetime.now(UTC), + tools=[ + {"name": "notion-search", "description": "search", "input_schema": {}}, + {"name": "notion-update-page", "description": "write", "input_schema": {}}, + ], +) +_SERVER_CONFIG = {"url": "https://example.com/mcp", "headers": {}} + + +async def test_allowlist_match_filters_and_flags_readonly(): + tools = await _load_http_mcp_tools( + 1, + "Notion", + _SERVER_CONFIG, + allowed_tools=["notion-search"], + readonly_tools=frozenset({"notion-search"}), + cached_tools=_CACHED, + ) + assert [t.name for t in tools] == ["notion-search"] + assert tools[0].metadata["hitl"] is False + + +async def test_stale_allowlist_falls_back_to_all_tools_hitl_gated(): + tools = await _load_http_mcp_tools( + 1, + "Notion", + _SERVER_CONFIG, + allowed_tools=["search", "update-page"], # server renamed everything + readonly_tools=frozenset({"search"}), + cached_tools=_CACHED, + ) + assert sorted(t.name for t in tools) == ["notion-search", "notion-update-page"] + # Renamed tools match no readonly entry -> every tool requires approval. + assert all(t.metadata["hitl"] is True for t in tools) diff --git a/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py b/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py index da3e51de4..b0e64032f 100644 --- a/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py +++ b/surfsense_backend/tests/unit/capabilities/access/test_rest_router.py @@ -123,6 +123,41 @@ async def test_capabilities_endpoint_lists_verbs_with_input_schema(monkeypatch): assert "properties" in entry["output_schema"] +@pytest.mark.asyncio +async def test_capabilities_endpoint_exposes_live_pricing(monkeypatch): + """Billed verbs report their per-item rate; free verbs report an empty list.""" + from app.capabilities.core.types import BillingUnit + from app.config import config + + monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", True) + monkeypatch.setattr(config, "YOUTUBE_MICROS_PER_VIDEO", 2500) + + billed = Capability( + name="test.billed", + description="Billed verb for tests.", + input_schema=_EchoInput, + output_schema=_EchoOutput, + executor=_echo_executor, + billing_unit=BillingUnit.YOUTUBE_VIDEO, + ) + + app = _build_app([_ECHO, billed], monkeypatch) + async with _client(app) as client: + resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities") + assert resp.status_code == 200 + by_name = {entry["name"]: entry for entry in resp.json()} + assert by_name["test.echo"]["pricing"] == [] + assert by_name["test.billed"]["pricing"] == [ + {"unit": "video", "micros_per_unit": 2500} + ] + + # Rates are read live: a config retune shows up without a router rebuild. + monkeypatch.setattr(config, "PLATFORM_SCRAPE_BILLING_ENABLED", False) + async with _client(app) as client: + resp = await client.get("/api/v1/workspaces/7/scrapers/capabilities") + assert resp.json()[1]["pricing"] == [] + + @pytest.mark.asyncio async def test_over_budget_is_blocked_before_the_executor(monkeypatch): from app.capabilities.core.access import rest diff --git a/surfsense_backend/tests/unit/utils/test_validators.py b/surfsense_backend/tests/unit/utils/test_validators.py index 2f1e2aa96..2fedcf39e 100644 --- a/surfsense_backend/tests/unit/utils/test_validators.py +++ b/surfsense_backend/tests/unit/utils/test_validators.py @@ -344,6 +344,9 @@ def test_validate_connector_config_invalid(): "SEARXNG_API", "LINKUP_API", "BAIDU_SEARCH_API", + "YOUTUBE_CONNECTOR", + "WEBCRAWLER_CONNECTOR", + "ELASTICSEARCH_CONNECTOR", ], ) def test_raise_if_connector_deprecated_blocks(connector_type): diff --git a/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx b/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx index b54ce0aad..1f4948548 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/client-layout.tsx @@ -14,6 +14,7 @@ import { modelRolesAtom, } from "@/atoms/model-connections/model-connections-query.atoms"; import { activeWorkspaceIdAtom } from "@/atoms/workspaces/workspace-query.atoms"; +import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; import { DocumentUploadDialogProvider } from "@/components/assistant-ui/document-upload-popup"; import { LayoutDataProvider } from "@/components/layout"; import { OnboardingTour } from "@/components/onboarding-tour"; @@ -214,7 +215,10 @@ export function DashboardClientLayout({ return ( - {children} + + {children} + + ); } diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx index 2ebfb08f1..741ed291f 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-index.tsx @@ -2,11 +2,22 @@ import { ArrowRight, History, KeyRound } from "lucide-react"; import Link from "next/link"; +import { useMemo } from "react"; +import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { PLAYGROUND_PLATFORMS } from "@/lib/playground/catalog"; +import { formatPricing } from "@/lib/playground/format"; export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) { const base = `/dashboard/${workspaceId}/playground`; + // The grid renders from the static catalog immediately; pricing fills in + // once the capabilities fetch lands (blank while loading, never blocking). + const { data: capabilities } = useScraperCapabilities(workspaceId); + const pricingByName = useMemo( + () => new Map(capabilities?.map((c) => [c.name, formatPricing(c.pricing)])), + [capabilities] + ); + return (
@@ -67,6 +78,11 @@ export function PlaygroundIndex({ workspaceId }: { workspaceId: number }) {
{verb.name} + {pricingByName.has(verb.name) ? ( + + {pricingByName.get(verb.name)} + + ) : null} ))}
diff --git a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx index 6ef16c9e8..3b76a7d09 100644 --- a/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx +++ b/surfsense_web/app/dashboard/[workspace_id]/playground/components/playground-runner.tsx @@ -10,7 +10,7 @@ import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { scrapersApiService } from "@/lib/apis/scrapers-api.service"; import { AbortedError, AppError } from "@/lib/error"; import { findVerb } from "@/lib/playground/catalog"; -import { formatCost, formatDuration } from "@/lib/playground/format"; +import { formatCost, formatDuration, formatPricing } from "@/lib/playground/format"; import { buildPayload, initialFormValues, parseSchemaFields } from "@/lib/playground/json-schema"; import { ApiReference } from "./api-reference"; import { OutputViewer } from "./output-viewer"; @@ -198,6 +198,12 @@ export function PlaygroundRunner({ workspaceId, platform, verb }: PlaygroundRunn POST /workspaces/{workspaceId}/scrapers/{platform}/{verb} +
+ + + {formatPricing(capability.pricing)} + +
OAuth connect, + * one -> edit view, many -> accounts list. Used by "Connect" and "Manage". + * - `mode: "connect"` always starts a fresh OAuth connect, even when an account + * already exists. Used by "Add another account". + */ +export interface ImportConnectorRequest { + connectorType: string; + mode: "auto" | "connect"; +} + +export const importConnectorRequestAtom = atom(null); diff --git a/surfsense_web/components/assistant-ui/connector-popup.tsx b/surfsense_web/components/assistant-ui/connector-popup.tsx index d702a4209..b1a02b953 100644 --- a/surfsense_web/components/assistant-ui/connector-popup.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup.tsx @@ -189,7 +189,7 @@ export const ConnectorIndicator = forwardRef - Manage Connectors + Manage External MCP Connectors {/* YouTube Crawler View - shown when adding YouTube videos */} {isYouTubeView && workspaceId ? ( diff --git a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx index 25ad22523..435685b0f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/components/connector-dialog-header.tsx @@ -31,7 +31,7 @@ export const ConnectorDialogHeader: FC = ({ > - Manage Connectors + Manage External MCP Connectors Connect Surfsense to your favorite tools and services. diff --git a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/github-connect-form.tsx b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/github-connect-form.tsx index 56609dbdd..4a7fbee72 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/github-connect-form.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/connect-forms/components/github-connect-form.tsx @@ -293,7 +293,7 @@ export const GithubConnectForm: FC = ({ onSubmit, isSubmitting {/* Documentation Link */}
([ EnumConnectorName.SEARXNG_API, EnumConnectorName.LINKUP_API, EnumConnectorName.BAIDU_SEARCH_API, + // Legacy content crawlers/search retired in favor of the file Import menu and + // hosted MCP tooling. Existing rows stay manageable; new connections refused. + EnumConnectorName.YOUTUBE_CONNECTOR, + EnumConnectorName.WEBCRAWLER_CONNECTOR, + EnumConnectorName.ELASTICSEARCH_CONNECTOR, +]); + +/** + * File-import connectors surfaced through the Documents sidebar "Import" menu + * instead of the external-MCP connector catalog. They index remote files into + * the knowledge base, so they belong with uploads, not with live MCP tools. + */ +export const IMPORT_CONNECTOR_TYPES = new Set([ + EnumConnectorName.GOOGLE_DRIVE_CONNECTOR, + EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR, + EnumConnectorName.ONEDRIVE_CONNECTOR, + EnumConnectorName.DROPBOX_CONNECTOR, ]); /** diff --git a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts index a3eaf632f..4e599b577 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts +++ b/surfsense_web/components/assistant-ui/connector-popup/hooks/use-connector-dialog.ts @@ -2,7 +2,10 @@ import { format } from "date-fns"; import { useAtom, useAtomValue } from "jotai"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; -import { connectorDialogOpenAtom } from "@/atoms/connector-dialog/connector-dialog.atoms"; +import { + connectorDialogOpenAtom, + importConnectorRequestAtom, +} from "@/atoms/connector-dialog/connector-dialog.atoms"; import { createConnectorMutationAtom, deleteConnectorMutationAtom, @@ -67,6 +70,8 @@ export const useConnectorDialog = () => { // Use global atom for dialog open state so it can be controlled from anywhere const [isOpen, setIsOpen] = useAtom(connectorDialogOpenAtom); + // Import-flow request from the Documents sidebar "Import" menu + const [importConnectorRequest, setImportConnectorRequest] = useAtom(importConnectorRequestAtom); const [activeTab, setActiveTab] = useState("all"); const [connectingId, setConnectingId] = useState(null); const [isScrolled, setIsScrolled] = useState(false); @@ -1372,6 +1377,47 @@ export const useConnectorDialog = () => { [isStartingIndexing, isDisconnecting, isSaving, isCreatingConnector, setIsOpen] ); + // Consume an import request from the Documents sidebar "Import" menu. + // mode "connect" -> always OAuth connect (add account). mode "auto" routes by + // account count: none -> connect, one -> edit view, many -> accounts list. + useEffect(() => { + if (!importConnectorRequest || !workspaceId) return; + + const { connectorType, mode } = importConnectorRequest; + setImportConnectorRequest(null); + + const connectorDef = + OAUTH_CONNECTORS.find((c) => c.connectorType === connectorType) || + COMPOSIO_CONNECTORS.find((c) => c.connectorType === connectorType); + + const existing = (allConnectors || []).filter( + (c: SearchSourceConnector) => c.connector_type === connectorType + ); + + if (mode === "connect" || existing.length === 0) { + if (connectorDef) { + handleConnectOAuth(connectorDef); + } + return; + } + + setIsOpen(true); + if (existing.length === 1) { + handleStartEdit(existing[0]); + } else { + handleViewAccountsList(connectorType, connectorDef?.title); + } + }, [ + importConnectorRequest, + workspaceId, + allConnectors, + setImportConnectorRequest, + handleConnectOAuth, + handleStartEdit, + handleViewAccountsList, + setIsOpen, + ]); + const handleTabChange = useCallback((value: string) => { setActiveTab(value); }, []); diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx index 42532859a..24ee6473f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/active-connectors-tab.tsx @@ -11,6 +11,7 @@ import { getDocumentTypeLabel } from "@/lib/documents/document-type-labels"; import { cn } from "@/lib/utils"; import { COMPOSIO_CONNECTORS, + IMPORT_CONNECTOR_TYPES, LIVE_CONNECTOR_TYPES, OAUTH_CONNECTORS, } from "../constants/connector-constants"; @@ -33,12 +34,17 @@ export const ActiveConnectorsTab: FC = ({ searchQuery, hasSources, activeDocumentTypes, - connectors, + connectors: allActiveConnectors, indexingConnectorIds, onTabChange: _onTabChange, onManage, onViewAccountsList, }) => { + // Import connectors (Drive/OneDrive/Dropbox) are managed via the Documents + // sidebar "Import" menu, so they are excluded from the MCP connector list. + const connectors = allActiveConnectors.filter( + (c) => !IMPORT_CONNECTOR_TYPES.has(c.connector_type) + ); // Convert activeDocumentTypes array to Record for utility function const documentTypeCounts = activeDocumentTypes.reduce( (acc, [docType, count]) => { diff --git a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx index 7adde28b0..ea98f258f 100644 --- a/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx +++ b/surfsense_web/components/assistant-ui/connector-popup/tabs/all-connectors-tab.tsx @@ -14,6 +14,7 @@ import { CRAWLERS, DEPRECATED_CONNECTOR_TYPES, getConnectorCategory, + IMPORT_CONNECTOR_TYPES, OAUTH_CONNECTORS, OTHER_CONNECTORS, } from "../constants/connector-constants"; @@ -82,30 +83,41 @@ export const AllConnectorsTab: FC = ({ const passesDeploymentFilter = (c: DeploymentFilterableConnector) => (!c.selfHostedOnly || selfHosted) && (!c.desktopOnly || isDesktop); - // Filter connectors based on search and deployment mode + // Import connectors (Drive/OneDrive/Dropbox) are managed via the Documents + // sidebar "Import" menu, so they never appear in the MCP connector catalog. + const notImport = (c: { connectorType?: string }) => + !c.connectorType || !IMPORT_CONNECTOR_TYPES.has(c.connectorType); + + // Filter connectors based on search, deployment mode, and import exclusion const filteredOAuth = OAUTH_CONNECTORS.filter( - (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) + (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c) ); const filteredCrawlers = CRAWLERS.filter( - (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) + (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c) ); const filteredOther = OTHER_CONNECTORS.filter( - (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) + (c) => matchesSearch(c.title, c.description) && passesDeploymentFilter(c) && notImport(c) ); // Filter Composio connectors const filteredComposio = COMPOSIO_CONNECTORS.filter( (c) => - c.title.toLowerCase().includes(searchQuery.toLowerCase()) || - c.description.toLowerCase().includes(searchQuery.toLowerCase()) + (c.title.toLowerCase().includes(searchQuery.toLowerCase()) || + c.description.toLowerCase().includes(searchQuery.toLowerCase())) && + notImport(c) ); + const isDeprecated = (c: { connectorType?: string }) => + !!c.connectorType && DEPRECATED_CONNECTOR_TYPES.has(c.connectorType); + const inCategory = (category: ConnectorCategory) => (connector: T): boolean => - !!connector.connectorType && getConnectorCategory(connector.connectorType) === category; + !isDeprecated(connector) && + !!connector.connectorType && + getConnectorCategory(connector.connectorType) === category; const knowledgeBase = { oauth: filteredOAuth.filter(inCategory("knowledge_base")), @@ -120,6 +132,16 @@ export const AllConnectorsTab: FC = ({ crawlers: filteredCrawlers.filter(inCategory("tools_live")), }; + // Deprecated cards render in a single section at the very bottom. Discord and + // Teams come from OAUTH_CONNECTORS; Luma, Tavily, Linkup, Baidu, Elasticsearch + // from OTHER_CONNECTORS (SearXNG has no catalog card); YouTube and Web Pages + // from CRAWLERS. + const deprecated = { + oauth: filteredOAuth.filter(isDeprecated), + other: filteredOther.filter(isDeprecated), + crawlers: filteredCrawlers.filter(isDeprecated), + }; + const renderOAuthCard = (connector: OAuthConnector | ComposioConnector) => { const isConnected = connectedTypes.has(connector.connectorType); const isConnecting = connectingId === connector.id; @@ -245,6 +267,9 @@ export const AllConnectorsTab: FC = ({ isConnecting={isConnecting} documentCount={documentCount} isIndexing={isIndexing} + deprecated={ + !!crawler.connectorType && DEPRECATED_CONNECTOR_TYPES.has(crawler.connectorType) + } onConnect={handleConnect} onManage={actualConnector && onManage ? () => onManage(actualConnector) : undefined} /> @@ -261,8 +286,10 @@ export const AllConnectorsTab: FC = ({ toolsLive.composio.length > 0 || toolsLive.other.length > 0 || toolsLive.crawlers.length > 0; + const hasDeprecated = + deprecated.oauth.length > 0 || deprecated.other.length > 0 || deprecated.crawlers.length > 0; - const hasAnyResults = hasKnowledgeBase || hasToolsLive; + const hasAnyResults = hasKnowledgeBase || hasToolsLive || hasDeprecated; if (!hasAnyResults && searchQuery) { return ( @@ -307,6 +334,19 @@ export const AllConnectorsTab: FC = ({
)} + + {hasDeprecated && ( +
+
+

Deprecated

+
+
+ {deprecated.oauth.map(renderOAuthCard)} + {deprecated.crawlers.map(renderCrawlerCard)} + {deprecated.other.map(renderOtherCard)} +
+
+ )} ); }; diff --git a/surfsense_web/components/assistant-ui/thread.tsx b/surfsense_web/components/assistant-ui/thread.tsx index ac8c118e7..1b8548e3d 100644 --- a/surfsense_web/components/assistant-ui/thread.tsx +++ b/surfsense_web/components/assistant-ui/thread.tsx @@ -22,7 +22,7 @@ import { Wrench, X, } from "lucide-react"; -import { AnimatePresence, motion } from "motion/react"; +import { AnimatePresence, motion, useReducedMotion } from "motion/react"; import Image from "next/image"; import { useParams } from "next/navigation"; import { type FC, useCallback, useEffect, useMemo, useRef, useState } from "react"; @@ -56,7 +56,6 @@ import { currentUserAtom } from "@/atoms/user/user-query.atoms"; import { AssistantMessage } from "@/components/assistant-ui/assistant-message"; import { ChatSessionStatus } from "@/components/assistant-ui/chat-session-status"; import { ChatViewport } from "@/components/assistant-ui/chat-viewport"; -import { ConnectorIndicator } from "@/components/assistant-ui/connector-popup"; import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup"; import { InlineMentionEditor, @@ -94,6 +93,7 @@ import { import { Popover, PopoverAnchor } from "@/components/ui/popover"; import { Skeleton } from "@/components/ui/skeleton"; import { Switch } from "@/components/ui/switch"; +import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; import { CONNECTOR_ICON_TO_TYPES, @@ -105,9 +105,11 @@ import { useBatchCommentsPreload } from "@/hooks/use-comments"; import { useCommentsSync } from "@/hooks/use-comments-sync"; import { useMediaQuery } from "@/hooks/use-media-query"; import { useElectronAPI } from "@/hooks/use-platform"; +import { useScraperCapabilities } from "@/hooks/use-scraper-capabilities"; import { captureDisplayToPngDataUrl } from "@/lib/chat/display-media-capture"; import { getMentionDocKey } from "@/lib/chat/mention-doc-key"; import { slideoutOpenedTickAtom } from "@/lib/layout-events"; +import { findPlatform, type PlaygroundPlatform } from "@/lib/playground/catalog"; import { getWorkspaceIdNumber } from "@/lib/route-params"; import { cn } from "@/lib/utils"; import { @@ -965,7 +967,6 @@ const Composer: FC = () => { workspaceId={workspaceId ?? 0} onChatModelSelected={handleChatModelSelected} /> - { ); }; +/** + * Full-color brand marks for the platform-native scraper APIs (web, Google + * Search, Google Maps, Reddit, YouTube) available in this workspace, shown beside the + * composer "+" so the user can see these native endpoints are connected. Laid + * out as a clean row (not stacked) after a hairline divider that separates them + * from the composer actions. The capability registry is the source of truth; + * icons are display-only with a status tooltip. One-time staggered entrance, + * reduced-motion aware. + */ +const ConnectedScraperIcons: FC<{ workspaceId: number }> = ({ workspaceId }) => { + const { data: capabilities } = useScraperCapabilities(workspaceId); + const reduceMotion = useReducedMotion(); + + const platforms = useMemo(() => { + if (!capabilities?.length) return []; + const seen = new Set(); + const result: PlaygroundPlatform[] = []; + for (const cap of capabilities) { + const platformId = cap.name.split(".")[0]; + if (seen.has(platformId)) continue; + seen.add(platformId); + const platform = findPlatform(platformId); + if (platform) result.push(platform); + } + return result; + }, [capabilities]); + + if (platforms.length === 0) return null; + + return ( +
+
+
+ {platforms.map((platform, i) => { + const Icon = platform.icon; + return ( + + + + + + + {platform.label} · Connected + + ); + })} +
+
+ ); +}; + interface ComposerActionProps { isBlockedByOtherUser?: boolean; workspaceId: number; @@ -1142,7 +1203,7 @@ const ComposerAction: FC = ({ setConnectorDialogOpen(true)}> - Manage Connectors + Manage External MCP Connectors setToolsPopoverOpen(true)}> @@ -1362,7 +1423,7 @@ const ComposerAction: FC = ({ setConnectorDialogOpen(true)}> - Manage Connectors + Manage External MCP Connectors = ({ )} +
{!hasModelConfigured && (
diff --git a/surfsense_web/components/documents/FolderTreeView.tsx b/surfsense_web/components/documents/FolderTreeView.tsx index 27098c587..6c896b4b4 100644 --- a/surfsense_web/components/documents/FolderTreeView.tsx +++ b/surfsense_web/components/documents/FolderTreeView.tsx @@ -313,7 +313,7 @@ export function FolderTreeView({

No documents found

- Use the plus menu to upload files or manage connectors + Use the Import button above to add files, or the plus menu to manage connectors

); diff --git a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx index 1fe83c3b8..528cfbe9e 100644 --- a/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx +++ b/surfsense_web/components/layout/ui/sidebar/DocumentsSidebar.tsx @@ -2,16 +2,29 @@ import { useQuery } from "@rocicorp/zero/react"; import { useAtom, useAtomValue, useSetAtom } from "jotai"; -import { FolderPlus, ListFilter, SlidersVertical, Trash2 } from "lucide-react"; +import { + FolderInput, + FolderPlus, + ListFilter, + Plus, + Settings2, + SlidersVertical, + Trash2, + Upload, +} from "lucide-react"; import { useParams } from "next/navigation"; import { useTranslations } from "next-intl"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { toast } from "sonner"; import { makeFolderMention, mentionedDocumentsAtom } from "@/atoms/chat/mentioned-documents.atom"; +import { importConnectorRequestAtom } from "@/atoms/connector-dialog/connector-dialog.atoms"; +import { connectorsAtom } from "@/atoms/connectors/connector-query.atoms"; import { deleteDocumentMutationAtom } from "@/atoms/documents/document-mutation.atoms"; import { expandedFolderIdsAtom } from "@/atoms/documents/folder.atoms"; import { agentCreatedDocumentsAtom } from "@/atoms/documents/ui.atoms"; import { openEditorPanelAtom } from "@/atoms/editor/editor-panel.atom"; +import { useConnectorStatus } from "@/components/assistant-ui/connector-popup/hooks/use-connector-status"; +import { useDocumentUploadDialog } from "@/components/assistant-ui/document-upload-popup"; import { CreateFolderDialog } from "@/components/documents/CreateFolderDialog"; import type { DocumentNodeDoc } from "@/components/documents/DocumentNode"; import { getDocumentTypeIcon } from "@/components/documents/DocumentTypeIcon"; @@ -19,7 +32,7 @@ import type { FolderDisplay } from "@/components/documents/FolderNode"; import { FolderPickerDialog } from "@/components/documents/FolderPickerDialog"; import { FolderTreeView } from "@/components/documents/FolderTreeView"; import { VersionHistoryDialog } from "@/components/documents/version-history"; -import { useRuntimeConfig } from "@/components/providers/runtime-config"; +import { useIsSelfHosted, useRuntimeConfig } from "@/components/providers/runtime-config"; import { EXPORT_FILE_EXTENSIONS } from "@/components/shared/ExportMenuItems"; import { DEFAULT_EXCLUDE_PATTERNS } from "@/components/sources/FolderWatchDialog"; import { @@ -38,6 +51,7 @@ import { DropdownMenuCheckboxItem, DropdownMenuContent, DropdownMenuItem, + DropdownMenuSeparator, DropdownMenuSub, DropdownMenuSubContent, DropdownMenuSubTrigger, @@ -47,6 +61,9 @@ import { Skeleton } from "@/components/ui/skeleton"; import { Spinner } from "@/components/ui/spinner"; import { useAnonymousMode, useIsAnonymous } from "@/contexts/anonymous-mode"; import { useLoginGate } from "@/contexts/login-gate"; +import { EnumConnectorName } from "@/contracts/enums/connector"; +import { getConnectorIcon } from "@/contracts/enums/connectorIcons"; +import type { SearchSourceConnector } from "@/contracts/types/connector.types"; import type { DocumentTypeEnum } from "@/contracts/types/document.types"; import { useElectronAPI, usePlatform } from "@/hooks/use-platform"; import { documentsApiService } from "@/lib/apis/documents-api.service"; @@ -163,6 +180,127 @@ export function EmbeddedDocumentsMenu({ ); } +/** + * Import menu: local file upload plus the cloud-drive import connectors + * (Google Drive / OneDrive / Dropbox). Drive/OneDrive/Dropbox set + * `importConnectorRequestAtom`, which the connector dialog consumes to run + * OAuth or open the existing account's config. In anonymous mode, `gate` + * intercepts every item to trigger the login flow. + */ +export function EmbeddedImportMenu({ gate }: { gate?: (feature: string) => void }) { + const { openDialog } = useDocumentUploadDialog(); + const setImportRequest = useSetAtom(importConnectorRequestAtom); + const selfHosted = useIsSelfHosted(); + const { isConnectorEnabled, getConnectorStatusMessage } = useConnectorStatus(); + const { data: connectors } = useAtomValue(connectorsAtom); + + // Native Google Drive connector self-hosted only; hosted deployments use Composio. + const driveType = selfHosted + ? EnumConnectorName.GOOGLE_DRIVE_CONNECTOR + : EnumConnectorName.COMPOSIO_GOOGLE_DRIVE_CONNECTOR; + + const cloudItems = [ + { type: driveType, label: "Google Drive" }, + { type: EnumConnectorName.ONEDRIVE_CONNECTOR, label: "OneDrive" }, + { type: EnumConnectorName.DROPBOX_CONNECTOR, label: "Dropbox" }, + ]; + + return ( + + + + + + (gate ? gate("upload files") : openDialog())}> + + Upload Files + + + {cloudItems.map((item) => { + const enabled = gate ? true : isConnectorEnabled(item.type); + const statusMessage = enabled ? null : getConnectorStatusMessage(item.type); + const icon = getConnectorIcon(item.type, "h-4 w-4"); + // gate = anonymous mode; treat every connector as unconnected so items + // route through the login gate rather than reading workspace connectors. + const accountCount = gate + ? 0 + : (connectors ?? []).filter( + (c: SearchSourceConnector) => c.connector_type === item.type + ).length; + + // Unavailable (e.g. maintenance): non-actionable item explaining why. + if (!enabled) { + return ( + + {icon} + {item.label} + + ); + } + + // Connected: manage the existing account(s) or add another. + if (accountCount > 0) { + return ( + + + {icon} + {item.label} + {accountCount} + + + + gate + ? gate("manage import connectors") + : setImportRequest({ connectorType: item.type, mode: "auto" }) + } + > + + {accountCount > 1 ? "Manage accounts" : "Manage"} + + + gate + ? gate("import from cloud storage") + : setImportRequest({ connectorType: item.type, mode: "connect" }) + } + > + + Add another account + + + + ); + } + + // Not connected: single click starts the first OAuth connect. + return ( + + gate + ? gate("import from cloud storage") + : setImportRequest({ connectorType: item.type, mode: "auto" }) + } + > + {icon} + {item.label} + + ); + })} + + + ); +} + function CloudDocumentsSkeleton() { const rows = [ { id: "row-1", widthClass: "w-44" }, @@ -1041,12 +1179,15 @@ function AuthenticatedDocumentsSidebarBase({ defaultOpen={true} contentClassName="px-0" persistentAction={ - handleCreateFolder(null)} - /> +
+ + handleCreateFolder(null)} + /> +
} > {renderDocumentTree()} @@ -1214,12 +1355,15 @@ function AnonymousDocumentsSidebar({ embedded = false }: DocumentsSidebarProps) defaultOpen={true} contentClassName="px-0" persistentAction={ - {}} - onCreateFolder={() => gate("create folders")} - /> +
+ + {}} + onCreateFolder={() => gate("create folders")} + /> +
} > - Never share your client secret publicly. - - ---- - -## Running SurfSense with Airtable Connector - -Add the Airtable credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -AIRTABLE_CLIENT_ID=your_airtable_client_id -AIRTABLE_CLIENT_SECRET=your_airtable_client_secret -AIRTABLE_REDIRECT_URI=http://localhost:8000/api/v1/auth/airtable/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/clickup.mdx b/surfsense_web/content/docs/connectors/clickup.mdx deleted file mode 100644 index 31709dfdd..000000000 --- a/surfsense_web/content/docs/connectors/clickup.mdx +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: ClickUp -description: Connect your ClickUp workspace to SurfSense ---- - -# ClickUp OAuth Integration Setup Guide - -This guide walks you through setting up a ClickUp OAuth integration for SurfSense. - -## Step 1: Access ClickUp API Settings - -1. Open your ClickUp workspace -2. Navigate to **Settings** (gear icon) → **ClickUp API** -3. You'll see the **ClickUp API Settings** page - -![ClickUp API Settings Page](/docs/connectors/clickup/clickup-api-settings.png) - -## Step 2: Create an App - -1. Click **"+ Create an App"** in the top-right corner -2. Fill in the app details: - -| Field | Value | -|-------|-------| -| **App Name** | `SurfSense` | -| **Redirect URL(s)** | `localhost:8000` | - -3. Click **"Save"** to create the app - -![App Created with Credentials](/docs/connectors/clickup/clickup-app-credentials.png) - -## Step 3: Get OAuth Credentials - -After creating the app, you'll see your credentials: - -1. Copy your **Client ID** -2. Copy your **Client Secret** (click "Show" to reveal, or "Regenerate" if needed) - - - Never share your client secret publicly. - - ---- - -## Running SurfSense with ClickUp Connector - -Add the ClickUp credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -CLICKUP_CLIENT_ID=your_clickup_client_id -CLICKUP_CLIENT_SECRET=your_clickup_client_secret -CLICKUP_REDIRECT_URI=http://localhost:8000/api/v1/auth/clickup/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` \ No newline at end of file diff --git a/surfsense_web/content/docs/connectors/confluence.mdx b/surfsense_web/content/docs/connectors/confluence.mdx deleted file mode 100644 index 7e38d9a32..000000000 --- a/surfsense_web/content/docs/connectors/confluence.mdx +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Confluence -description: Connect your Confluence spaces to SurfSense ---- - -# Confluence OAuth Integration Setup Guide - -This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Confluence spaces. - -## Step 1: Access the Developer Console - -1. Navigate to [developer.atlassian.com](https://developer.atlassian.com) -2. Click your profile icon in the top-right corner -3. Select **"Developer console"** from the dropdown - -![Atlassian Developer Console Access](/docs/connectors/atlassian/atlassian-dev-console-access.png) - -## Step 2: Create a New OAuth 2.0 Integration - -1. In the Developer Console, under **My apps**, click the **"Create"** button -2. Select **"OAuth 2.0 integration"** from the dropdown - -![Create OAuth 2.0 Integration](/docs/connectors/atlassian/atlassian-create-app.png) - -## Step 3: Name Your Integration - -1. Enter **Name**: `SurfSense` -2. Check the box to agree to Atlassian's developer terms -3. Click **"Create"** - - - New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse. - - -![Create New Integration Form](/docs/connectors/atlassian/atlassian-name-integration.png) - -## Step 4: Configure Callback URL - -1. In the left sidebar, click **"Authorization"** -2. Under **Callback URLs**, enter the redirect URI: - ``` - http://localhost:8000/api/v1/auth/confluence/connector/callback - ``` -3. Click **"Save changes"** - - - You can enter up to 10 redirect URIs, one per line. - - -![Authorization Callback URLs](/docs/connectors/atlassian/atlassian-authorization.png) - -## Step 5: Configure API Permissions - -1. In the left sidebar, click **"Permissions"** -2. You'll see a list of available APIs including Confluence API - -![Permissions Overview](/docs/connectors/atlassian/atlassian-permissions.png) - -## Step 6: Configure Confluence API Scopes - -1. Click **"Configure"** next to **Confluence API** - -### Classic Scopes - -Select the **"Classic scopes"** tab and enable: - -| Scope Name | Code | Description | -|------------|------|-------------| -| Read user | `read:confluence-user` | View user information in Confluence that you have access to, including usernames, email addresses and profile pictures | - -![Confluence API Classic Scopes](/docs/connectors/atlassian/confluence/atlassian-confluence-classic-scopes.png) - -### Granular Scopes - -Select the **"Granular scopes"** tab and enable: - -| Scope Name | Code | Description | -|------------|------|-------------| -| View pages | `read:page:confluence` | View page content | -| View comments | `read:comment:confluence` | View comments on pages or blogposts | -| View spaces | `read:space:confluence` | View space details | -| Write pages | `write:page:confluence` | Create and update page content | -| Delete pages | `delete:page:confluence` | Delete pages | - -4. Click **"Save"** - -![Confluence API Granular Scopes](/docs/connectors/atlassian/confluence/atlassian-confluence-granular-scopes.png) - -## Step 7: Get OAuth Credentials - -1. In the left sidebar, click **"Settings"** -2. Copy your **Client ID** and **Client Secret** - - - Never share your client secret publicly. - - ---- - -## Running SurfSense with Confluence Connector - -Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -ATLASSIAN_CLIENT_ID=your_atlassian_client_id -ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret -CONFLUENCE_REDIRECT_URI=http://localhost:8000/api/v1/auth/confluence/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/baidu-search.mdx b/surfsense_web/content/docs/connectors/deprecated/baidu-search.mdx similarity index 100% rename from surfsense_web/content/docs/connectors/baidu-search.mdx rename to surfsense_web/content/docs/connectors/deprecated/baidu-search.mdx diff --git a/surfsense_web/content/docs/connectors/discord.mdx b/surfsense_web/content/docs/connectors/deprecated/discord.mdx similarity index 88% rename from surfsense_web/content/docs/connectors/discord.mdx rename to surfsense_web/content/docs/connectors/deprecated/discord.mdx index cef0c2d10..850f3156f 100644 --- a/surfsense_web/content/docs/connectors/discord.mdx +++ b/surfsense_web/content/docs/connectors/deprecated/discord.mdx @@ -5,6 +5,10 @@ description: Connect your Discord servers to SurfSense # Discord OAuth Integration Setup Guide + +The Discord connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only. + + This guide walks you through setting up a Discord OAuth integration for SurfSense. ## Step 1: Create a New Discord Application @@ -64,7 +68,7 @@ You'll also see your **Application ID** and **Public Key** on this page. ## Running SurfSense with Discord Connector -Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): +Add the Discord credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): ```bash DISCORD_CLIENT_ID=your_discord_client_id diff --git a/surfsense_web/content/docs/connectors/elasticsearch.mdx b/surfsense_web/content/docs/connectors/deprecated/elasticsearch.mdx similarity index 94% rename from surfsense_web/content/docs/connectors/elasticsearch.mdx rename to surfsense_web/content/docs/connectors/deprecated/elasticsearch.mdx index f2615a9c6..7d2f04071 100644 --- a/surfsense_web/content/docs/connectors/elasticsearch.mdx +++ b/surfsense_web/content/docs/connectors/deprecated/elasticsearch.mdx @@ -5,6 +5,10 @@ description: Connect your Elasticsearch cluster to SurfSense # Elasticsearch Integration Setup Guide + +The Elasticsearch connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only. + + This guide walks you through connecting your Elasticsearch cluster to SurfSense. ## How it works diff --git a/surfsense_web/content/docs/connectors/luma.mdx b/surfsense_web/content/docs/connectors/deprecated/luma.mdx similarity index 90% rename from surfsense_web/content/docs/connectors/luma.mdx rename to surfsense_web/content/docs/connectors/deprecated/luma.mdx index a218be46b..5867e0587 100644 --- a/surfsense_web/content/docs/connectors/luma.mdx +++ b/surfsense_web/content/docs/connectors/deprecated/luma.mdx @@ -5,6 +5,10 @@ description: Connect your Luma events to SurfSense # Luma Integration Setup Guide + +The Luma connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only. + + This guide walks you through connecting your Luma events to SurfSense for event search and AI-powered insights. ## How it works diff --git a/surfsense_web/content/docs/connectors/deprecated/meta.json b/surfsense_web/content/docs/connectors/deprecated/meta.json new file mode 100644 index 000000000..96e1494ff --- /dev/null +++ b/surfsense_web/content/docs/connectors/deprecated/meta.json @@ -0,0 +1,5 @@ +{ + "title": "Deprecated", + "pages": ["discord", "microsoft-teams", "luma", "baidu-search", "elasticsearch", "web-crawler"], + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/connectors/microsoft-teams.mdx b/surfsense_web/content/docs/connectors/deprecated/microsoft-teams.mdx similarity index 87% rename from surfsense_web/content/docs/connectors/microsoft-teams.mdx rename to surfsense_web/content/docs/connectors/deprecated/microsoft-teams.mdx index 166004c1f..fe1f57a29 100644 --- a/surfsense_web/content/docs/connectors/microsoft-teams.mdx +++ b/surfsense_web/content/docs/connectors/deprecated/microsoft-teams.mdx @@ -7,8 +7,12 @@ description: Connect your Microsoft Teams to SurfSense This guide walks you through setting up a Microsoft Teams OAuth integration for SurfSense using Azure App Registration. + +The Microsoft Teams connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Existing connections remain manageable. The guide below is retained for reference only. + + - Microsoft Teams and [Microsoft OneDrive](/docs/connectors/microsoft-onedrive) share the same Azure App Registration. If you have already created an app for OneDrive, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3). + Microsoft Teams and [Microsoft OneDrive](/docs/connectors/external/onedrive) share the same Azure App Registration. If you have already created an app for OneDrive, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3). ## Step 1: Access Azure App Registrations @@ -101,7 +105,7 @@ After registration, you will be taken to the app's **Overview** page. Here you w ## Running SurfSense with Microsoft Teams Connector -Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): +Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation)): ```bash MICROSOFT_CLIENT_ID=your_microsoft_client_id diff --git a/surfsense_web/content/docs/connectors/deprecated/web-crawler.mdx b/surfsense_web/content/docs/connectors/deprecated/web-crawler.mdx new file mode 100644 index 000000000..92aa587b6 --- /dev/null +++ b/surfsense_web/content/docs/connectors/deprecated/web-crawler.mdx @@ -0,0 +1,10 @@ +--- +title: Web Crawler +description: The web crawler connector is deprecated +--- + +# Web Crawler + + +The web crawler connector is **deprecated** and can no longer be connected (the backend refuses new connections with HTTP 410). Crawling is now built into the platform as the [Web Crawl native scraper](/docs/connectors/native/web-crawl). To save individual webpages, you can also use the [SurfSense browser extension](https://github.com/MODSetter/SurfSense/tree/main/surfsense_browser_extension), which captures pages behind authentication. + diff --git a/surfsense_web/content/docs/connectors/dropbox.mdx b/surfsense_web/content/docs/connectors/dropbox.mdx deleted file mode 100644 index d1e5f4451..000000000 --- a/surfsense_web/content/docs/connectors/dropbox.mdx +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: Dropbox -description: Connect your Dropbox to SurfSense ---- - -# Dropbox OAuth Integration Setup Guide - -This guide walks you through setting up a Dropbox OAuth integration for SurfSense using the Dropbox App Console. - -## Step 1: Access the Dropbox App Console - -1. Navigate to [dropbox.com/developers/apps](https://www.dropbox.com/developers/apps) -2. Sign in with your Dropbox account - -## Step 2: Create a New App - -1. Click **"Create app"** -2. Fill in the app details: - -| Field | Value | -|-------|-------| -| **Choose an API** | Select **"Scoped access"** | -| **Choose the type of access** | Select **"Full Dropbox"** | -| **Name your app** | `SurfSense` (or any unique name) | - -3. Click **"Create app"** - -## Step 3: Configure Redirect URI - -1. On your app's **Settings** page, scroll to the **OAuth 2** section -2. Under **Redirect URIs**, add: `http://localhost:8000/api/v1/auth/dropbox/connector/callback` -3. Click **"Add"** - -## Step 4: Get App Key and Secret - -On the same **Settings** page, you will find: - -1. **App key** - this is your `DROPBOX_APP_KEY` -2. **App secret** - click **"Show"** to reveal, then copy. This is your `DROPBOX_APP_SECRET` - - - Never share your app secret publicly or include it in code repositories. - - -## Step 5: Configure Permissions - -1. Go to the **Permissions** tab of your app -2. Enable the following scopes: - -| Permission | Description | -|------------|-------------| -| `files.metadata.read` | View information about files and folders | -| `files.content.read` | View and download file content | -| `files.content.write` | Create, modify, and delete files | -| `account_info.read` | View basic account information | - -3. Click **"Submit"** to save the permissions - - - All four permissions listed above are required. The connector will not authenticate successfully if any are missing. - - ---- - -## Running SurfSense with Dropbox Connector - -Add the Dropbox OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -DROPBOX_APP_KEY=your_dropbox_app_key -DROPBOX_APP_SECRET=your_dropbox_app_secret -DROPBOX_REDIRECT_URI=http://localhost:8000/api/v1/auth/dropbox/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/external/airtable.mdx b/surfsense_web/content/docs/connectors/external/airtable.mdx new file mode 100644 index 000000000..fd0dbad1a --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/airtable.mdx @@ -0,0 +1,69 @@ +--- +title: Airtable +description: Let the SurfSense agent browse your Airtable bases, tables, and records +--- + +The Airtable connector connects SurfSense to Airtable's hosted MCP server, giving the agent live tools to list your bases, browse tables, and read records. Nothing is indexed in the background. + + +This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**. + + +## Step 1: Register an OAuth Integration + +1. Navigate to [airtable.com/create/oauth](https://airtable.com/create/oauth) +2. In the **Builder Hub**, under **Developers**, click **"OAuth integrations"** +3. Click **"Register an OAuth integration"** + +![Airtable OAuth Integrations Page](/docs/connectors/airtable/airtable-oauth-integrations.png) + +Fill in the details: + +| Field | Value | +|-------|-------| +| **Name** | `SurfSense` | +| **OAuth redirect URL** | `http://localhost:3929/api/v1/auth/mcp/airtable/connector/callback` | + + +`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. + + +Click **"Register integration"** + +![Register Integration Form](/docs/connectors/airtable/airtable-register-integration.png) + +## Step 2: Configure Scopes + +Enable the scopes SurfSense requests: + +| Scope | Description | +|-------|-------------| +| `data.records:read` | See the data in records | +| `schema.bases:read` | See the structure of a base, like table names or field types | + +![Scopes Configuration](/docs/connectors/airtable/airtable-scopes.png) + +## Step 3: Fill in Support Information + +Add a support email (plus privacy policy and terms URLs if you have them) and click **"Save changes"**. + +![Support Information & Preview](/docs/connectors/airtable/airtable-support-info.png) + +## Step 4: Add the Credentials to SurfSense + +Copy the **Client ID** and **Client Secret** from the integration page and add them to your `.env` file: + +```bash +AIRTABLE_CLIENT_ID=your_airtable_client_id +AIRTABLE_CLIENT_SECRET=your_airtable_client_secret +``` + +Restart SurfSense, open the **Connectors** dialog, and click **Connect** on Airtable: + +```bash +docker compose up -d +``` + + +Never share your client secret publicly. + diff --git a/surfsense_web/content/docs/connectors/bookstack.mdx b/surfsense_web/content/docs/connectors/external/bookstack.mdx similarity index 90% rename from surfsense_web/content/docs/connectors/bookstack.mdx rename to surfsense_web/content/docs/connectors/external/bookstack.mdx index f1f695e11..3443ace04 100644 --- a/surfsense_web/content/docs/connectors/bookstack.mdx +++ b/surfsense_web/content/docs/connectors/external/bookstack.mdx @@ -5,6 +5,10 @@ description: Connect your Bookstack instance to SurfSense # BookStack Integration Setup Guide + +The BookStack indexing pipeline is currently deprecated — the knowledge base now stores files, notes, and uploads only. Existing BookStack connectors remain manageable, but new indexing runs are refused by the backend. + + This guide walks you through connecting your BookStack instance to SurfSense. ## How it works diff --git a/surfsense_web/content/docs/connectors/circleback.mdx b/surfsense_web/content/docs/connectors/external/circleback.mdx similarity index 100% rename from surfsense_web/content/docs/connectors/circleback.mdx rename to surfsense_web/content/docs/connectors/external/circleback.mdx diff --git a/surfsense_web/content/docs/connectors/external/dropbox.mdx b/surfsense_web/content/docs/connectors/external/dropbox.mdx new file mode 100644 index 000000000..0cc5acf40 --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/dropbox.mdx @@ -0,0 +1,71 @@ +--- +title: Dropbox +description: Import your Dropbox files into the SurfSense knowledge base +--- + +The Dropbox connector lets you pick files from Dropbox and index them into your knowledge base. It lives in the Documents sidebar's **Import** menu, not the connector catalog. + + +This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just use **Import → Dropbox**. + + +## Step 1: Create a Dropbox App + +1. Navigate to [dropbox.com/developers/apps](https://www.dropbox.com/developers/apps) and sign in +2. Click **"Create app"** and fill in: + +| Field | Value | +|-------|-------| +| **Choose an API** | **"Scoped access"** | +| **Choose the type of access** | **"Full Dropbox"** | +| **Name your app** | `SurfSense` (or any unique name) | + +3. Click **"Create app"** + +## Step 2: Configure the Redirect URI + +1. On your app's **Settings** page, scroll to the **OAuth 2** section +2. Under **Redirect URIs**, add: + +``` +http://localhost:3929/api/v1/auth/dropbox/connector/callback +``` + + +`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. + + +## Step 3: Configure Permissions + +1. Go to the **Permissions** tab of your app +2. Enable all of the following scopes and click **"Submit"**: + +| Permission | Description | +|------------|-------------| +| `files.metadata.read` | View information about files and folders | +| `files.content.read` | View and download file content | +| `files.content.write` | Create, modify, and delete files | +| `account_info.read` | View basic account information | + + +All four permissions are required. The connector will not authenticate successfully if any are missing. + + +## Step 4: Add the Credentials to SurfSense + +On the **Settings** page, copy the **App key** and **App secret** (click "Show" to reveal), and add them to your `.env` file: + +```bash +DROPBOX_APP_KEY=your_dropbox_app_key +DROPBOX_APP_SECRET=your_dropbox_app_secret +``` + +Restart SurfSense, open **Documents → Import → Dropbox**, and authorize: + +```bash +docker compose up -d +``` + + +Never share your app secret publicly or include it in code repositories. + diff --git a/surfsense_web/content/docs/connectors/github.mdx b/surfsense_web/content/docs/connectors/external/github.mdx similarity index 90% rename from surfsense_web/content/docs/connectors/github.mdx rename to surfsense_web/content/docs/connectors/external/github.mdx index ce60d2a32..12531a319 100644 --- a/surfsense_web/content/docs/connectors/github.mdx +++ b/surfsense_web/content/docs/connectors/external/github.mdx @@ -5,6 +5,10 @@ description: Connect your GitHub repositories to SurfSense # GitHub Integration Setup Guide + +The GitHub indexing pipeline is currently deprecated — the knowledge base now stores files, notes, and uploads only. Existing GitHub connectors remain manageable, but new indexing runs are refused by the backend. + + This guide walks you through connecting your GitHub repositories to SurfSense for code search and AI-powered insights. ## How it works diff --git a/surfsense_web/content/docs/connectors/external/google.mdx b/surfsense_web/content/docs/connectors/external/google.mdx new file mode 100644 index 000000000..50d8fa081 --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/google.mdx @@ -0,0 +1,97 @@ +--- +title: Google (Drive, Gmail, Calendar) +description: Set up one Google OAuth app for the Drive, Gmail, and Calendar connectors +--- + +One Google Cloud OAuth app powers Google login and all three Google connectors: + +- **Google Drive** — import Drive files into your knowledge base via the Documents sidebar's **Import** menu +- **Gmail** — the agent can search, read, draft, and send emails +- **Google Calendar** — the agent can search and manage your events + + +This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**. + + + +The redirect URIs below use `http://localhost:3929` — the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. + + +## Step 1: Create a Google Cloud Project + +1. Navigate to the [Google Cloud Console](https://console.cloud.google.com/) +2. Select an existing project or create a new one + +## Step 2: Enable the APIs + +1. Go to **APIs & Services** > **Library** +2. Search for and enable: + - **People API** (required for Google OAuth itself) + - **Google Drive API** (for the Drive connector) + - **Gmail API** (for the Gmail connector) + - **Google Calendar API** (for the Calendar connector) + +You only need to enable the APIs for the connectors you plan to use — the People API is always required. + +![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) + +## Step 3: Configure the OAuth Consent Screen + +1. Go to **APIs & Services** > **OAuth consent screen** +2. Select **External** user type (or Internal if using Google Workspace) +3. Fill in the required information: + - **App name**: `SurfSense` + - **User support email**: Your email address + - **Developer contact information**: Your email address +4. Click **Save and Continue** + +![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) + +### Add Scopes + +Click **Add or Remove Scopes** and add: + +| Scope | Used by | +|-------|---------| +| `openid`, `.../auth/userinfo.email`, `.../auth/userinfo.profile` | All (basic OAuth) | +| `https://www.googleapis.com/auth/drive` | Drive connector | +| `https://www.googleapis.com/auth/gmail.modify` | Gmail connector | +| `https://www.googleapis.com/auth/calendar.events` | Calendar connector | + + +The write-capable scopes (`drive`, `gmail.modify`, `calendar.events`) are what let the agent create files, send emails, and manage events — always with your confirmation in chat. If you only want read access, use `drive.readonly`, `gmail.readonly`, and `calendar.readonly` instead; the write tools then won't work. + + +## Step 4: Create the OAuth Client + +1. Go to **APIs & Services** > **Credentials** +2. Click **Create Credentials** > **OAuth client ID** +3. Select **Web application** as the application type +4. Enter **Name**: `SurfSense` +5. Under **Authorized redirect URIs**, add one URI per connector you plan to use: + +``` +http://localhost:3929/api/v1/auth/google/drive/connector/callback +http://localhost:3929/api/v1/auth/google/gmail/connector/callback +http://localhost:3929/api/v1/auth/google/calendar/connector/callback +``` + +6. Click **Create** + +![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) + +## Step 5: Add the Credentials to SurfSense + +Copy the **Client ID** and **Client Secret** from the confirmation dialog, then add them to your `.env` file (see the Google section of `.env.example` for the exact variable names — the same client ID and secret are shared by Google login and all three connectors, plus one redirect URI variable per connector). + +![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) + +Restart SurfSense and the Google connectors are ready to connect: + +```bash +docker compose up -d +``` + + +Never share your client secret publicly. + diff --git a/surfsense_web/content/docs/connectors/external/index.mdx b/surfsense_web/content/docs/connectors/external/index.mdx new file mode 100644 index 000000000..3645e9e03 --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/index.mdx @@ -0,0 +1,101 @@ +--- +title: External Connectors +description: Connect third-party services like Notion, Slack, Google, GitHub, and Obsidian +--- + +import { Card, Cards } from 'fumadocs-ui/components/card'; + +External connectors link SurfSense to third-party services. Open the **Connectors** dialog in your workspace, pick a service, and connect — via OAuth, an API token, a webhook, or a companion plugin, depending on the service. You can connect multiple accounts of the same service. + +Most of them act as **live tools for the AI agent** — when you chat, the agent can search your Notion pages, read Slack threads, or create Jira issues in real time. File sources (Google Drive, OneDrive, Dropbox) instead show up in the Documents sidebar's **Import** menu, where you pick files to index into your knowledge base. + +## One-click connectors (no setup) + +These work out of the box on every deployment — SurfSense connects to the service's hosted MCP server and the service issues credentials automatically. Just click **Connect**: + +| Connector | What the agent can do | +|-----------|----------------------| +| **Notion** | Search, read, create, and update pages | +| **Linear** | Search, read, and manage issues and projects | +| **Jira** | Search issues with JQL, browse projects, create and edit issues | +| **Confluence** | Search and read spaces, create and update pages | +| **ClickUp** | Search and read tasks | + +Write actions (creating a page, editing an issue) always ask for your confirmation in chat before they run. + +## Connectors that need OAuth credentials (self-hosted) + +For these, a self-hosted deployment needs its own OAuth app registered with the provider. Create the app once, put the credentials in your `.env` (each variable is documented in `.env.example`), restart, and the **Connect** button works for everyone on your instance: + + + + + + + + + + +On SurfSense Cloud these are already configured — just click **Connect**. The setup guides above only apply to self-hosted deployments. + + +## Token, webhook, and plugin connectors + +These you configure yourself with an API token, a webhook, or a companion plugin: + + + + + + + + +There's also a generic **Custom MCP** connector in the catalog that lets you plug any MCP server into the agent — including API-key-based services like Tavily or Linkup. + +## Importing files from Drive, OneDrive, and Dropbox + +Google Drive, OneDrive, and Dropbox are file sources, so they don't appear in the connector catalog. Instead, open the **Documents** sidebar, click **Import**, and pick the service. After authorizing, browse and select the files you want — they're indexed into your knowledge base and kept searchable alongside your uploads and notes. + +## Managing connections + +Everything happens in the **Connectors** dialog: + +- **Active** tab — see connected accounts, live status, and document counts. +- **Reconnect** — if a token expires, the connector card prompts you to re-authorize. +- **Disconnect** — removes the connection. For knowledge-base connectors this also removes the documents it indexed. diff --git a/surfsense_web/content/docs/connectors/external/meta.json b/surfsense_web/content/docs/connectors/external/meta.json new file mode 100644 index 000000000..4c420179f --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/meta.json @@ -0,0 +1,15 @@ +{ + "title": "External Connectors", + "pages": [ + "google", + "slack", + "airtable", + "onedrive", + "dropbox", + "github", + "bookstack", + "circleback", + "obsidian" + ], + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/connectors/obsidian.mdx b/surfsense_web/content/docs/connectors/external/obsidian.mdx similarity index 100% rename from surfsense_web/content/docs/connectors/obsidian.mdx rename to surfsense_web/content/docs/connectors/external/obsidian.mdx diff --git a/surfsense_web/content/docs/connectors/external/onedrive.mdx b/surfsense_web/content/docs/connectors/external/onedrive.mdx new file mode 100644 index 000000000..376bc7a23 --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/onedrive.mdx @@ -0,0 +1,78 @@ +--- +title: Microsoft OneDrive +description: Import your OneDrive files into the SurfSense knowledge base +--- + +The OneDrive connector lets you pick files from OneDrive and index them into your knowledge base. It lives in the Documents sidebar's **Import** menu, not the connector catalog. + + +This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just use **Import → OneDrive**. + + +## Step 1: Create an Azure App Registration + +1. Navigate to [portal.azure.com](https://portal.azure.com) +2. Search for **"App registrations"** and open it +3. Click **"+ New registration"** and fill in: + +| Field | Value | +|-------|-------| +| **Name** | `SurfSense` | +| **Supported account types** | **"Accounts in any organizational directory (Any Microsoft Entra ID tenant - Multitenant) and personal Microsoft accounts"** | +| **Redirect URI** | Platform: `Web`, URI: `http://localhost:3929/api/v1/auth/onedrive/connector/callback` | + + +`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. + + +4. Click **"Register"** + +## Step 2: Get the Application (Client) ID + +On the app's **Overview** page, copy the **Application (client) ID** — this is your Client ID. + +## Step 3: Create a Client Secret + +1. Under **Manage**, click **"Certificates & secrets"** +2. On the **"Client secrets"** tab, click **"+ New client secret"** +3. Enter a description (e.g., `SurfSense`) and pick an expiration period +4. Click **"Add"** +5. **Important**: Copy the secret **Value** immediately — it will not be shown again! + +## Step 4: Configure API Permissions + +1. Under **Manage**, click **"API permissions"** > **"+ Add a permission"** +2. Select **"Microsoft Graph"** > **"Delegated permissions"** +3. Add all of the following: + +| Permission | Description | +|------------|-------------| +| `Files.Read.All` | Read all files the user can access | +| `Files.ReadWrite.All` | Read and write all files the user can access | +| `offline_access` | Maintain access to granted data | +| `User.Read` | Sign in and read user profile | + +4. Click **"Add permissions"** + + +All four permissions are required. The connector will not authenticate successfully if any are missing. + + +## Step 5: Add the Credentials to SurfSense + +Add the credentials to your `.env` file: + +```bash +MICROSOFT_CLIENT_ID=your_microsoft_client_id +MICROSOFT_CLIENT_SECRET=your_microsoft_client_secret +``` + +Restart SurfSense, open **Documents → Import → OneDrive**, and authorize: + +```bash +docker compose up -d +``` + + +Never share your client secret publicly or include it in code repositories. + diff --git a/surfsense_web/content/docs/connectors/external/slack.mdx b/surfsense_web/content/docs/connectors/external/slack.mdx new file mode 100644 index 000000000..c04782599 --- /dev/null +++ b/surfsense_web/content/docs/connectors/external/slack.mdx @@ -0,0 +1,98 @@ +--- +title: Slack +description: Let the SurfSense agent search and read your Slack workspace +--- + +The Slack connector connects SurfSense to Slack's hosted MCP server, giving the agent live tools to search channels and read threads on your behalf. Nothing is indexed in the background — the agent queries Slack in real time when you ask. + + +This setup is only needed on **self-hosted** deployments. On SurfSense Cloud, just click **Connect**. + + +## Step 1: Create a New Slack App + +1. Navigate to [api.slack.com/apps](https://api.slack.com/apps) +2. Click **"Create New App"** +3. Select **"From scratch"** + +![Create an App Dialog](/docs/connectors/slack/slack-create-app.png) + +## Step 2: Name App & Choose Workspace + +1. Enter **App Name**: `SurfSense` +2. Select the workspace to develop your app in +3. Click **"Create App"** + + +You won't be able to change the workspace later. The workspace will control the app even if you leave it. + + +![Name App & Choose Workspace](/docs/connectors/slack/slack-name-workspace.png) + +## Step 3: Get App Credentials + +On the **Basic Information** page: + +1. Copy your **Client ID** +2. Copy your **Client Secret** (click Show to reveal) + +![Basic Information - App Credentials](/docs/connectors/slack/slack-app-credentials.png) + +## Step 4: Configure the Redirect URL + +1. In the left sidebar, click **"OAuth & Permissions"** +2. Under **Redirect URLs**, click **"Add New Redirect URL"** +3. Enter your SurfSense callback URL: + +``` +http://localhost:3929/api/v1/auth/mcp/slack/connector/callback +``` + + +`http://localhost:3929` is the default public URL of a Docker install. If you run SurfSense [manually](/docs/manual-installation), use your backend URL (`http://localhost:8000`); in production, use your public domain. Note that Slack requires HTTPS redirect URLs for production apps. + + +4. Click **"Add"**, then **"Save URLs"** + +![Redirect URLs Configuration](/docs/connectors/slack/slack-redirect-urls.png) + +## Step 5: Configure User Token Scopes + +On the same **OAuth & Permissions** page, scroll to **Scopes**. The connector acts on behalf of the connecting user, so add these under **User Token Scopes**: + +| OAuth Scope | Description | +|-------------|-------------| +| `search:read.public` | Search public channels | +| `search:read.private` | Search private channels the user is in | +| `search:read.mpim` | Search group direct messages | +| `search:read.im` | Search direct messages | +| `channels:history` | Read messages in public channels | +| `groups:history` | Read messages in private channels | +| `mpim:history` | Read group direct messages | +| `im:history` | Read direct messages | + +## Step 6: Enable Public Distribution + +1. In the left sidebar, click **"Manage Distribution"** +2. Under **Share Your App with Other Workspaces**, enable distribution so users can authorize the app in their own workspaces + +![Manage Distribution](/docs/connectors/slack/slack-distribution.png) + +## Step 7: Add the Credentials to SurfSense + +Add the Client ID and Client Secret to your `.env` file: + +```bash +SLACK_CLIENT_ID=your_slack_client_id +SLACK_CLIENT_SECRET=your_slack_client_secret +``` + +Restart SurfSense, open the **Connectors** dialog, and click **Connect** on Slack: + +```bash +docker compose up -d +``` + + +Never share your app credentials publicly. + diff --git a/surfsense_web/content/docs/connectors/gmail.mdx b/surfsense_web/content/docs/connectors/gmail.mdx deleted file mode 100644 index 2a1adce0e..000000000 --- a/surfsense_web/content/docs/connectors/gmail.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: Gmail -description: Connect your Gmail to SurfSense ---- - -# Gmail OAuth Integration Setup Guide - -This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Gmail account. - -## Step 1: Access the Google Cloud Console - -1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) -2. Select an existing project or create a new one - -## Step 2: Enable Required APIs - -1. Go to **APIs & Services** > **Library** -2. Search for and enable the following APIs: - - **People API** (required for Google OAuth) - - **Gmail API** (required for Gmail connector) - -![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) - -## Step 3: Configure OAuth Consent Screen - -1. Go to **APIs & Services** > **OAuth consent screen** -2. Select **External** user type (or Internal if using Google Workspace) -3. Fill in the required information: - - **App name**: `SurfSense` - - **User support email**: Your email address - - **Developer contact information**: Your email address -4. Click **Save and Continue** - -![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) - -### Add Scopes - -1. Click **Add or Remove Scopes** -2. Add the following scopes: - - `https://www.googleapis.com/auth/gmail.modify` - Read, compose, send, and permanently delete Gmail messages - - `https://www.googleapis.com/auth/userinfo.email` - View user email address - - `https://www.googleapis.com/auth/userinfo.profile` - View user profile info - - `openid` - OpenID Connect authentication -3. Click **Update** and then **Save and Continue** - - - The `gmail.modify` scope is required for HITL (Human-in-the-Loop) tools like sending emails, creating drafts, and trashing messages. If you only need read access, you can use `gmail.readonly` instead, but HITL tools will not work. - - -## Step 4: Create OAuth Client ID - -1. Go to **APIs & Services** > **Credentials** -2. Click **Create Credentials** > **OAuth client ID** -3. Select **Web application** as the application type -4. Enter **Name**: `SurfSense` -5. Under **Authorized redirect URIs**, add: - ``` - http://localhost:8000/api/v1/auth/google/gmail/connector/callback - ``` -6. Click **Create** - -![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) - -## Step 5: Get OAuth Credentials - -1. After creating the OAuth client, you'll see a dialog with your credentials -2. Copy your **Client ID** and **Client Secret** - - - Never share your client secret publicly. - - -![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) - ---- - -## Running SurfSense with Gmail Connector - -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -GOOGLE_OAUTH_CLIENT_ID=your_google_client_id -GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret -GOOGLE_GMAIL_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/gmail/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/google-calendar.mdx b/surfsense_web/content/docs/connectors/google-calendar.mdx deleted file mode 100644 index a01d225bc..000000000 --- a/surfsense_web/content/docs/connectors/google-calendar.mdx +++ /dev/null @@ -1,88 +0,0 @@ ---- -title: Google Calendar -description: Connect your Google Calendar to SurfSense ---- - -# Google Calendar OAuth Integration Setup Guide - -This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Calendar. - -## Step 1: Access the Google Cloud Console - -1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) -2. Select an existing project or create a new one - -## Step 2: Enable Required APIs - -1. Go to **APIs & Services** > **Library** -2. Search for and enable the following APIs: - - **People API** (required for Google OAuth) - - **Google Calendar API** (required for Calendar connector) - -![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) - -## Step 3: Configure OAuth Consent Screen - -1. Go to **APIs & Services** > **OAuth consent screen** -2. Select **External** user type (or Internal if using Google Workspace) -3. Fill in the required information: - - **App name**: `SurfSense` - - **User support email**: Your email address - - **Developer contact information**: Your email address -4. Click **Save and Continue** - -![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) - -### Add Scopes - -1. Click **Add or Remove Scopes** -2. Add the following scope: - - `https://www.googleapis.com/auth/calendar.events` - Read, create, update, and delete Google Calendar events -3. Click **Update** and then **Save and Continue** - - - The `calendar.events` scope is required for HITL (Human-in-the-Loop) tools like creating, updating, and deleting calendar events. If you only need read access, you can use `calendar.readonly` instead, but HITL tools will not work. - - -## Step 4: Create OAuth Client ID - -1. Go to **APIs & Services** > **Credentials** -2. Click **Create Credentials** > **OAuth client ID** -3. Select **Web application** as the application type -4. Enter **Name**: `SurfSense` -5. Under **Authorized redirect URIs**, add: - ``` - http://localhost:8000/api/v1/auth/google/calendar/connector/callback - ``` -6. Click **Create** - -![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) - -## Step 5: Get OAuth Credentials - -1. After creating the OAuth client, you'll see a dialog with your credentials -2. Copy your **Client ID** and **Client Secret** - - - Never share your client secret publicly. - - -![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) - ---- - -## Running SurfSense with Google Calendar Connector - -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -GOOGLE_OAUTH_CLIENT_ID=your_google_client_id -GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret -GOOGLE_CALENDAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/calendar/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/google-drive.mdx b/surfsense_web/content/docs/connectors/google-drive.mdx deleted file mode 100644 index 3ff846035..000000000 --- a/surfsense_web/content/docs/connectors/google-drive.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: Google Drive -description: Connect your Google Drive to SurfSense ---- - -# Google Drive OAuth Integration Setup Guide - -This guide walks you through setting up a Google OAuth 2.0 integration for SurfSense to connect your Google Drive. - -## Step 1: Access the Google Cloud Console - -1. Navigate to [Google Cloud Console](https://console.cloud.google.com/) -2. Select an existing project or create a new one - -## Step 2: Enable Required APIs - -1. Go to **APIs & Services** > **Library** -2. Search for and enable the following APIs: - - **People API** (required for Google OAuth) - - **Google Drive API** (required for Drive connector) - -![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) - -## Step 3: Configure OAuth Consent Screen - -1. Go to **APIs & Services** > **OAuth consent screen** -2. Select **External** user type (or Internal if using Google Workspace) -3. Fill in the required information: - - **App name**: `SurfSense` - - **User support email**: Your email address - - **Developer contact information**: Your email address -4. Click **Save and Continue** - -![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) - -### Add Scopes - -1. Click **Add or Remove Scopes** -2. Add the following scopes: - - `https://www.googleapis.com/auth/drive` - Full access to Google Drive (read, create, and trash files) - - `https://www.googleapis.com/auth/userinfo.email` - View user email address - - `https://www.googleapis.com/auth/userinfo.profile` - View user profile info - - `openid` - OpenID Connect authentication -3. Click **Update** and then **Save and Continue** - - - The `drive` scope is required for HITL (Human-in-the-Loop) tools like creating and trashing files. If you only need read access, you can use `drive.readonly` instead, but HITL tools will not work. - - -## Step 4: Create OAuth Client ID - -1. Go to **APIs & Services** > **Credentials** -2. Click **Create Credentials** > **OAuth client ID** -3. Select **Web application** as the application type -4. Enter **Name**: `SurfSense` -5. Under **Authorized redirect URIs**, add: - ``` - http://localhost:8000/api/v1/auth/google/drive/connector/callback - ``` -6. Click **Create** - -![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) - -## Step 5: Get OAuth Credentials - -1. After creating the OAuth client, you'll see a dialog with your credentials -2. Copy your **Client ID** and **Client Secret** - - - Never share your client secret publicly. - - -![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) - ---- - -## Running SurfSense with Google Drive Connector - -Add the Google OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -GOOGLE_OAUTH_CLIENT_ID=your_google_client_id -GOOGLE_OAUTH_CLIENT_SECRET=your_google_client_secret -GOOGLE_DRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/google/drive/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/index.mdx b/surfsense_web/content/docs/connectors/index.mdx index 9b8fa5f93..1ee9209e6 100644 --- a/surfsense_web/content/docs/connectors/index.mdx +++ b/surfsense_web/content/docs/connectors/index.mdx @@ -1,121 +1,36 @@ --- title: Connectors -description: Integrate with third-party services +description: Connect SurfSense to your tools and services --- import { Card, Cards } from 'fumadocs-ui/components/card'; +import { Zap, Cable } from 'lucide-react'; -Connect SurfSense to your favorite tools and services. Browse the available integrations below to sync data from productivity apps, communication platforms, knowledge bases, and more. +Connectors bring data into SurfSense — either as searchable knowledge or as live tools the AI agent can use during chat. There are two kinds: } + title="Native Connectors" + description="SurfSense's built-in scraper APIs: Reddit, YouTube, Google Maps, Google Search, and Web Crawl — usable in chat, the API Playground, or via REST" + href="/docs/connectors/native" /> - - - - - - - - - - - - - - - - - - - - } + title="External Connectors" + description="Third-party integrations: Notion, Slack, Linear, Jira, Google, GitHub, Obsidian, and more" + href="/docs/connectors/external" /> + +## How connectors surface in SurfSense + +- **Native scrapers** — the AI agent uses them as tools automatically in chat, and you can run them yourself from the **API Playground** or your own code via the REST API. +- **Tools & Live Sources** — external connectors like Notion, Slack, Jira, or Linear give the agent real-time tools. Ask a question in chat and the agent searches or acts on the connected service directly; nothing is copied into SurfSense in the background. +- **Knowledge Base** — file sources like Google Drive, OneDrive, and Dropbox are imported through the Documents sidebar's **Import** menu and indexed into your knowledge base alongside uploads and notes. + +To manage external connectors, open the **Connectors** dialog inside your workspace. Connected accounts, sync status, and disconnect options all live there. + +## Deprecated connectors + +Some older connectors (Discord, Microsoft Teams, Luma, Elasticsearch, the old web crawler, and standalone search APIs) have been retired. Existing connections remain manageable, but new ones can't be added. Their docs are kept for reference under [Deprecated](/docs/connectors/deprecated/discord). Public web search and crawling are now built in as [native scrapers](/docs/connectors/native), and other search APIs can still be wired up through a custom MCP server. diff --git a/surfsense_web/content/docs/connectors/jira.mdx b/surfsense_web/content/docs/connectors/jira.mdx deleted file mode 100644 index 9eb6576c3..000000000 --- a/surfsense_web/content/docs/connectors/jira.mdx +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: Jira -description: Connect your Jira projects to SurfSense ---- - -# Jira OAuth Integration Setup Guide - -This guide walks you through setting up an Atlassian OAuth 2.0 (3LO) integration for SurfSense to connect your Jira projects. - -## Step 1: Access the Developer Console - -1. Navigate to [developer.atlassian.com](https://developer.atlassian.com) -2. Click your profile icon in the top-right corner -3. Select **"Developer console"** from the dropdown - -![Atlassian Developer Console Access](/docs/connectors/atlassian/atlassian-dev-console-access.png) - -## Step 2: Create a New OAuth 2.0 Integration - -1. In the Developer Console, under **My apps**, click the **"Create"** button -2. Select **"OAuth 2.0 integration"** from the dropdown - -![Create OAuth 2.0 Integration](/docs/connectors/atlassian/atlassian-create-app.png) - -## Step 3: Name Your Integration - -1. Enter **Name**: `SurfSense` -2. Check the box to agree to Atlassian's developer terms -3. Click **"Create"** - - - New OAuth 2.0 integrations use rotating refresh tokens, which improve security by limiting token validity and enabling automatic detection of token reuse. - - -![Create New Integration Form](/docs/connectors/atlassian/atlassian-name-integration.png) - -## Step 4: Configure Callback URL - -1. In the left sidebar, click **"Authorization"** -2. Under **Callback URLs**, enter the redirect URI: - ``` - http://localhost:8000/api/v1/auth/jira/connector/callback - ``` -3. Click **"Save changes"** - - - You can enter up to 10 redirect URIs, one per line. - - -![Authorization Callback URLs](/docs/connectors/atlassian/atlassian-authorization.png) - -## Step 5: Configure API Permissions - -1. In the left sidebar, click **"Permissions"** -2. You'll see a list of available APIs including Jira API - -![Permissions Overview](/docs/connectors/atlassian/atlassian-permissions.png) - -## Step 6: Configure Jira API Scopes - -1. Click **"Configure"** next to **Jira API** -2. Select the **"Classic scopes"** tab -3. Under **Jira platform REST API**, select the following scopes: - -| Scope Name | Code | Description | -|------------|------|-------------| -| View Jira issue data | `read:jira-work` | Read Jira project and issue data, search for issues, and objects associated with issues like attachments and worklogs | -| View user profiles | `read:jira-user` | View user information in Jira that the user has access to, including usernames, email addresses, and avatars | -| Create and manage issues | `write:jira-work` | Create, edit, and delete issues, comments, worklogs, and other Jira data the user has access to | - -4. Click **"Save"** - -![Jira API Scopes](/docs/connectors/atlassian/jira/atlassian-jira-scopes.png) - -## Step 7: Get OAuth Credentials - -1. In the left sidebar, click **"Settings"** -2. Copy your **Client ID** and **Client Secret** - - - Never share your client secret publicly. - - ---- - -## Running SurfSense with Jira Connector - -Add the Atlassian credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -ATLASSIAN_CLIENT_ID=your_atlassian_client_id -ATLASSIAN_CLIENT_SECRET=your_atlassian_client_secret -JIRA_REDIRECT_URI=http://localhost:8000/api/v1/auth/jira/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/linear.mdx b/surfsense_web/content/docs/connectors/linear.mdx deleted file mode 100644 index 1dd5af9d5..000000000 --- a/surfsense_web/content/docs/connectors/linear.mdx +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Linear -description: Connect your Linear workspace to SurfSense ---- - -# Linear OAuth Integration Setup Guide - -This guide walks you through setting up a Linear OAuth integration for SurfSense. - -## Step 1: Access Linear API Settings - -1. Navigate to your workspace's API settings at `linear.app//settings/api` -2. Under **OAuth Applications**, click **"+ New OAuth application"** - -![Linear API Settings Page](/docs/connectors/linear/linear-api-settings.png) - -## Step 2: Create New Application - -Fill in the application details: - -| Field | Value | -|-------|-------| -| **Application icon** | Upload an icon (at least 256x256px) | -| **Application name** | `SurfSense` | -| **Developer name** | `SurfSense` | -| **Developer URL** | `https://www.surfsense.com/` | -| **Description** | Connect any LLM to your internal knowledge sources and chat with it in real time alongside your team. | -| **Callback URLs** | `http://localhost:8000/api/v1/auth/linear/connector/callback` | -| **GitHub username** | Your GitHub username (optional) | - -### Settings - -- **Public** - Enable this to allow the application to be installed by other workspaces - -Click **Create** to create the application. - -![Create New Application Form](/docs/connectors/linear/linear-new-application.png) - -## Step 3: Get OAuth Credentials - -After creating the application, you'll see your OAuth credentials: - -1. Copy your **Client ID** -2. Copy your **Client Secret** - - - Never share your client secret publicly. - - -![OAuth Credentials](/docs/connectors/linear/linear-oauth-credentials.png) - ---- - -## Running SurfSense with Linear Connector - -Add the Linear credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -LINEAR_CLIENT_ID=your_linear_client_id -LINEAR_CLIENT_SECRET=your_linear_client_secret -LINEAR_REDIRECT_URI=http://localhost:8000/api/v1/auth/linear/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` - diff --git a/surfsense_web/content/docs/connectors/meta.json b/surfsense_web/content/docs/connectors/meta.json index d61651575..3ae8f5f72 100644 --- a/surfsense_web/content/docs/connectors/meta.json +++ b/surfsense_web/content/docs/connectors/meta.json @@ -1,28 +1,6 @@ { "title": "Connectors", "icon": "Unplug", - "pages": [ - "google-drive", - "gmail", - "google-calendar", - "dropbox", - "notion", - "slack", - "discord", - "jira", - "linear", - "microsoft-teams", - "microsoft-onedrive", - "confluence", - "airtable", - "clickup", - "github", - "baidu-search", - "luma", - "circleback", - "elasticsearch", - "bookstack", - "obsidian" - ], + "pages": ["native", "external", "deprecated"], "defaultOpen": false } diff --git a/surfsense_web/content/docs/connectors/microsoft-onedrive.mdx b/surfsense_web/content/docs/connectors/microsoft-onedrive.mdx deleted file mode 100644 index 8bd76532e..000000000 --- a/surfsense_web/content/docs/connectors/microsoft-onedrive.mdx +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: Microsoft OneDrive -description: Connect your Microsoft OneDrive to SurfSense ---- - -# Microsoft OneDrive OAuth Integration Setup Guide - -This guide walks you through setting up a Microsoft OneDrive OAuth integration for SurfSense using Azure App Registration. - - - Microsoft OneDrive and [Microsoft Teams](/docs/connectors/microsoft-teams) share the same Azure App Registration. If you have already created an app for Teams, you can reuse the same Client ID and Client Secret. Just make sure both redirect URIs are added (see Step 3). - - -## Step 1: Access Azure App Registrations - -1. Navigate to [portal.azure.com](https://portal.azure.com) -2. In the search bar, type **"app reg"** -3. Select **"App registrations"** from the Services results - -## Step 2: Create New Registration - -1. On the **App registrations** page, click **"+ New registration"** - -## Step 3: Register the Application - -Fill in the application details: - -| Field | Value | -|-------|-------| -| **Name** | `SurfSense` | -| **Supported account types** | Select **"Accounts in any organizational directory (Any Microsoft Entra ID tenant - Multitenant) and personal Microsoft accounts"** | -| **Redirect URI** | Platform: `Web`, URI: `http://localhost:8000/api/v1/auth/onedrive/connector/callback` | - -Click **"Register"** - -After registration, add the Teams redirect URI as well (if you plan to use the Teams connector): - -1. Go to **Authentication** in the left sidebar -2. Under **Platform configurations** > **Web** > **Redirect URIs**, click **Add URI** -3. Add: `http://localhost:8000/api/v1/auth/teams/connector/callback` -4. Click **Save** - -## Step 4: Get Application (Client) ID - -After registration, you will be taken to the app's **Overview** page. Here you will find: - -1. Copy the **Application (client) ID** - this is your Client ID -2. Note the **Directory (tenant) ID** if needed - -## Step 5: Create Client Secret - -1. In the left sidebar under **Manage**, click **"Certificates & secrets"** -2. Select the **"Client secrets"** tab -3. Click **"+ New client secret"** -4. Enter a description (e.g., `SurfSense`) and select an expiration period -5. Click **"Add"** -6. **Important**: Copy the secret **Value** immediately. It will not be shown again! - - - Never share your client secret publicly or include it in code repositories. - - -## Step 6: Configure API Permissions - -1. In the left sidebar under **Manage**, click **"API permissions"** -2. Click **"+ Add a permission"** -3. Select **"Microsoft Graph"** -4. Select **"Delegated permissions"** -5. Add the following permissions: - -| Permission | Type | Description | Admin Consent | -|------------|------|-------------|---------------| -| `Files.Read.All` | Delegated | Read all files the user can access | No | -| `Files.ReadWrite.All` | Delegated | Read and write all files the user can access | No | -| `offline_access` | Delegated | Maintain access to data you have given it access to | No | -| `User.Read` | Delegated | Sign in and read user profile | No | - -6. Click **"Add permissions"** - - - All four permissions listed above are required. The connector will not authenticate successfully if any are missing. - - ---- - -## Running SurfSense with Microsoft OneDrive Connector - -Add the Microsoft OAuth credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -MICROSOFT_CLIENT_ID=your_microsoft_client_id -MICROSOFT_CLIENT_SECRET=your_microsoft_client_secret -ONEDRIVE_REDIRECT_URI=http://localhost:8000/api/v1/auth/onedrive/connector/callback -``` - - - The `MICROSOFT_CLIENT_ID` and `MICROSOFT_CLIENT_SECRET` are shared between the OneDrive and Teams connectors. You only need to set them once. - - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/native/google-maps.mdx b/surfsense_web/content/docs/connectors/native/google-maps.mdx new file mode 100644 index 000000000..6b4ad87d2 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/google-maps.mdx @@ -0,0 +1,64 @@ +--- +title: Google Maps +description: Scrape public Google Maps places and reviews +--- + +The Google Maps scraper has two verbs: **scrape** for place data and **reviews** for a place's review feed. + +## Scrape places + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/google_maps/scrape +``` + +Give it search queries (optionally scoped by location), Google Maps URLs, or place IDs. Returns structured place items — name, address, category, phone, website, rating, review count, coordinates, and opening hours. + +At least one of `search_queries`, `urls`, or `place_ids` is required. + +| Field | Default | Description | +|-------|---------|-------------| +| `search_queries` | — | Search terms, e.g. `"coffee shops"` (max 20 sources per call, combined with URLs and place IDs) | +| `urls` | — | Place page (`/maps/place/...`) or search-results URLs | +| `place_ids` | — | Known Google place IDs (`ChIJ...`) | +| `location` | — | Scope for search queries, e.g. `"New York, USA"` | +| `max_places` | `10` | Max places per search query (max 1000) | +| `include_details` | `false` | Also fetch each place's detail page: opening hours, popular times, extra contact info (slower) | +| `max_reviews` | `0` | Reviews to attach per place | +| `max_images` | `0` | Images to attach per place | +| `language` | `en` | Result language code | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/google_maps/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "search_queries": ["specialty coffee"], + "location": "Amsterdam, Netherlands", + "max_places": 20, + "include_details": true + }' +``` + +This verb is dual-metered: billed per returned place, plus per attached review when `max_reviews > 0`. + +## Fetch reviews + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/google_maps/reviews +``` + +Give it place URLs or place IDs; returns review items with author, text, star rating, like count, owner response, and timestamps. + +At least one of `urls` or `place_ids` is required. + +| Field | Default | Description | +|-------|---------|-------------| +| `urls` / `place_ids` | — | Up to 20 places per call | +| `max_reviews` | `20` | Max reviews per place (max 100,000) | +| `sort_by` | `newest` | `newest`, `mostRelevant`, `highestRanking`, or `lowestRanking` | +| `start_date` | — | Only reviews on/after this ISO date | +| `language` | `en` | Review language code | + +Billing is per returned review. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Google Maps** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/google-search.mdx b/surfsense_web/content/docs/connectors/native/google-search.mdx new file mode 100644 index 000000000..0b9f6fd11 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/google-search.mdx @@ -0,0 +1,39 @@ +--- +title: Google Search +description: Search Google and get structured SERP results +--- + +The Google Search scraper runs searches and returns structured SERP data: organic results (title, URL, description), related queries, people-also-ask questions, and any AI overview. + +## Endpoint + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/google_search/scrape +``` + +## Inputs + +| Field | Default | Description | +|-------|---------|-------------| +| `queries` | required | 1–20 search terms or full Google Search URLs — terms are searched, URLs are scraped as-is | +| `max_pages_per_query` | `1` | Result pages to fetch per query (max 10) | +| `country_code` | — | Two-letter country to search from, e.g. `us`, `fr` | +| `language_code` | — | Result language code (blank = Google default) | +| `site` | — | Restrict results to a single domain, e.g. `example.com` | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/google_search/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "queries": ["open source notebooklm alternative"], + "max_pages_per_query": 2, + "country_code": "us" + }' +``` + +The response is `{ "items": [...] }` — one item per fetched SERP page. Billing is per SERP page. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Google Search → Scrape** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/index.mdx b/surfsense_web/content/docs/connectors/native/index.mdx new file mode 100644 index 000000000..a29c6b887 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/index.mdx @@ -0,0 +1,91 @@ +--- +title: Native Connectors +description: SurfSense's built-in scraper APIs for Reddit, YouTube, Google Maps, Google Search, and the web +--- + +import { Card, Cards } from 'fumadocs-ui/components/card'; + +Native connectors are SurfSense's own scraper APIs — built into the platform, no third-party account or OAuth app required. They pull structured, public data from the platforms below and return clean JSON. + + + + + + + + + +## Three ways to use them + +Every scraper is available through the same three doors: + +1. **In chat** — the AI agent uses these scrapers as tools automatically. Ask "what is r/selfhosted saying about SurfSense?" and the agent runs the Reddit scraper for you. +2. **API Playground** — open **API Playground** in your workspace sidebar, pick a scraper, fill in the form, and run it interactively. Great for exploring what a scraper returns before writing code. +3. **REST API** — call the scrapers from your own code. Each one is a single `POST`: + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/{platform}/{verb} +Authorization: Bearer +``` + +The playground's **API reference** section on every scraper page generates ready-to-paste snippets (cURL, Python, JavaScript, Go, and more) with your workspace ID already filled in, plus the full input and output JSON schemas. + +## API keys + +To call the REST API you need two things, both under **API Playground → API Keys**: + +1. Toggle **API key access** on for the workspace. +2. Create a personal API key and send it as an `Authorization: Bearer` header. + +## Sync and async runs + +By default a `POST` blocks until the scrape finishes and returns the results. For long scrapes, append `?mode=async` — you get a `202` with a `run_id` immediately, then: + +- **Stream progress**: `GET .../scrapers/runs/{run_id}/events` (Server-Sent Events, ends with a `run.finished` event) +- **Fetch the result**: `GET .../scrapers/runs/{run_id}` +- **Cancel**: `POST .../scrapers/runs/{run_id}/cancel` + +## Avoiding blocks with a proxy (self-hosted) + +Scrapers make real requests to the target platforms, and heavy use from a single server IP will eventually get rate-limited or blocked. If you self-host, we recommend routing scraper traffic through a proxy — any HTTP proxy or rotating residential/datacenter gateway you already use works; SurfSense doesn't require a specific vendor. + +Set it up in your backend `.env` and restart: + +```bash +# A single endpoint (user:pass@host:port), used for all scraper traffic +PROXY_URL=http://username:password@proxy.example.com:8080 + +# Or a comma-separated pool that SurfSense rotates through per request. +# Gateways that rotate server-side just need the single PROXY_URL above. +PROXY_URLS=http://user:pass@host1:port,http://user:pass@host2:port +``` + +Leave both unset and requests go out directly from your server's IP — fine for light use. See the proxy section of `.env.example` for the full details, including `PROXY_PROVIDER` if your vendor has a built-in integration. + + +On SurfSense Cloud, proxying is already handled — nothing to configure. + + +## Runs and pricing + +Every run — from chat, the playground, or the API — is recorded under **API Playground → Runs** with its input, output, duration, and cost. Scrapers are metered per item returned (per post, video, comment, place, review, SERP page, or crawled page); the current rate is shown on each scraper's playground card, and pricing is returned by the capabilities API (`GET .../scrapers/capabilities`). diff --git a/surfsense_web/content/docs/connectors/native/meta.json b/surfsense_web/content/docs/connectors/native/meta.json new file mode 100644 index 000000000..53f991fbc --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/meta.json @@ -0,0 +1,5 @@ +{ + "title": "Native Connectors", + "pages": ["reddit", "youtube", "google-maps", "google-search", "web-crawl"], + "defaultOpen": false +} diff --git a/surfsense_web/content/docs/connectors/native/reddit.mdx b/surfsense_web/content/docs/connectors/native/reddit.mdx new file mode 100644 index 000000000..ce027c2b6 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/reddit.mdx @@ -0,0 +1,49 @@ +--- +title: Reddit +description: Scrape public Reddit posts, comments, subreddits, and users +--- + +The Reddit scraper pulls structured public data from Reddit. Give it URLs (a post, a subreddit, a user page, or a search URL) and/or search terms, and it returns posts (title, body, score, comment count, subreddit, author), their comment trees, and community/user metadata. + +## Endpoint + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/reddit/scrape +``` + +## Inputs + +At least one of `urls`, `search_queries`, or `community` is required. + +| Field | Default | Description | +|-------|---------|-------------| +| `urls` | — | Reddit URLs: a post, `/r/`, `/user/`, or a search URL (max 20 sources per call, combined with queries) | +| `search_queries` | — | Search terms to run on Reddit | +| `community` | — | Subreddit name (without `r/`) to scope searches to; with no queries, its listing is scraped | +| `sort` | `new` | `relevance`, `hot`, `top`, `new`, `rising`, or `comments` | +| `time_filter` | — | Window for `top`/`controversial`: `hour`, `day`, `week`, `month`, `year`, `all` | +| `max_items` | `10` | Max total items returned across all sources (hard cap 100) | +| `max_posts` | `10` | Max posts per subreddit/user/search target | +| `max_comments` | `10` | Max comments per post (`0` = none) | +| `skip_comments` | `false` | Skip comment trees entirely (faster) | +| `include_nsfw` | `true` | Include over-18 posts | +| `post_date_limit` / `comment_date_limit` | — | ISO dates for incremental scrapes: only return newer content | + +## Example + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/reddit/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "search_queries": ["self-hosted AI research agent"], + "community": "selfhosted", + "sort": "top", + "time_filter": "month", + "max_items": 25 + }' +``` + +The response is `{ "items": [...] }` — one item per post, comment, community, or user. Billing is per returned item. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Reddit → Scrape** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/web-crawl.mdx b/surfsense_web/content/docs/connectors/native/web-crawl.mdx new file mode 100644 index 000000000..52bf5dfe1 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/web-crawl.mdx @@ -0,0 +1,58 @@ +--- +title: Web Crawl +description: Scrape any page or crawl a whole site into clean markdown +--- + +The web crawler fetches one or more pages — or spiders a whole site — and returns each page as clean markdown, plus metadata, every link with its anchor text, and harvested contact signals (emails, phone numbers, social profiles). JS-rendered pages are loaded in a real browser and auto-scrolled, so lazy-loaded listings are captured too. + +## Endpoint + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/web/crawl +``` + +## Inputs + +| Field | Default | Description | +|-------|---------|-------------| +| `startUrls` | required | 1–20 seed URLs | +| `maxCrawlDepth` | `0` | Link-hops to follow from each seed. `0` = fetch only the seeds; `1` = also the pages they link to; up to 5. The spider stays on the seed's site | +| `maxCrawlPages` | `10` | Total pages fetched per call, seeds included (max 200) | +| `maxLength` | `50000` | Max characters of markdown kept per page | +| `includeUrlPatterns` | — | Regexes a discovered link must match to be followed (empty = follow every same-site link) | +| `excludeUrlPatterns` | — | Regexes that exclude links from being followed (wins over include) | + +## Example + +Scrape a single page: + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/web/crawl" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ "startUrls": ["https://example.com/pricing"] }' +``` + +Crawl a site's blog, two hops deep: + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/web/crawl" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "startUrls": ["https://example.com/blog"], + "maxCrawlDepth": 2, + "maxCrawlPages": 50, + "includeUrlPatterns": ["/blog/"] + }' +``` + +## Output + +One item per fetched page (in crawl order) with its markdown, metadata (title, description), crawl provenance (depth, referrer), links classified as internal/external/social/email/tel, and page-level contacts. The response also includes a site-wide `contacts` summary that deduplicates every email, phone, and social profile found — `siteWide: true` marks header/footer values (the company's own contacts) versus page-local finds like individual team members. + +Contact details often live on about/contact/privacy pages, so crawl with `maxCrawlDepth >= 1` to surface them — useful for lead generation and competitive intelligence. + +Billing is per successfully fetched page. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → Web → Crawl** in your workspace. diff --git a/surfsense_web/content/docs/connectors/native/youtube.mdx b/surfsense_web/content/docs/connectors/native/youtube.mdx new file mode 100644 index 000000000..72c678642 --- /dev/null +++ b/surfsense_web/content/docs/connectors/native/youtube.mdx @@ -0,0 +1,54 @@ +--- +title: YouTube +description: Scrape public YouTube videos, channels, playlists, and comments +--- + +The YouTube scraper has two verbs: **scrape** for video metadata (and optionally subtitles) and **comments** for a video's comment threads. + +## Scrape videos + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/youtube/scrape +``` + +Give it YouTube URLs (video, channel `/@handle`, playlist, shorts, or hashtag pages) and/or search queries. Returns structured video items — title, views, likes, publish date, channel info, description, and optionally subtitles. + +At least one of `urls` or `search_queries` is required. + +| Field | Default | Description | +|-------|---------|-------------| +| `urls` | — | Video, channel, playlist, shorts, or hashtag URLs (max 20 sources per call, combined with queries) | +| `search_queries` | — | Search terms; each returns up to `max_results` videos | +| `max_results` | `10` | Max items per source and per content type (a channel's videos, shorts, and streams are capped independently; max 1000) | +| `download_subtitles` | `false` | Also fetch each video's subtitle track (slower) | +| `subtitles_language` | `en` | Subtitle language code | + +```bash +curl -X POST "$BASE_URL/api/v1/workspaces/1/scrapers/youtube/scrape" \ + -H "Authorization: Bearer $SURFSENSE_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "urls": ["https://www.youtube.com/@surfsense"], + "max_results": 20 + }' +``` + +Billing is per returned video/short/stream. + +## Fetch comments + +```bash +POST /api/v1/workspaces/{workspace_id}/scrapers/youtube/comments +``` + +Give it video URLs; returns comment items with author, text, like count, reply relationships, and timestamps — useful for gauging sentiment on specific videos. + +| Field | Default | Description | +|-------|---------|-------------| +| `urls` | required | 1–20 video URLs | +| `max_comments` | `20` | Max items per video, counting top-level comments and replies (max 100,000) | +| `sort_by` | `NEWEST_FIRST` | `TOP_COMMENTS` or `NEWEST_FIRST` | + +Billing is per returned comment or reply. + +For the full input and output JSON schemas and generated code snippets in your language, open **API Playground → YouTube** in your workspace. diff --git a/surfsense_web/content/docs/connectors/notion.mdx b/surfsense_web/content/docs/connectors/notion.mdx deleted file mode 100644 index 99c95d8bd..000000000 --- a/surfsense_web/content/docs/connectors/notion.mdx +++ /dev/null @@ -1,106 +0,0 @@ ---- -title: Notion -description: Connect your Notion workspaces to SurfSense ---- - -# Notion OAuth Integration Setup Guide - -This guide walks you through setting up a Notion OAuth integration for SurfSense. - -## Step 1: Access Notion Integrations - -1. Navigate to [notion.so/profile/integrations](https://notion.so/profile/integrations) -2. Click the **"New integration"** button - -![Notion Integrations Page](/docs/connectors/notion/notion-integrations-page.png) - -## Step 2: Configure New Integration - -Fill in the integration details: - -| Field | Value | -|-------|-------| -| **Integration Name** | `SurfSense` | -| **Associated workspace** | Select your workspace | -| **Type** | `Public` | -| **Company name** | Your company name | -| **Website** | Your website URL | -| **Tagline** | Brief description | -| **Privacy Policy URL** | Your privacy policy URL | -| **Terms of Use URL** | Your terms of use URL | -| **Email** | Your developer email | -| **Logo** | Upload a 512x512 logo | - -### OAuth Redirect URI - -Under **OAuth domains & URIs**, set the **Redirect URI** to: - -``` -http://localhost:8000/api/v1/auth/notion/connector/callback -``` - -Click **Save** to create the integration. - -![New Integration Form](/docs/connectors/notion/notion-new-integration-form.png) - -## Step 3: Get OAuth Credentials & Configure Capabilities - -After creating the integration, you'll see the configuration page with your credentials: - -1. Copy your **OAuth Client ID** -2. Copy your **OAuth Client Secret** (click Refresh if needed) - -### Set Required Capabilities - -Enable the following capabilities: - -| Capability Type | Required Setting | -|----------------|------------------| -| **Content Capabilities** | Read content | -| **Comment Capabilities** | Read comments | -| **User Capabilities** | Read user information including email addresses | - -Click **Save** to apply the capabilities. - -![Integration Configuration](/docs/connectors/notion/notion-integration-config.png) - ---- - -## Limitations & Unsupported Content - -Notion's API has limitations on certain block types that cannot be retrieved. SurfSense will automatically skip these unsupported blocks and continue syncing all other content. - -### Unsupported Block Types - -The following Notion features are **not accessible via the Notion API** and will be skipped during sync: - -- **Transcription blocks** - Audio/video transcriptions from Notion AI -- **AI blocks** - AI-generated content blocks - -### Learn More - -The Notion API only supports specific block types for retrieval. The official list of **supported block types** is documented in Notion's Block reference: - -- **[Block Object Reference](https://developers.notion.com/reference/block)** - Official documentation listing all supported block types. Any block type not listed here (such as `transcription` and `ai_block`) is not accessible via the Notion API. - -For additional information: -- [Working with Page Content](https://developers.notion.com/docs/working-with-page-content) - Guide on how the Notion API handles page content -- [Notion API Reference](https://developers.notion.com/reference) - Complete API documentation - ---- - -## Running SurfSense with Notion Connector - -Add the Notion credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -NOTION_OAUTH_CLIENT_ID=your_notion_client_id -NOTION_OAUTH_CLIENT_SECRET=your_notion_client_secret -NOTION_REDIRECT_URI=http://localhost:8000/api/v1/auth/notion/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/slack.mdx b/surfsense_web/content/docs/connectors/slack.mdx deleted file mode 100644 index f68d6e875..000000000 --- a/surfsense_web/content/docs/connectors/slack.mdx +++ /dev/null @@ -1,95 +0,0 @@ ---- -title: Slack -description: Connect your Slack workspace to SurfSense ---- - -# Slack OAuth Integration Setup Guide - -This guide walks you through setting up a Slack OAuth integration for SurfSense. - -## Step 1: Create a New Slack App - -1. Navigate to [api.slack.com/apps](https://api.slack.com/apps) -2. Click **"Create New App"** -3. Select **"From scratch"** to manually configure your app - -![Create an App Dialog](/docs/connectors/slack/slack-create-app.png) - -## Step 2: Name App & Choose Workspace - -1. Enter **App Name**: `SurfSense` -2. Select the workspace to develop your app in -3. Click **"Create App"** - - - You won't be able to change the workspace later. The workspace will control the app even if you leave it. - - -![Name App & Choose Workspace](/docs/connectors/slack/slack-name-workspace.png) - -## Step 3: Get App Credentials - -After creating the app, you'll be taken to the **Basic Information** page. Here you'll find your credentials: - -1. Copy your **Client ID** -2. Copy your **Client Secret** (click Show to reveal) - - - Never share your app credentials publicly. - - -![Basic Information - App Credentials](/docs/connectors/slack/slack-app-credentials.png) - -## Step 4: Configure Redirect URLs - -1. In the left sidebar, click **"OAuth & Permissions"** -2. Scroll down to **Redirect URLs** -3. Click **"Add New Redirect URL"** -4. Enter: `https://localhost:8000/api/v1/auth/slack/connector/callback` -5. Click **"Add"**, then **"Save URLs"** - -![Redirect URLs Configuration](/docs/connectors/slack/slack-redirect-urls.png) - -## Step 5: Configure Bot Token Scopes - -On the same **OAuth & Permissions** page, scroll to **Scopes** and add the following **Bot Token Scopes**: - -| OAuth Scope | Description | -|-------------|-------------| -| `channels:history` | View messages and other content in public channels | -| `channels:read` | View basic information about public channels | -| `groups:history` | View messages and other content in private channels | -| `groups:read` | View basic information about private channels | -| `im:history` | View messages and other content in direct messages | -| `mpim:history` | View messages and other content in group direct messages | -| `users:read` | View people in a workspace | - -Click **"Add an OAuth Scope"** to add each scope. - -![Bot Token Scopes](/docs/connectors/slack/slack-scopes.png) - -## Step 6: Enable Public Distribution - -1. In the left sidebar, click **"Manage Distribution"** -2. Under **Share Your App with Other Workspaces**, ensure distribution is enabled -3. You can use the **"Add to Slack"** button or **Sharable URL** to install the app - -![Manage Distribution](/docs/connectors/slack/slack-distribution.png) - ---- - -## Running SurfSense with Slack Connector - -Add the Slack credentials to your `.env` file (created during [Docker installation](/docs/docker-installation/docker-compose)): - -```bash -SLACK_CLIENT_ID=your_slack_client_id -SLACK_CLIENT_SECRET=your_slack_client_secret -SLACK_REDIRECT_URI=http://localhost:8000/api/v1/auth/slack/connector/callback -``` - -Then restart the services: - -```bash -docker compose up -d -``` diff --git a/surfsense_web/content/docs/connectors/web-crawler.mdx b/surfsense_web/content/docs/connectors/web-crawler.mdx deleted file mode 100644 index 6ea5b1c2b..000000000 --- a/surfsense_web/content/docs/connectors/web-crawler.mdx +++ /dev/null @@ -1,6 +0,0 @@ ---- -title: Web Crawler -description: Crawl and index websites with SurfSense ---- - -# Documentation in progress \ No newline at end of file diff --git a/surfsense_web/content/docs/docker-installation/dev-compose.mdx b/surfsense_web/content/docs/docker-installation/dev-compose.mdx deleted file mode 100644 index e16c6a685..000000000 --- a/surfsense_web/content/docs/docker-installation/dev-compose.mdx +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: Docker Compose Development -description: Building SurfSense from source using docker-compose.dev.yml ---- - -If you're contributing to SurfSense and want to build from source, use `docker-compose.dev.yml` instead: - -```bash -cd SurfSense/docker -docker compose -f docker-compose.dev.yml up --build -``` - -This file builds the backend and frontend from your local source code (instead -of pulling prebuilt images) and includes pgAdmin for database inspection at -[http://localhost:5050](http://localhost:5050). It intentionally keeps raw -frontend, backend, and zero-cache ports published for debugging. Use the -production `docker-compose.yml` for the default Caddy single-origin setup. - -## Dev-Only Environment Variables - -The following `.env` variables are **only used by the dev compose file** (they have no effect on the production `docker-compose.yml`): - -| Variable | Description | Default | -|----------|-------------|---------| -| `PGADMIN_PORT` | pgAdmin web UI port | `5050` | -| `PGADMIN_DEFAULT_EMAIL` | pgAdmin login email | `admin@surfsense.com` | -| `PGADMIN_DEFAULT_PASSWORD` | pgAdmin login password | `surfsense` | -| `REDIS_PORT` | Exposed Redis port (internal-only in prod) | `6379` | -| `AUTH_TYPE` | Runtime auth mode | `LOCAL` | -| `ETL_SERVICE` | Runtime document parsing service | `DOCLING` | -| `DEPLOYMENT_MODE` | Runtime deployment mode | `self-hosted` | -| `ZERO_CACHE_PORT` | Exposed zero-cache port for debugging | `4848` | - -In the production compose file, the frontend reads `AUTH_TYPE`, `ETL_SERVICE`, -and `DEPLOYMENT_MODE` at request time. Browser API and Zero traffic are -same-origin relative through bundled Caddy. -Production Docker exposes only the bundled Caddy proxy by default; dev compose -keeps direct service ports so contributors can inspect and restart individual -services without going through the proxy. diff --git a/surfsense_web/content/docs/docker-installation/docker-compose.mdx b/surfsense_web/content/docs/docker-installation/docker-compose.mdx deleted file mode 100644 index bf71c077b..000000000 --- a/surfsense_web/content/docs/docker-installation/docker-compose.mdx +++ /dev/null @@ -1,419 +0,0 @@ ---- -title: Docker Compose -description: Manual Docker Compose setup for SurfSense ---- - -## Setup - -```bash -git clone https://github.com/MODSetter/SurfSense.git -cd SurfSense/docker -cp .env.example .env -# Edit .env, at minimum set SECRET_KEY -docker compose up -d -``` - -After starting, access SurfSense at: - -- **SurfSense**: [http://localhost:3929](http://localhost:3929) -- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1) -- **Zero sync**: `ws://localhost:3929/zero` ---- - -## Configuration - -All configuration lives in a single `docker/.env` file (or `surfsense/.env` if you used the install script). Copy `.env.example` to `.env` and edit the values you need. - -### Required - -| Variable | Description | -|----------|-------------| -| `SECRET_KEY` | JWT secret key. Generate with: `openssl rand -base64 32`. Auto-generated by the install script. | - -### Core Settings - -| Variable | Description | Default | -|----------|-------------|---------| -| `SURFSENSE_VERSION` | Image tag to deploy. Use `latest`, a clean version (e.g. `0.0.14`), or a specific build (e.g. `0.0.14.1`) | `latest` | -| `SURFSENSE_VARIANT` | Backend image variant. Leave empty for CPU, set `cuda` for CUDA 12.8, or `cuda126` for CUDA 12.6. | *(empty)* | -| `AUTH_TYPE` | Authentication method: `LOCAL` (email/password) or `GOOGLE` (OAuth) | `LOCAL` | -| `ETL_SERVICE` | Document parsing: `DOCLING` (local), `UNSTRUCTURED`, or `LLAMACLOUD` | `DOCLING` | -| `EMBEDDING_MODEL` | Embedding model for vector search | `sentence-transformers/all-MiniLM-L6-v2` | -| `TTS_SERVICE` | Text-to-speech provider for podcasts | `local/kokoro` | -| `STT_SERVICE` | Speech-to-text provider for audio files | `local/base` | -| `REGISTRATION_ENABLED` | Allow new user registrations | `TRUE` | - -### Image Variants - -SurfSense publishes CPU and CUDA backend image variants. The frontend image is not variant-specific. - -| Backend tag | Use case | `SURFSENSE_VARIANT` | -|-------------|----------|---------------------| -| `:latest` | CPU-only default | *(empty)* | -| `:latest-cuda` | NVIDIA CUDA 12.8 backend image | `cuda` | -| `:latest-cuda126` | NVIDIA CUDA 12.6 backend image for older driver stacks | `cuda126` | - -All backend variants are published for `linux/amd64` and `linux/arm64`. CUDA on `linux/arm64` is best-effort. - - -GPU acceleration needs two settings: `SURFSENSE_VARIANT` selects the CUDA image, and `COMPOSE_FILE` enables the GPU device overlay. The host must have the NVIDIA Container Toolkit installed. - - -### NVIDIA GPU Acceleration - -For most NVIDIA systems, add these values to `.env` to use the CUDA 12.8 image: - -```dotenv -SURFSENSE_VARIANT=cuda -COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml -SURFSENSE_GPU_COUNT=1 -``` - -Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks or older GPUs that need the CUDA 12.6 fallback image. - -On Windows, use `;` instead of `:` in `COMPOSE_FILE` inside `.env`: - -```dotenv -COMPOSE_FILE=docker-compose.yml;docker-compose.gpu.yml -``` - -To switch variants later, edit `SURFSENSE_VARIANT` and `COMPOSE_FILE` in `.env`, then run: - -```bash -docker compose pull -docker compose up -d --wait -``` - -### Automatic Updates - -Manual Docker Compose installs do not start Watchtower automatically. To enable external automatic updates, run Watchtower separately: - -```bash -docker run -d --name watchtower \ - --restart unless-stopped \ - -v /var/run/docker.sock:/var/run/docker.sock \ - nickfedor/watchtower \ - --label-enable \ - --interval 86400 -``` - -SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services. - -### Public URL and Ports - -| Variable | Description | Default | -|----------|-------------|---------| -| `SURFSENSE_PUBLIC_URL` | Public origin used by the frontend, backend OAuth callbacks, and Zero browser URL | `http://localhost:3929` | -| `SURFSENSE_SITE_ADDRESS` | Caddy site address. `:80` means local plain HTTP; a hostname enables automatic HTTPS | `:80` | -| `LISTEN_HTTP_PORT` | Host port mapped to Caddy's HTTP listener | `3929` | -| `LISTEN_HTTPS_PORT` | Host port mapped to Caddy's HTTPS listener for domain mode | `443` | - -SurfSense includes Caddy by default. The `frontend`, `backend`, and -`zero-cache` containers are internal-only in the production compose file; the -browser reaches them through Caddy path routing. - -### Custom Domain / Automatic HTTPS - -For a real domain, point DNS at the Docker host and set: - -```dotenv -SURFSENSE_SITE_ADDRESS=surf.example.com -LISTEN_HTTP_PORT=80 -LISTEN_HTTPS_PORT=443 -CERT_EMAIL=you@example.com -SURFSENSE_PUBLIC_URL=https://surf.example.com -``` - -Caddy will issue and renew Let's Encrypt certificates automatically. Ports 80 -and 443 must be reachable from the internet for the default HTTP-01 challenge. - -| Variable | Description | -|----------|-------------| -| `CERT_EMAIL` | Optional ACME contact email | -| `CERT_ACME_CA` | ACME directory URL; use Let's Encrypt staging when testing cert issuance | -| `CERT_ACME_DNS` | DNS-01 challenge config; requires the custom Caddy build | -| `TRUSTED_PROXIES` | CIDR ranges trusted for forwarded client IP headers | -| `SURFSENSE_MAX_BODY_SIZE` | Upload limit enforced at the proxy | - -### Bring Your Own Proxy - -If you already run nginx, Traefik, Cloudflare Tunnel, or another ingress, you -can comment out the `proxy` service and route traffic to the internal services -with the same path contract: - -| Public path | Upstream | -|-------------|----------| -| `/auth/*` | `backend:8000` | -| `/api/v1/*` | `backend:8000` | -| `/zero/*` | `zero-cache:4848` | -| `/*` | `frontend:3000` | - -Alternative proxies must preserve WebSocket upgrades for `/zero`, avoid -buffering streaming responses, allow long-running requests, and support large -uploads. For DNS-01 or wildcard certificates with Caddy, build -`docker/proxy/Dockerfile` and set `CERT_ACME_DNS` for your DNS provider. - -### Zero-cache (Real-Time Sync) - -Defaults work out of the box. Change `ZERO_ADMIN_PASSWORD` for security in production. - -| Variable | Description | Default | -|----------|-------------|---------| -| `ZERO_ADMIN_PASSWORD` | Password for the zero-cache admin UI and `/statz` endpoint | `surfsense-zero-admin` | -| `ZERO_UPSTREAM_DB` | PostgreSQL connection URL for replication (must be a direct connection, not via pgbouncer) | *(built from DB_* vars)* | -| `ZERO_CVR_DB` | PostgreSQL connection URL for client view records | *(built from DB_* vars)* | -| `ZERO_CHANGE_DB` | PostgreSQL connection URL for replication log entries | *(built from DB_* vars)* | -| `ZERO_APP_PUBLICATIONS` | PostgreSQL publication restricting which tables are replicated (created by migration 116, verified by the `migrations` service before `zero-cache` starts) | `zero_publication` | -| `ZERO_NUM_SYNC_WORKERS` | Number of view-sync worker processes. Must be ≤ connection pool sizes | `4` | -| `ZERO_UPSTREAM_MAX_CONNS` | Max connections to upstream PostgreSQL for mutations | `20` | -| `ZERO_CVR_MAX_CONNS` | Max connections to the CVR database | `30` | - -### Database - -Defaults work out of the box. Change for security in production. - -| Variable | Description | Default | -|----------|-------------|---------| -| `DB_USER` | PostgreSQL username | `surfsense` | -| `DB_PASSWORD` | PostgreSQL password | `surfsense` | -| `DB_NAME` | PostgreSQL database name | `surfsense` | -| `DB_HOST` | PostgreSQL host | `db` | -| `DB_PORT` | PostgreSQL port | `5432` | -| `DB_SSLMODE` | SSL mode: `disable`, `require`, `verify-ca`, `verify-full` | `disable` | -| `DATABASE_URL` | Full connection URL override. Use for managed databases (RDS, Supabase, etc.) | *(built from above)* | - -### Authentication - -| Variable | Description | -|----------|-------------| -| `GOOGLE_OAUTH_CLIENT_ID` | Google OAuth client ID (required if `AUTH_TYPE=GOOGLE`) | -| `GOOGLE_OAUTH_CLIENT_SECRET` | Google OAuth client secret (required if `AUTH_TYPE=GOOGLE`) | - -Create credentials at the [Google Cloud Console](https://console.cloud.google.com/apis/credentials). - -### External API Keys - -| Variable | Description | -|----------|-------------| -| `UNSTRUCTURED_API_KEY` | [Unstructured.io](https://unstructured.io/) API key (required if `ETL_SERVICE=UNSTRUCTURED`) | -| `LLAMA_CLOUD_API_KEY` | [LlamaCloud](https://cloud.llamaindex.ai/) API key (required if `ETL_SERVICE=LLAMACLOUD`) | - -### Connector OAuth Keys - -Uncomment the connectors you want to use. Redirect URIs follow the single-origin -pattern `${SURFSENSE_PUBLIC_URL}/api/v1/auth//connector/callback`. -For local Docker defaults, that means -`http://localhost:3929/api/v1/auth//connector/callback`. - -| Connector | Variables | -|-----------|-----------| -| Google Drive / Gmail / Calendar | `GOOGLE_DRIVE_REDIRECT_URI`, `GOOGLE_GMAIL_REDIRECT_URI`, `GOOGLE_CALENDAR_REDIRECT_URI` | -| Notion | `NOTION_CLIENT_ID`, `NOTION_CLIENT_SECRET`, `NOTION_REDIRECT_URI` | -| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `SLACK_REDIRECT_URI` | -| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `DISCORD_REDIRECT_URI` | -| Atlassian (Jira & Confluence) | `ATLASSIAN_CLIENT_ID`, `ATLASSIAN_CLIENT_SECRET`, `JIRA_REDIRECT_URI`, `CONFLUENCE_REDIRECT_URI` | -| Linear | `LINEAR_CLIENT_ID`, `LINEAR_CLIENT_SECRET`, `LINEAR_REDIRECT_URI` | -| ClickUp | `CLICKUP_CLIENT_ID`, `CLICKUP_CLIENT_SECRET`, `CLICKUP_REDIRECT_URI` | -| Airtable | `AIRTABLE_CLIENT_ID`, `AIRTABLE_CLIENT_SECRET`, `AIRTABLE_REDIRECT_URI` | -| Microsoft (Teams & OneDrive) | `MICROSOFT_CLIENT_ID`, `MICROSOFT_CLIENT_SECRET`, `TEAMS_REDIRECT_URI`, `ONEDRIVE_REDIRECT_URI` | -| Dropbox | `DROPBOX_APP_KEY`, `DROPBOX_APP_SECRET`, `DROPBOX_REDIRECT_URI` | - -### Messaging Channels - -Configure these in the same `docker/.env` file when you want users to chat with -SurfSense from external apps. See [Messaging Channels](/docs/messaging-channels) -for full setup. - -| Channel | Variables | -|---------|-----------| -| Telegram | `TELEGRAM_SHARED_BOT_TOKEN`, `TELEGRAM_SHARED_BOT_USERNAME`, `TELEGRAM_WEBHOOK_SECRET`, `GATEWAY_BASE_URL`, `GATEWAY_TELEGRAM_INTAKE_MODE` | -| WhatsApp | `GATEWAY_WHATSAPP_INTAKE_MODE`, `WHATSAPP_SHARED_BUSINESS_TOKEN`, `WHATSAPP_SHARED_PHONE_NUMBER_ID`, `WHATSAPP_SHARED_DISPLAY_PHONE_NUMBER`, `WHATSAPP_SHARED_WABA_ID`, `WHATSAPP_WEBHOOK_VERIFY_TOKEN`, `WHATSAPP_WEBHOOK_APP_SECRET` | -| Slack | `SLACK_CLIENT_ID`, `SLACK_CLIENT_SECRET`, `GATEWAY_SLACK_ENABLED`, `GATEWAY_SLACK_SIGNING_SECRET`, `GATEWAY_SLACK_REDIRECT_URI` | -| Discord | `DISCORD_CLIENT_ID`, `DISCORD_CLIENT_SECRET`, `DISCORD_BOT_TOKEN`, `GATEWAY_DISCORD_ENABLED`, `GATEWAY_DISCORD_REDIRECT_URI` | - -### Observability (optional) - -| Variable | Description | -|----------|-------------| -| `LANGSMITH_TRACING` | Enable LangSmith tracing (`true` / `false`) | -| `LANGSMITH_ENDPOINT` | LangSmith API endpoint | -| `LANGSMITH_API_KEY` | LangSmith API key | -| `LANGSMITH_PROJECT` | LangSmith project name | - -### Advanced (optional) - -| Variable | Description | Default | -|----------|-------------|---------| -| `SCHEDULE_CHECKER_INTERVAL` | How often to check for scheduled connector tasks (e.g. `5m`, `1h`) | `5m` | -| `RERANKERS_ENABLED` | Enable document reranking for improved search | `FALSE` | -| `RERANKERS_MODEL_NAME` | Reranker model name (e.g. `ms-marco-MiniLM-L-12-v2`) | | -| `RERANKERS_MODEL_TYPE` | Reranker model type (e.g. `flashrank`) | | -| `PAGES_LIMIT` | Max pages per user for ETL services | unlimited | - ---- - -## Docker Services - -| Service | Description | -|---------|-------------| -| `proxy` | Caddy reverse proxy; the only public ingress in production Docker | -| `db` | PostgreSQL with pgvector extension | -| `migrations` | Short-lived: runs `alembic upgrade head` and verifies `zero_publication`, then exits | -| `redis` | Message broker for Celery | -| `searxng` | Local privacy-respecting search backend | -| `backend` | FastAPI application server | -| `celery_worker` | Background task processing (document indexing, etc.) | -| `celery_beat` | Periodic task scheduler (connector sync) | -| `zero-cache` | Rocicorp Zero real-time sync (replicates Postgres to clients) | -| `frontend` | Next.js web application, internal behind Caddy | - -All services start automatically with `docker compose up -d`. - -### How startup ordering works - -Schema migrations run as a dedicated `migrations` service that exits 0 on -success and non-zero on failure. Every other backend-image service gates on -it via `condition: service_completed_successfully`: - -```text -db (healthy) ──▶ migrations (alembic upgrade head + verify zero_publication) - │ - ├── exit 0 ─▶ backend ──▶ frontend - │ celery_worker - │ celery_beat - │ zero-cache ──▶ frontend - │ - └── exit ≠ 0 ─▶ compose halts the rest of the stack -``` - -This guarantees `zero-cache` only starts after `zero_publication` exists in -Postgres. Before this design, a silent migration failure would leave -`zero-cache` crash-looping with `Unknown or invalid publications. Specified: -[zero_publication]. Found: []`. - -### Readiness vs liveness - -The backend exposes two endpoints: - -- `GET /health`: lightweight liveness probe (always returns 200 if the - process is up). -- `GET /ready`: readiness probe that confirms `zero_publication` exists. - Returns 503 if not. The compose `backend.healthcheck` uses `/ready` so the - container only reports `healthy` once the schema is actually usable by - zero-cache. - -You can also monitor startup progress with `docker compose ps` (look for -`(health: starting)` → `(healthy)`). The install script polls these states -automatically and times out after 5 minutes if the stack does not converge. - ---- - -## Useful Commands - -```bash -# View logs (all services) -docker compose logs -f - -# View logs for a specific service -docker compose logs -f backend - -# Stop all services -docker compose down - -# Restart a specific service -docker compose restart backend - -# Stop and remove all containers + volumes (destructive!) -docker compose down -v -``` - ---- - -## Troubleshooting - -- **Port already in use**: Change `LISTEN_HTTP_PORT` in `.env` and restart. In domain mode, use ports `80` and `443` so Caddy can complete certificate issuance. -- **Permission errors on Linux**: You may need to prefix `docker` commands with `sudo`. -- **Real-time updates not working**: Open DevTools → Console and check for WebSocket errors. In production Docker the expected URL is `${SURFSENSE_PUBLIC_URL}/zero`. -- **Line ending issues on Windows**: Run `git config --global core.autocrlf true` before cloning. - -### Migration service exited non-zero - -The `migrations` service exits non-zero in two cases: - -1. `alembic upgrade head` failed (timeout or SQL error). -2. `alembic` succeeded but `zero_publication` is still missing from - `pg_publication`. - -Inspect the logs and the alembic state: - -```bash -docker compose logs migrations -docker compose exec db psql -U surfsense -d surfsense \ - -c 'SELECT * FROM alembic_version;' -docker compose exec db psql -U surfsense -d surfsense \ - -c 'SELECT pubname FROM pg_publication;' -``` - -The default migration timeout is 900 seconds. Slow disks (Windows / WSL2) -may need more. Set `MIGRATION_TIMEOUT` in `.env` to increase it. - -### Zero-cache stuck on `Unknown or invalid publications` - -Symptom (in `docker compose logs zero-cache`): - -```text -Error: Unknown or invalid publications. Specified: [zero_publication]. Found: [] -``` - -This means `zero-cache` started before `zero_publication` was created or the -publication does not match SurfSense's canonical Zero shape. With the current -compose files this should be impossible: the `migrations` service blocks -`zero-cache` from starting and verifies the publication before exiting -successfully. If you see it, your stack predates the fix or you brought up -`zero-cache` manually with `docker compose up zero-cache` before the migrations -service ran. - -Recovery: - -```bash -docker compose down -docker volume rm surfsense-zero-cache # wipe half-built SQLite replica -docker compose up -d # migrations runs first, then zero-cache -``` - -### Zero-cache crashes with `_zero.tableMetadata` errors - -This indicates a half-initialized SQLite replica left behind by a previous -crash. Zero's own event triggers and `ZERO_AUTO_RESET` handle schema and -replication halts automatically. If the local SQLite replica is wedged, run the -recovery one-liner above to wipe `surfsense-zero-cache`; zero-cache will -re-sync from Postgres on the next start. - -### Ensuring `wal_level = logical` - -Logical replication is required by zero-cache. The bundled -`docker/postgresql.conf` sets `wal_level = logical` automatically. If you -swap in your own config or use a managed Postgres, confirm with: - -```bash -docker compose exec db psql -U surfsense -d surfsense \ - -c "SHOW wal_level;" -``` - -### Using `docker-compose.deps-only.yml` - -`docker-compose.deps-only.yml` runs only the dependencies (Postgres, Redis, -SearXNG, zero-cache) on Docker while the backend and frontend run on the -host. Because there is no backend container in this stack, there is no -`migrations` service either, and you must run alembic on the host **before** -bringing the stack up: - -```bash -cd surfsense_backend -uv run alembic upgrade head -cd ../docker -docker compose -f docker-compose.deps-only.yml up -d -``` - -If you skip the alembic step, `zero-cache` will crash-loop with `Unknown or -invalid publications. Specified: [zero_publication]`. diff --git a/surfsense_web/content/docs/docker-installation/index.mdx b/surfsense_web/content/docs/docker-installation/index.mdx index 929192164..53a155a92 100644 --- a/surfsense_web/content/docs/docker-installation/index.mdx +++ b/surfsense_web/content/docs/docker-installation/index.mdx @@ -1,36 +1,181 @@ --- title: Docker Installation -description: Deploy SurfSense using Docker +description: Run SurfSense with Docker in minutes --- -import { Card, Cards } from 'fumadocs-ui/components/card'; +Docker is the recommended way to run SurfSense. Everything — database, backend, frontend, background workers, real-time sync, and a reverse proxy — comes pre-configured. -Choose your preferred Docker deployment method below. +**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) (or Docker Engine with Compose) must be installed and running. - - - - - - - +## Option 1: One-Line Install Script (Recommended) + +**Linux/macOS:** + +```bash +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash +``` + +**Windows (PowerShell):** + +```powershell +irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex +``` + +The script creates a `./surfsense/` directory with the compose files and a `.env`, generates a secret key, starts all services, and waits until they're healthy. It also: + +- Detects NVIDIA GPUs and asks whether to use GPU acceleration, picking the compatible backend image automatically. +- Asks whether to enable automatic daily updates via [Watchtower](https://github.com/nicholas-fedor/watchtower) (updates can download several GB in the background — pass `--no-watchtower` to skip). +- Detects a legacy all-in-one installation and migrates its data automatically (see [Updating](/docs/docker-installation/updating#migrating-from-the-all-in-one-container)). + +## Option 2: Manual Docker Compose + +```bash +git clone https://github.com/MODSetter/SurfSense.git +cd SurfSense/docker +cp .env.example .env +# Edit .env — at minimum set SECRET_KEY (generate with: openssl rand -base64 32) +docker compose up -d +``` + +## Access SurfSense + +After starting, everything is served through the bundled Caddy reverse proxy on a single origin: + +- **SurfSense**: [http://localhost:3929](http://localhost:3929) +- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1) +- **Zero sync**: `ws://localhost:3929/zero` + +Sign up with email/password (local auth is the default) and you're in. + +## Configuration + +All configuration lives in a single `.env` file — `surfsense/.env` if you used the install script, `docker/.env` if you cloned manually. The bundled `.env.example` documents every option inline with comments and working defaults; edit the values you need and restart: + +```bash +docker compose up -d +``` + +The defaults give you local email/password auth, local document parsing with Docling (no API keys), and local TTS/STT. Common things you might change, all documented in the file: + +- **Authentication** — switch to Google OAuth login. +- **Document parsing** — switch to Unstructured or LlamaCloud (both need API keys). +- **Connector credentials** — OAuth apps for [external connectors](/docs/connectors/external) that need them on self-hosted deployments. +- **Messaging channels** — Telegram, WhatsApp, Slack, and Discord bots (see [Messaging Channels](/docs/messaging-channels)). + +### NVIDIA GPU Acceleration + +Add these values to `.env` to use the CUDA backend image (the host needs the NVIDIA Container Toolkit): + +```dotenv +SURFSENSE_VARIANT=cuda +COMPOSE_FILE=docker-compose.yml:docker-compose.gpu.yml +SURFSENSE_GPU_COUNT=1 +``` + +Use `SURFSENSE_VARIANT=cuda126` for older NVIDIA driver stacks. On Windows, use `;` instead of `:` in `COMPOSE_FILE`. Then apply: + +```bash +docker compose pull && docker compose up -d --wait +``` + +### Custom Domain / Automatic HTTPS + +Point DNS at your Docker host and set in `.env`: + +```dotenv +SURFSENSE_SITE_ADDRESS=surf.example.com +LISTEN_HTTP_PORT=80 +LISTEN_HTTPS_PORT=443 +CERT_EMAIL=you@example.com +SURFSENSE_PUBLIC_URL=https://surf.example.com +``` + +Then `docker compose up -d --wait`. Caddy issues and renews Let's Encrypt certificates automatically — ports 80 and 443 must be reachable from the internet. + +### Bring Your Own Proxy + +If you already run nginx, Traefik, Cloudflare Tunnel, or another ingress, comment out the `proxy` service and route traffic to the internal services with the same path contract: + +| Public path | Upstream | +|-------------|----------| +| `/auth/*` | `backend:8000` | +| `/api/v1/*` | `backend:8000` | +| `/zero/*` | `zero-cache:4848` | +| `/*` | `frontend:3000` | + +Your proxy must preserve WebSocket upgrades for `/zero`, avoid buffering streaming responses, allow long-running requests, and support large uploads. + +## What's Running + +| Service | Description | +|---------|-------------| +| `proxy` | Caddy reverse proxy — the only public ingress | +| `db` | PostgreSQL with pgvector | +| `migrations` | Short-lived: runs database migrations, then exits | +| `redis` | Message broker for background tasks | +| `backend` | FastAPI application server | +| `celery_worker` | Background task processing (document indexing, etc.) | +| `celery_beat` | Periodic task scheduler | +| `zero-cache` | Real-time sync (replicates Postgres to browsers) | +| `frontend` | Next.js web application | + +Migrations run first on every startup; everything else waits for them to succeed, so `docker compose up -d` after an update is always safe. Monitor startup with `docker compose ps` — services go from `(health: starting)` to `(healthy)`. + +## Useful Commands + +```bash +# View logs (all services, or one) +docker compose logs -f +docker compose logs -f backend + +# Restart a service +docker compose restart backend + +# Stop all services +docker compose down + +# Stop and remove containers + volumes (destroys your data!) +docker compose down -v +``` + +To update SurfSense, see [Updating](/docs/docker-installation/updating). + +## Building from Source (Contributors) + +If you're contributing to SurfSense and want to build the images from your local checkout, use the dev compose file: + +```bash +cd SurfSense/docker +docker compose -f docker-compose.dev.yml up --build +``` + +It builds the backend and frontend from source, publishes raw service ports for debugging, and includes pgAdmin at [http://localhost:5050](http://localhost:5050). There's also a `docker-compose.deps-only.yml` that runs just the dependencies (Postgres, Redis, zero-cache) in Docker while you run the backend and frontend natively — see [Manual Installation](/docs/manual-installation#alternative-let-docker-manage-all-dependencies). + +## Troubleshooting + +- **Port already in use**: Change `LISTEN_HTTP_PORT` in `.env` and restart. In domain mode, ports 80/443 are required for certificate issuance. +- **Permission errors on Linux**: Prefix `docker` commands with `sudo`, or add your user to the `docker` group. +- **Real-time updates not working**: Open DevTools → Console and check for WebSocket errors. The expected URL is `${SURFSENSE_PUBLIC_URL}/zero`. +- **Line ending issues on Windows**: Run `git config --global core.autocrlf true` before cloning. + +### Migration service exited non-zero + +The stack halts if migrations fail. Inspect what happened: + +```bash +docker compose logs migrations +``` + +Slow disks (Windows / WSL2) may hit the default 900-second migration timeout — set `MIGRATION_TIMEOUT` in `.env` to increase it. + +### Zero-cache stuck on `Unknown or invalid publications` + +This means zero-cache started before migrations created its publication — usually only possible if you started services individually. Recover with: + +```bash +docker compose down +docker volume rm surfsense-zero-cache # wipe the half-built replica +docker compose up -d # migrations run first, then zero-cache +``` + +The same recovery fixes `_zero.tableMetadata` crashes (a half-initialized replica left behind by a previous crash). diff --git a/surfsense_web/content/docs/docker-installation/install-script.mdx b/surfsense_web/content/docs/docker-installation/install-script.mdx deleted file mode 100644 index fb7e6b5b6..000000000 --- a/surfsense_web/content/docs/docker-installation/install-script.mdx +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: One-Line Install Script -description: One-command installation of SurfSense using Docker ---- - -Downloads the compose files, generates a `SECRET_KEY`, starts all services with `docker compose up -d --wait`, and starts [Watchtower](https://github.com/nicholas-fedor/watchtower) as an external updater for automatic daily updates. - -**Prerequisites:** [Docker Desktop](https://www.docker.com/products/docker-desktop/) must be installed and running. - -### For Linux/macOS users: - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -``` - -### For Windows users (PowerShell): - -```powershell -irm https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.ps1 | iex -``` - -This creates a `./surfsense/` directory with `docker-compose.yml`, `docker-compose.gpu.yml`, and `.env`, then runs `docker compose up -d --wait`. - -If an NVIDIA GPU and NVIDIA Container Toolkit are detected, the installer asks whether to use GPU acceleration and chooses the compatible backend image automatically. Non-interactive installs default to CPU unless you pass an explicit flag. - -Interactive installs also ask whether to enable automatic daily updates with Watchtower, noting that updates may download several GB in the background. - -### GPU options - -Linux/macOS: - -```bash -# CUDA 12.8 -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda - -# CUDA 12.6 fallback for older driver stacks -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --variant=cuda126 - -# Reserve all available GPUs -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --gpu --gpu-count=all -``` - -PowerShell: - -```powershell -# Save the script locally first when passing PowerShell parameters. -.\install.ps1 -Variant cuda -.\install.ps1 -Variant cuda126 -GpuCount all -``` - -The installer writes the same `.env` settings you would configure manually: `SURFSENSE_VARIANT` selects the backend image and `COMPOSE_FILE` enables the GPU overlay. - -To skip Watchtower (e.g. in production where you manage updates yourself, or to avoid large background image downloads): - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -s -- --no-watchtower -``` - -To customise the check interval (default 24h), use `--watchtower-interval=SECONDS`. - -Manual updates use the same compose state stored in `.env`, so GPU overlays and variants are preserved: - -```bash -cd surfsense -docker compose pull -docker compose up -d --wait -``` - -If Watchtower is enabled, it preserves the running image variant tag automatically. Because SurfSense images are large, use `--no-watchtower` when you prefer to manage update timing yourself. - ---- - -## Access SurfSense - -After starting, access SurfSense at: - -- **SurfSense**: [http://localhost:3929](http://localhost:3929) -- **Backend API**: [http://localhost:3929/api/v1](http://localhost:3929/api/v1) -- **Zero sync**: `ws://localhost:3929/zero` - -The installer uses the bundled Caddy reverse proxy by default. The backend and -zero-cache containers are not published on separate host ports in the production -stack. - -For a custom domain, edit `surfsense/.env` after installation: - -```dotenv -SURFSENSE_SITE_ADDRESS=surf.example.com -LISTEN_HTTP_PORT=80 -LISTEN_HTTPS_PORT=443 -CERT_EMAIL=you@example.com -SURFSENSE_PUBLIC_URL=https://surf.example.com -``` - -Then run: - -```bash -cd surfsense -docker compose up -d --wait -``` diff --git a/surfsense_web/content/docs/docker-installation/meta.json b/surfsense_web/content/docs/docker-installation/meta.json index 13683547b..2a2b76f7b 100644 --- a/surfsense_web/content/docs/docker-installation/meta.json +++ b/surfsense_web/content/docs/docker-installation/meta.json @@ -1,6 +1,6 @@ { "title": "Docker Installation", - "pages": ["install-script", "docker-compose", "updating", "dev-compose", "migrate-from-allinone"], + "pages": ["updating"], "icon": "Container", "defaultOpen": false } diff --git a/surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx b/surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx deleted file mode 100644 index c623b59e7..000000000 --- a/surfsense_web/content/docs/docker-installation/migrate-from-allinone.mdx +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Migrate from the All-in-One Container -description: How to migrate your data from the legacy surfsense all-in-one Docker image to the current multi-container setup ---- - -The original SurfSense all-in-one image (`ghcr.io/modsetter/surfsense:latest`, run via `docker-compose.quickstart.yml`) stored all data — PostgreSQL, Redis, and configuration — in a single Docker volume named `surfsense-data`. The current setup uses separate named volumes and has upgraded PostgreSQL from **version 14 to 17**. - -Because PostgreSQL data files are not compatible between major versions, a **logical dump and restore** is required. This is a one-time migration. - - -This guide only applies to users who ran the legacy `docker-compose.quickstart.yml` (the all-in-one `surfsense` container). If you were already using `docker/docker-compose.yml`, you do not need to migrate. - - ---- - -## Option A — One command (recommended) - -`install.sh` detects the legacy `surfsense-data` volume and handles the full migration automatically — no separate migration script needed. Just run the same install command you would use for a fresh install: - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -``` - -**What it does automatically:** - -1. Downloads all SurfSense files (including `migrate-database.sh`) into `./surfsense/` -2. Detects the `surfsense-data` volume and enters migration mode -3. Stops the old all-in-one container if it is still running -4. Starts a temporary PostgreSQL 14 container and dumps your database -5. Recovers your `SECRET_KEY` from the old volume -6. Starts PostgreSQL 17, restores the dump, runs a smoke test -7. Starts all services - -Your original `surfsense-data` volume is **never deleted** — you remove it manually after verifying. - -### After it completes - -1. Open [http://localhost:3000](http://localhost:3000) and confirm your data is intact. -2. Once satisfied, remove the old volume (irreversible): - ```bash - docker volume rm surfsense-data - ``` -3. Delete the dump file once you no longer need it as a backup: - ```bash - rm ./surfsense_migration_backup.sql - ``` - -### If the migration fails mid-way - -The dump file is saved to `./surfsense_migration_backup.sql` as a checkpoint. Simply re-run `install.sh` — it will detect the existing dump and skip straight to the restore step without re-extracting. - ---- - -## Option B — Manual migration script (custom credentials) - -If you launched the old all-in-one container with custom database credentials (`POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB` environment variables), the automatic path will use wrong credentials. Run `migrate-database.sh` manually first: - -```bash -# 1. Extract data with your custom credentials -bash ./surfsense/scripts/migrate-database.sh --db-user myuser --db-password mypass --db-name mydb - -# 2. Install and restore (detects the dump automatically) -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash -``` - -Or download and run if you haven't run `install.sh` yet: - -```bash -curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/migrate-database.sh -o migrate-database.sh -bash migrate-database.sh --db-user myuser --db-password mypass --db-name mydb -``` - -### Migration script options - -| Flag | Description | Default | -|------|-------------|---------| -| `--db-user USER` | Old PostgreSQL username | `surfsense` | -| `--db-password PASS` | Old PostgreSQL password | `surfsense` | -| `--db-name NAME` | Old PostgreSQL database | `surfsense` | -| `--yes` / `-y` | Skip confirmation prompts (used automatically by `install.sh`) | — | - ---- - -## Troubleshooting - -### `install.sh` runs normally with a blank database (no migration happened) - -The legacy volume was not detected. Confirm it exists: - -```bash -docker volume ls | grep surfsense-data -``` - -If it doesn't appear, the old container may have used a different volume name. Check with: - -```bash -docker volume ls | grep -i surfsense -``` - -### Extraction fails with permission errors - -The script detects the UID of the data files and runs the temporary PG14 container as that user. If you see permission errors in `./surfsense-migration.log`, run `migrate-database.sh` manually and check the log for details. - -### Cannot find `/data/.secret_key` - -The all-in-one entrypoint always writes the key to `/data/.secret_key` unless you explicitly set `SECRET_KEY=` as an environment variable. If the key is missing, the migration script auto-generates a new one (with a warning). You can update it manually in `./surfsense/.env` afterwards. Note that a new key invalidates all existing browser sessions — users will need to log in again. - -### Restore errors after re-running `install.sh` - -If `surfsense-postgres` volume already exists from a previous partial run, remove it before retrying: - -```bash -docker volume rm surfsense-postgres -``` diff --git a/surfsense_web/content/docs/docker-installation/updating.mdx b/surfsense_web/content/docs/docker-installation/updating.mdx index 6ef2fcecc..b566a93e6 100644 --- a/surfsense_web/content/docs/docker-installation/updating.mdx +++ b/surfsense_web/content/docs/docker-installation/updating.mdx @@ -3,11 +3,20 @@ title: Updating description: How to update your SurfSense Docker deployment --- -## Watchtower Daemon (recommended) +## Manual Update -Auto-updates every 24 hours. If you used the [install script](/docs/docker-installation/install-script), Watchtower is already running. No extra setup needed. +```bash +cd surfsense # or SurfSense/docker if you cloned manually +docker compose pull && docker compose up -d +``` -For [manual Docker Compose](/docs/docker-installation/docker-compose) installs, start Watchtower separately: +Database migrations are applied automatically on every startup. GPU overlays and image variants set in `.env` are preserved. + +## Automatic Updates with Watchtower + +Auto-updates every 24 hours. If you used the [install script](/docs/docker-installation) and enabled updates, Watchtower is already running — no extra setup needed. + +For manual Docker Compose installs, start Watchtower separately: ```bash docker run -d --name watchtower \ @@ -18,7 +27,9 @@ docker run -d --name watchtower \ --interval 86400 ``` -## Watchtower One-Time Update +SurfSense containers are labeled for Watchtower, so `--label-enable` limits updates to the SurfSense services. + +For a one-time update instead of a daemon: ```bash docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ @@ -30,21 +41,32 @@ docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ Use `nickfedor/watchtower`. The original `containrrr/watchtower` is no longer maintained and may fail with newer Docker versions. -## Manual Update - -```bash -cd surfsense # or SurfSense/docker if you cloned manually -docker compose pull && docker compose up -d -``` - -Database migrations are applied automatically on every startup. - --- ## Migrating from the All-in-One Container - -If you were previously using `docker-compose.quickstart.yml` (the legacy all-in-one `surfsense` container), your data lives in a `surfsense-data` volume and requires a **one-time migration** before switching to the current setup. PostgreSQL has been upgraded from version 14 to 17, so a simple volume swap will not work. +If you previously ran the legacy all-in-one image (`ghcr.io/modsetter/surfsense:latest` via `docker-compose.quickstart.yml`), your data lives in a single `surfsense-data` volume and PostgreSQL has since been upgraded from version 14 to 17 — so a simple volume swap won't work. A one-time dump and restore is required, and the install script does it for you. -See the full step-by-step guide: [Migrate from the All-in-One Container](/docs/docker-installation/migrate-from-allinone). +Just run the normal install command: + +```bash +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash +``` + +It detects the legacy `surfsense-data` volume, stops the old container, dumps your database with a temporary PostgreSQL 14 container, recovers your `SECRET_KEY`, restores everything into PostgreSQL 17, and starts the new stack. Your original volume is **never deleted** — the dump is also saved to `./surfsense_migration_backup.sql` as a checkpoint, so if anything fails mid-way you can simply re-run the script. + +After it completes: + +1. Open SurfSense and confirm your data is intact. +2. Remove the old volume (irreversible): `docker volume rm surfsense-data` +3. Delete the dump file once you no longer need it: `rm ./surfsense_migration_backup.sql` + + +If your old container used custom database credentials (`POSTGRES_USER` / `POSTGRES_PASSWORD` / `POSTGRES_DB`), run the migration script manually first with your credentials, then run the installer: + +```bash +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/migrate-database.sh -o migrate-database.sh +bash migrate-database.sh --db-user myuser --db-password mypass --db-name mydb +curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash +``` diff --git a/surfsense_web/content/docs/how-to/zero-sync.mdx b/surfsense_web/content/docs/how-to/zero-sync.mdx index 728da9b86..d33df878a 100644 --- a/surfsense_web/content/docs/how-to/zero-sync.mdx +++ b/surfsense_web/content/docs/how-to/zero-sync.mdx @@ -57,10 +57,14 @@ If running the frontend outside Docker (e.g. `pnpm dev`), you need: -e ZERO_REPLICA_FILE=/data/zero.db \ -e ZERO_ADMIN_PASSWORD=surfsense-zero-admin \ -e ZERO_APP_PUBLICATIONS=zero_publication \ + -e ZERO_NUM_SYNC_WORKERS=4 \ + -e ZERO_UPSTREAM_MAX_CONNS=20 \ + -e ZERO_CVR_MAX_CONNS=30 \ -e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" \ -e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" \ + -e ZERO_QUERY_FORWARD_COOKIES=true \ -v surfsense-zero-cache:/data \ - rocicorp/zero:1.4.0 + rocicorp/zero:1.6.0 ``` Run `uv run alembic upgrade head` from `surfsense_backend/` **before** starting this container so the `zero_publication` exists. diff --git a/surfsense_web/content/docs/index.mdx b/surfsense_web/content/docs/index.mdx index c8540fed0..5b25d7ed4 100644 --- a/surfsense_web/content/docs/index.mdx +++ b/surfsense_web/content/docs/index.mdx @@ -5,47 +5,39 @@ icon: BookOpen --- import { Card, Cards } from 'fumadocs-ui/components/card'; -import { ClipboardCheck, Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle, Cpu } from 'lucide-react'; +import { Download, Container, Wrench, Cable, BookOpen, FlaskConical, Heart, MessageCircle, Cpu } from 'lucide-react'; -Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense. Dive in to explore how SurfSense can be your AI-powered research companion. +Welcome to **SurfSense's Documentation!** Here, you'll find everything you need to get the most out of SurfSense. + +## Getting Started + +The fastest way to run SurfSense is the [Docker installation](/docs/docker-installation), one command and you're up. If you want full control over each component (or want to contribute), use the [manual installation](/docs/manual-installation). - } - title="Prerequisites" - description="Required setup before installing SurfSense" - href="/docs/prerequisites" - /> - } - title="Installation" - description="Choose your installation method" - href="/docs/installation" - /> } title="Docker Installation" - description="Deploy SurfSense with Docker Compose" + description="One command, all dependencies pre-configured (recommended)" href="/docs/docker-installation" /> } title="Manual Installation" - description="Set up SurfSense manually from source" + description="Set up SurfSense from source, component by component" href="/docs/manual-installation" /> - } - title="Local Models" - description="Connect local model servers" - href="/docs/local-models" - /> } title="Connectors" - description="Integrate with third-party services" + description="Built-in scraper APIs plus Notion, Slack, Google, Jira, and more" href="/docs/connectors" /> + } + title="Local Models" + description="Connect local model servers like Ollama and LM Studio" + href="/docs/local-models" + /> } title="Messaging Channels" diff --git a/surfsense_web/content/docs/manual-installation.mdx b/surfsense_web/content/docs/manual-installation.mdx index f35939048..f482256e6 100644 --- a/surfsense_web/content/docs/manual-installation.mdx +++ b/surfsense_web/content/docs/manual-installation.mdx @@ -1,56 +1,45 @@ --- title: Manual Installation -description: Setting up SurfSense manually for customized deployments +description: Set up SurfSense from source, component by component icon: Wrench --- -# Manual Installation (Preferred) +This guide sets up SurfSense without Docker (except for zero-cache, which is simplest to run as a container). Choose this path if you want to contribute to SurfSense or need full control over each component. If you just want to run SurfSense, the [Docker installation](/docs/docker-installation) is much faster. -This guide provides step-by-step instructions for setting up SurfSense without Docker. This approach gives you more control over the installation process and allows for customization of the environment. +## What You'll Need -## Prerequisites +- **Python 3.12+** — backend runtime +- **Node.js 20+** and **pnpm** — frontend runtime +- **PostgreSQL 14+** with the **pgvector** extension — database +- **Redis** — message broker for background tasks +- **Docker** — to run zero-cache (the real-time sync server) +- **Git** — to clone the repository +- **uv** — Python package manager ([install instructions](https://docs.astral.sh/uv/getting-started/installation/)) -Before beginning the manual installation, ensure you have the following installed and configured: +Clone the repository first: -### Required Software -- **Python 3.12+** - Backend runtime environment -- **Node.js 20+** - Frontend runtime environment -- **PostgreSQL 14+** - Database server (must be configured with `wal_level = logical` for [Zero real-time sync](/docs/how-to/zero-sync)) -- **PGVector** - PostgreSQL extension for vector similarity search -- **Redis** - Message broker for Celery task queue -- **Zero-cache** - Rocicorp Zero real-time sync server (run via Docker; see [Zero-Cache Setup](#zero-cache-setup) below) -- **Docker** - Required to run zero-cache (the simplest way; the Postgres + Redis can be installed natively) -- **Git** - Version control (to clone the repository) +```bash +git clone https://github.com/MODSetter/SurfSense.git +cd SurfSense +``` -### Required Services & API Keys +## Decisions to Make Up Front -Complete all the [setup steps](/docs), including: +**Authentication.** SurfSense supports local email/password auth (the default, no setup needed) or Google OAuth login. For Google login you need an OAuth client from the [Google Cloud Console](https://console.cloud.google.com/apis/credentials) — the [Google connectors guide](/docs/connectors/external/google) walks through creating one. -- **Authentication Setup** (choose one): - - Google OAuth credentials (for `AUTH_TYPE=GOOGLE`) - - Local authentication setup (for `AUTH_TYPE=LOCAL`) -- **File Processing ETL Service** (choose one): - - Unstructured.io API key (Supports 34+ formats) - - LlamaCloud API key (enhanced parsing, supports 50+ formats) - - Docling (local processing, no API key required, supports PDF, Office docs, images, HTML, CSV) -- **Other API keys** as needed for your use case +**Document parsing (ETL).** SurfSense converts uploaded files with one of three services: + +- **Docling** (default) — runs locally, no API key, privacy-friendly. Supports PDF, Office docs, images, HTML, CSV. +- **Unstructured** — needs an API key from [Unstructured Platform](https://platform.unstructured.io/). Supports 34+ formats. +- **LlamaCloud** — needs an API key from [LlamaCloud](https://cloud.llamaindex.ai/). Supports 50+ formats. + +You only need one. Docling is the easiest way to start. ## Backend Setup -The backend is the core of SurfSense. Follow these steps to set it up: +### 1. Configure the Environment -### Optional: Messaging Channels - -SurfSense can expose the same backend agent through Telegram, WhatsApp, Slack, -and Discord. For manual installs, configure the relevant channel variables in -`surfsense_backend/.env`. - -See [Messaging Channels](/docs/messaging-channels) for the channel-specific -setup guides. - -### 1. Environment Configuration - -First, create and configure your environment variables by copying the example file: +Copy the example environment file: **Linux/macOS:** @@ -59,13 +48,6 @@ cd surfsense_backend cp .env.example .env ``` -**Windows (Command Prompt):** - -```cmd -cd surfsense_backend -copy .env.example .env -``` - **Windows (PowerShell):** ```powershell @@ -73,157 +55,18 @@ cd surfsense_backend Copy-Item -Path .env.example -Destination .env ``` -Edit the `.env` file and set the following variables: - -| ENV VARIABLE | DESCRIPTION | -| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| DATABASE_URL | PostgreSQL connection string (e.g., `postgresql+asyncpg://postgres:postgres@localhost:5432/surfsense`) | -| SECRET_KEY | JWT Secret key for authentication (should be a secure random string) | -| NEXT_FRONTEND_URL | URL where your frontend application is hosted (e.g., `http://localhost:3000`) | -| BACKEND_URL | (Optional) Public URL of the backend for OAuth callbacks (e.g., `https://api.yourdomain.com`). Required when running behind a reverse proxy with HTTPS. Used to set correct OAuth redirect URLs and secure cookies. | -| AUTH_TYPE | Authentication method: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication | -| GOOGLE_OAUTH_CLIENT_ID | (Optional) Client ID from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | -| GOOGLE_OAUTH_CLIENT_SECRET | (Optional) Client secret from Google Cloud Console (required if AUTH_TYPE=GOOGLE) | -| EMBEDDING_MODEL | Name of the embedding model (e.g., `sentence-transformers/all-MiniLM-L6-v2`, `openai://text-embedding-ada-002`) | -| RERANKERS_ENABLED | (Optional) Enable or disable document reranking for improved search results (e.g., `TRUE` or `FALSE`, default: `FALSE`) | -| RERANKERS_MODEL_NAME | Name of the reranker model (e.g., `ms-marco-MiniLM-L-12-v2`) (required if RERANKERS_ENABLED=TRUE) | -| RERANKERS_MODEL_TYPE | Type of reranker model (e.g., `flashrank`) (required if RERANKERS_ENABLED=TRUE) | -| TTS_SERVICE | Text-to-Speech API provider for Podcasts (e.g., `local/kokoro`, `openai/tts-1`). See [supported providers](https://docs.litellm.ai/docs/text_to_speech#supported-providers) | -| TTS_SERVICE_API_KEY | (Optional if local) API key for the Text-to-Speech service | -| TTS_SERVICE_API_BASE | (Optional) Custom API base URL for the Text-to-Speech service | -| STT_SERVICE | Speech-to-Text API provider for Audio Files (e.g., `local/base`, `openai/whisper-1`). See [supported providers](https://docs.litellm.ai/docs/audio_transcription#supported-providers) | -| STT_SERVICE_API_KEY | (Optional if local) API key for the Speech-to-Text service | -| STT_SERVICE_API_BASE | (Optional) Custom API base URL for the Speech-to-Text service | -| ETL_SERVICE | Document parsing service: `UNSTRUCTURED` (supports 34+ formats), `LLAMACLOUD` (supports 50+ formats including legacy document types), or `DOCLING` (local processing, supports PDF, Office docs, images, HTML, CSV) | -| UNSTRUCTURED_API_KEY | API key for Unstructured.io service for document parsing (required if ETL_SERVICE=UNSTRUCTURED) | -| LLAMA_CLOUD_API_KEY | API key for LlamaCloud service for document parsing (required if ETL_SERVICE=LLAMACLOUD) | -| REDIS_URL | Single Redis connection URL for the Celery broker, result backend, and app cache (e.g., `redis://localhost:6379/0`). Optionally override per use with `CELERY_BROKER_URL`, `CELERY_RESULT_BACKEND`, or `REDIS_APP_URL`. | -| SCHEDULE_CHECKER_INTERVAL | (Optional) How often to check for scheduled connector tasks. Format: `` where unit is `m` (minutes) or `h` (hours). Examples: `1m`, `5m`, `1h`, `2h` (default: `1m`) | -| REGISTRATION_ENABLED | (Optional) Enable or disable new user registration (e.g., `TRUE` or `FALSE`, default: `TRUE`) | -| PAGES_LIMIT | (Optional) Maximum pages limit per user for ETL services (default: `999999999` for unlimited in OSS version) | - -**Google Connector OAuth Configuration:** -| ENV VARIABLE | DESCRIPTION | -| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| GOOGLE_CALENDAR_REDIRECT_URI | (Optional) Redirect URI for Google Calendar connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/calendar/connector/callback`) | -| GOOGLE_GMAIL_REDIRECT_URI | (Optional) Redirect URI for Gmail connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/gmail/connector/callback`) | -| GOOGLE_DRIVE_REDIRECT_URI | (Optional) Redirect URI for Google Drive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/google/drive/connector/callback`) | - -**Connector OAuth Configurations (Optional):** - -| ENV VARIABLE | DESCRIPTION | -| -------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| AIRTABLE_CLIENT_ID | (Optional) Airtable OAuth client ID from [Airtable Developer Hub](https://airtable.com/create/oauth) | -| AIRTABLE_CLIENT_SECRET | (Optional) Airtable OAuth client secret | -| AIRTABLE_REDIRECT_URI | (Optional) Redirect URI for Airtable connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/airtable/connector/callback`) | -| CLICKUP_CLIENT_ID | (Optional) ClickUp OAuth client ID | -| CLICKUP_CLIENT_SECRET | (Optional) ClickUp OAuth client secret | -| CLICKUP_REDIRECT_URI | (Optional) Redirect URI for ClickUp connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/clickup/connector/callback`) | -| DISCORD_CLIENT_ID | (Optional) Discord OAuth client ID | -| DISCORD_CLIENT_SECRET | (Optional) Discord OAuth client secret | -| DISCORD_REDIRECT_URI | (Optional) Redirect URI for Discord connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/discord/connector/callback`) | -| DISCORD_BOT_TOKEN | (Optional) Discord bot token from Developer Portal | -| ATLASSIAN_CLIENT_ID | (Optional) Atlassian OAuth client ID (for Jira and Confluence) | -| ATLASSIAN_CLIENT_SECRET | (Optional) Atlassian OAuth client secret | -| JIRA_REDIRECT_URI | (Optional) Redirect URI for Jira connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/jira/connector/callback`) | -| CONFLUENCE_REDIRECT_URI | (Optional) Redirect URI for Confluence connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/confluence/connector/callback`) | -| LINEAR_CLIENT_ID | (Optional) Linear OAuth client ID | -| LINEAR_CLIENT_SECRET | (Optional) Linear OAuth client secret | -| LINEAR_REDIRECT_URI | (Optional) Redirect URI for Linear connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/linear/connector/callback`) | -| NOTION_CLIENT_ID | (Optional) Notion OAuth client ID | -| NOTION_CLIENT_SECRET | (Optional) Notion OAuth client secret | -| NOTION_REDIRECT_URI | (Optional) Redirect URI for Notion connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/notion/connector/callback`) | -| SLACK_CLIENT_ID | (Optional) Slack OAuth client ID | -| SLACK_CLIENT_SECRET | (Optional) Slack OAuth client secret | -| SLACK_REDIRECT_URI | (Optional) Redirect URI for Slack connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/slack/connector/callback`) | -| MICROSOFT_CLIENT_ID | (Optional) Microsoft OAuth client ID (shared for Teams and OneDrive) | -| MICROSOFT_CLIENT_SECRET | (Optional) Microsoft OAuth client secret (shared for Teams and OneDrive) | -| TEAMS_REDIRECT_URI | (Optional) Redirect URI for Teams connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/teams/connector/callback`) | -| ONEDRIVE_REDIRECT_URI | (Optional) Redirect URI for OneDrive connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/onedrive/connector/callback`) | -| DROPBOX_APP_KEY | (Optional) Dropbox OAuth app key | -| DROPBOX_APP_SECRET | (Optional) Dropbox OAuth app secret | -| DROPBOX_REDIRECT_URI | (Optional) Redirect URI for Dropbox connector OAuth callback (e.g., `http://localhost:8000/api/v1/auth/dropbox/connector/callback`) | - -**(Optional) Backend LangSmith Observability:** -| ENV VARIABLE | DESCRIPTION | -|--------------|-------------| -| LANGSMITH_TRACING | Enable LangSmith tracing (e.g., `true`) | -| LANGSMITH_ENDPOINT | LangSmith API endpoint (e.g., `https://api.smith.langchain.com`) | -| LANGSMITH_API_KEY | Your LangSmith API key | -| LANGSMITH_PROJECT | LangSmith project name (e.g., `surfsense`) | - -**(Optional) Uvicorn Server Configuration** -| ENV VARIABLE | DESCRIPTION | DEFAULT VALUE | -|------------------------------|---------------------------------------------|---------------| -| UVICORN_HOST | Host address to bind the server | 0.0.0.0 | -| UVICORN_PORT | Port to run the backend API | 8000 | -| UVICORN_LOG_LEVEL | Logging level (e.g., info, debug, warning) | info | -| UVICORN_PROXY_HEADERS | Enable/disable proxy headers | false | -| UVICORN_FORWARDED_ALLOW_IPS | Comma-separated list of allowed IPs | 127.0.0.1 | -| UVICORN_WORKERS | Number of worker processes | 1 | -| UVICORN_ACCESS_LOG | Enable/disable access log (true/false) | true | -| UVICORN_LOOP | Event loop implementation | auto | -| UVICORN_HTTP | HTTP protocol implementation | auto | -| UVICORN_WS | WebSocket protocol implementation | auto | -| UVICORN_LIFESPAN | Lifespan implementation | auto | -| UVICORN_LOG_CONFIG | Path to logging config file or empty string | | -| UVICORN_SERVER_HEADER | Enable/disable Server header | true | -| UVICORN_DATE_HEADER | Enable/disable Date header | true | -| UVICORN_LIMIT_CONCURRENCY | Max concurrent connections | | -| UVICORN_LIMIT_MAX_REQUESTS | Max requests before worker restart | | -| UVICORN_TIMEOUT_KEEP_ALIVE | Keep-alive timeout (seconds) | 5 | -| UVICORN_TIMEOUT_NOTIFY | Worker shutdown notification timeout (sec) | 30 | -| UVICORN_SSL_KEYFILE | Path to SSL key file | | -| UVICORN_SSL_CERTFILE | Path to SSL certificate file | | -| UVICORN_SSL_KEYFILE_PASSWORD | Password for SSL key file | | -| UVICORN_SSL_VERSION | SSL version | | -| UVICORN_SSL_CERT_REQS | SSL certificate requirements | | -| UVICORN_SSL_CA_CERTS | Path to CA certificates file | | -| UVICORN_SSL_CIPHERS | SSL ciphers | | -| UVICORN_HEADERS | Comma-separated list of headers | | -| UVICORN_USE_COLORS | Enable/disable colored logs | true | -| UVICORN_UDS | Unix domain socket path | | -| UVICORN_FD | File descriptor to bind to | | -| UVICORN_ROOT_PATH | Root path for the application | | - -Refer to the `.env.example` file for all available Uvicorn options and their usage. Uncomment and set in your `.env` file as needed. - -For more details, see the [Uvicorn documentation](https://www.uvicorn.org/#command-line-options). +`.env.example` is the source of truth for configuration — every variable is documented inline with comments and sensible defaults. At minimum, set your PostgreSQL connection string and a JWT secret key (generate one with `openssl rand -base64 32`). Everything else — auth type, ETL service, embeddings, TTS/STT, connector credentials — is optional and explained in the file itself. ### 2. Install Dependencies -Install the backend dependencies using `uv`: - -**Linux/macOS:** - ```bash -# Install uv if you don't have it -curl -fsSL https://astral.sh/uv/install.sh | bash - -# Install dependencies -uv sync -``` - -**Windows (PowerShell):** - -```powershell -# Install uv if you don't have it -iwr -useb https://astral.sh/uv/install.ps1 | iex - -# Install dependencies -uv sync -``` - -**Windows (Command Prompt):** - -```cmd -# Install dependencies with uv (after installing uv) +# From surfsense_backend/ uv sync ``` ### 3. Configure PostgreSQL for Zero Sync -SurfSense uses [Rocicorp Zero](https://zero.rocicorp.dev/) for real-time data synchronization (notifications, document status, chat comments, indexing progress). Zero replicates data from PostgreSQL via **logical replication**, which requires a one-time PostgreSQL configuration change. +SurfSense uses [Rocicorp Zero](https://zero.rocicorp.dev/) for real-time updates (notifications, document status, chat comments, indexing progress). Zero replicates data from PostgreSQL via **logical replication**, which requires a one-time PostgreSQL configuration change. Edit your `postgresql.conf` (typical locations: `/etc/postgresql//main/postgresql.conf` on Linux, `/usr/local/var/postgres/postgresql.conf` on macOS via Homebrew, `C:\Program Files\PostgreSQL\\data\postgresql.conf` on Windows) and set: @@ -260,7 +103,7 @@ psql -U postgres -d surfsense -c "SHOW wal_level;" # Should return: logical ``` -**Managed databases (RDS, Supabase, Cloud SQL, etc.):** Enable logical replication via your provider's parameter group (e.g. `rds.logical_replication=1` on RDS) and grant your database user the `REPLICATION` privilege: +**Managed databases (RDS, Supabase, Cloud SQL, etc.):** enable logical replication via your provider's parameter group (e.g. `rds.logical_replication=1` on RDS) and grant your database user the `REPLICATION` privilege: ```sql ALTER USER surfsense WITH REPLICATION; @@ -269,26 +112,13 @@ GRANT CREATE ON DATABASE surfsense TO surfsense; ### 4. Run Database Migrations -Before starting the backend, run Alembic migrations. This creates the schema **and** the `zero_publication` that zero-cache needs to start. Skipping this step will cause zero-cache to crash-loop with `Unknown or invalid publications. Specified: [zero_publication]`. - -**If using uv:** +This creates the schema **and** the `zero_publication` that zero-cache needs to start. Skipping this step will cause zero-cache to crash-loop with `Unknown or invalid publications. Specified: [zero_publication]`. ```bash # From surfsense_backend/ uv run alembic upgrade head ``` -**If using pip/venv:** - -```bash -# Activate virtual environment first -source .venv/bin/activate # Linux/macOS -# OR -.venv\Scripts\activate # Windows - -alembic upgrade head -``` - Verify the publication was created: ```bash @@ -296,182 +126,87 @@ psql -U postgres -d surfsense -c "SELECT pubname FROM pg_publication;" # Should include: zero_publication ``` -### 5. Start Redis Server - -Redis is required for Celery task queue. Start the Redis server: +### 5. Start Redis **Linux:** ```bash -# Start Redis server sudo systemctl start redis - -# Or if using Redis installed via package manager +# or run directly redis-server ``` -**macOS:** +**macOS (Homebrew):** ```bash -# If installed via Homebrew brew services start redis - -# Or run directly -redis-server ``` -**Windows:** +**Windows — run Redis in Docker (easiest):** ```powershell -# Option 1: If using Redis on Windows (via WSL or Windows port) -redis-server - -# Option 2: If installed as a Windows service -net start Redis -``` - -**Alternative for Windows - Run Redis in Docker:** - -If you have Docker Desktop installed, you can run Redis in a container: - -```powershell -# Pull and run Redis container docker run -d --name redis -p 6379:6379 redis:latest - -# To stop Redis -docker stop redis - -# To start Redis again -docker start redis - -# To remove Redis container -docker rm -f redis ``` -Verify Redis is running by connecting to it: +Verify it's running: ```bash redis-cli ping # Should return: PONG ``` -### 6. Start Celery Worker +### 6. Start the Celery Worker -In a new terminal window, start the Celery worker to handle background tasks. For external chat surfaces, Celery only runs maintenance tasks; agent turns run inside the FastAPI process. +In a new terminal, start the worker that handles background tasks (document indexing, connector syncs): -**If using uv:** +**Linux/macOS:** ```bash -# Make sure you're in the surfsense_backend directory cd surfsense_backend - -# Start Celery worker (consume default, connectors, and external chat maintenance queues) DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" ``` -**If using pip/venv:** +**Windows (PowerShell):** -```bash -# Make sure you're in the surfsense_backend directory +```powershell cd surfsense_backend - -# Activate virtual environment -source .venv/bin/activate # Linux/macOS -# OR -.venv\Scripts\activate # Windows - -# Start Celery worker (consume default, connectors, and external chat maintenance queues) -DEFAULT_Q="${CELERY_TASK_DEFAULT_QUEUE:-surfsense}" -celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="${DEFAULT_Q},${DEFAULT_Q}.connectors,${DEFAULT_Q}.gateway" +uv run celery -A celery_worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --queues="surfsense,surfsense.connectors,surfsense.gateway" ``` -**Optional: Start Flower for monitoring Celery tasks:** - -In another terminal window: +Optionally, run [Flower](https://flower.readthedocs.io/) in another terminal to monitor tasks at [http://localhost:5555](http://localhost:5555): ```bash -# If using uv uv run celery -A celery_worker.celery_app flower --port=5555 - -# If using pip/venv (activate venv first) -celery -A celery_worker.celery_app flower --port=5555 ``` -Access Flower at [http://localhost:5555](http://localhost:5555) to monitor your Celery tasks. - ### 7. Start Celery Beat (Scheduler) -In another new terminal window, start Celery Beat to enable periodic tasks (like scheduled connector indexing): - -**If using uv:** +In another terminal, start the scheduler that triggers periodic tasks (like scheduled connector syncs). Without it, scheduled tasks won't run. ```bash -# Make sure you're in the surfsense_backend directory cd surfsense_backend - -# Start Celery Beat uv run celery -A celery_worker.celery_app beat --loglevel=info ``` -**If using pip/venv:** - -```bash -# Make sure you're in the surfsense_backend directory -cd surfsense_backend - -# Activate virtual environment -source .venv/bin/activate # Linux/macOS -# OR -.venv\Scripts\activate # Windows - -# Start Celery Beat -celery -A celery_worker.celery_app beat --loglevel=info -``` - -**Important**: Celery Beat is required for the periodic indexing functionality to work. Without it, scheduled connector tasks won't run automatically. The schedule interval can be configured using the `SCHEDULE_CHECKER_INTERVAL` environment variable. - ### 8. Run the Backend -Start the backend server: - -**If using uv:** - ```bash -# Run without hot reloading +# From surfsense_backend/ uv run main.py # Or with hot reloading for development uv run main.py --reload ``` -**If using pip/venv:** - -```bash -# Activate virtual environment if not already activated -source .venv/bin/activate # Linux/macOS -# OR -.venv\Scripts\activate # Windows - -# Run without hot reloading -python main.py - -# Or with hot reloading for development -python main.py --reload -``` - -If everything is set up correctly, you should see output indicating the server is running on `http://localhost:8000`. +You should see the server running on `http://localhost:8000`. ## Zero-Cache Setup -**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates (notifications, document indexing status, chat comments, collaboration indicators) to all connected clients via WebSocket. The frontend connects to it on startup. Without zero-cache running, you will not see live updates and many parts of the UI will sit on stale data. - -For an overview of how Zero works and the list of synced tables, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide. +**zero-cache** is the Rocicorp Zero server that sits between PostgreSQL and the browser. It streams real-time updates to all connected clients via WebSocket. Without it, the UI sits on stale data. For an overview of how Zero works, see the [Real-Time Sync with Zero](/docs/how-to/zero-sync) guide. ### 1. Run Zero-Cache via Docker -The simplest way to run zero-cache is the official Docker image. Open a new terminal: - **Linux/macOS:** ```bash @@ -489,8 +224,9 @@ docker run -d --name surfsense-zero-cache \ -e ZERO_CVR_MAX_CONNS=30 \ -e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" \ -e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" \ + -e ZERO_QUERY_FORWARD_COOKIES=true \ -v surfsense-zero-cache:/data \ - rocicorp/zero:1.4.0 + rocicorp/zero:1.6.0 ``` **Windows (PowerShell):** @@ -510,34 +246,30 @@ docker run -d --name surfsense-zero-cache ` -e ZERO_CVR_MAX_CONNS=30 ` -e ZERO_QUERY_URL="http://host.docker.internal:3000/api/zero/query" ` -e ZERO_MUTATE_URL="http://host.docker.internal:3000/api/zero/mutate" ` + -e ZERO_QUERY_FORWARD_COOKIES=true ` -v surfsense-zero-cache:/data ` - rocicorp/zero:1.4.0 + rocicorp/zero:1.6.0 ``` -**Adjustments to make for your setup:** +**Adjustments for your setup:** -- Replace `postgres:postgres` in the connection URLs with your actual `DB_USER:DB_PASSWORD`. +- Replace `postgres:postgres` in the connection URLs with your actual database user and password. - On Linux without Docker Desktop, `host.docker.internal` may not resolve. Either keep the `--add-host=host.docker.internal:host-gateway` flag (Docker 20.10+) or replace `host.docker.internal` with your host's IP / `--network=host` + `localhost`. - For production / custom domains, set `ZERO_QUERY_URL` and `ZERO_MUTATE_URL` to your public frontend URL (e.g. `https://app.yourdomain.com/api/zero/query`). ### 2. Verify Zero-Cache -Confirm zero-cache is healthy: - ```bash curl http://localhost:4848/keepalive # Should return HTTP 200 -``` -Tail the logs to confirm initial replication completed without errors: - -```bash +# Tail logs to confirm initial replication completed without errors docker logs -f surfsense-zero-cache ``` -### Alternative: Use `docker-compose.deps-only.yml` +### Alternative: Let Docker Manage All Dependencies -If you would rather have Docker manage Postgres, Redis, SearXNG, and zero-cache together (while still running the backend and frontend natively), the repository ships a deps-only compose file. **Run alembic migrations on the host first** so `zero_publication` exists before zero-cache starts: +If you'd rather have Docker manage Postgres, Redis, and zero-cache together (while still running the backend and frontend natively), the repository ships a deps-only compose file. **Run alembic migrations on the host first** so `zero_publication` exists before zero-cache starts: ```bash cd surfsense_backend @@ -546,16 +278,11 @@ cd ../docker docker compose -f docker-compose.deps-only.yml up -d ``` -The deps-only stack exposes zero-cache on port `4848` by default. If your -frontend is not behind a reverse proxy that serves `/zero`, set -`NEXT_PUBLIC_ZERO_CACHE_URL=http://localhost:4848` before building/running the -frontend. +The deps-only stack exposes zero-cache on port `4848`. Point the frontend at it by setting the zero-cache URL in `surfsense_web/.env` (see the comments in `surfsense_web/.env.example`). ## Frontend Setup -### 1. Environment Configuration - -Set up the frontend environment: +### 1. Configure the Environment **Linux/macOS:** @@ -564,13 +291,6 @@ cd surfsense_web cp .env.example .env ``` -**Windows (Command Prompt):** - -```cmd -cd surfsense_web -copy .env.example .env -``` - **Windows (PowerShell):** ```powershell @@ -578,145 +298,57 @@ cd surfsense_web Copy-Item -Path .env.example -Destination .env ``` -Edit the `.env` file and set: +As with the backend, `.env.example` documents every option inline. Make sure the auth type and ETL service match what you configured for the backend, and that the backend URL points at `http://localhost:8000`. -| ENV VARIABLE | DESCRIPTION | -| --- | --- | -| `SURFSENSE_BACKEND_INTERNAL_URL` | Backend URL used by Next.js server routes, e.g. `http://localhost:8000` or `http://backend:8000` in Docker | -| `AUTH_TYPE` | Same value as backend auth type: `GOOGLE` for OAuth with Google, `LOCAL` for email/password authentication | -| `ETL_SERVICE` | Document parsing service (should match backend ETL_SERVICE): `UNSTRUCTURED`, `LLAMACLOUD`, or `DOCLING`; affects supported file formats in the upload interface | -| `DEPLOYMENT_MODE` | `self-hosted` or `cloud`; controls self-hosted-only connector visibility | -| `NEXT_PUBLIC_ZERO_CACHE_URL` | Only needed when the browser cannot reach Zero through same-origin `/zero`, e.g. manual local dev at `http://localhost:4848` | - -### 2. Install Dependencies - -Install the frontend dependencies: - -**Linux/macOS:** +### 2. Install Dependencies and Run ```bash # Install pnpm if you don't have it npm install -g pnpm -# Install dependencies pnpm install -``` - -**Windows:** - -```powershell -# Install pnpm if you don't have it -npm install -g pnpm - -# Install dependencies -pnpm install -``` - -### 3. Run the Frontend - -Start the Next.js development server: - -**Linux/macOS/Windows:** - -```bash pnpm run dev ``` -The frontend should now be running at `http://localhost:3000`. +The frontend should now be running at [http://localhost:3000](http://localhost:3000). -## Browser Extension Setup (Optional) +## Browser Extension (Optional) -The SurfSense browser extension allows you to save any webpage, including those protected behind authentication. - -### 1. Environment Configuration - -**Linux/macOS:** +The SurfSense browser extension saves any webpage — including those behind authentication — straight into your knowledge base. ```bash cd surfsense_browser_extension -cp .env.example .env -``` +cp .env.example .env # set the backend URL, documented inline -**Windows (Command Prompt):** - -```cmd -cd surfsense_browser_extension -copy .env.example .env -``` - -**Windows (PowerShell):** - -```powershell -cd surfsense_browser_extension -Copy-Item -Path .env.example -Destination .env -``` - -Edit the `.env` file: - -| ENV VARIABLE | DESCRIPTION | -| ------------------------- | ----------------------------------------------------- | -| PLASMO_PUBLIC_BACKEND_URL | SurfSense Backend URL (e.g., `http://127.0.0.1:8000`) | - -### 2. Build the Extension - -Build the extension for your browser using the [Plasmo framework](https://docs.plasmo.com/framework/workflows/build#with-a-specific-target). - -**Linux/macOS/Windows:** - -```bash -# Install dependencies pnpm install - -# Build for Chrome (default) -pnpm build - -# Or for other browsers -pnpm build --target=firefox -pnpm build --target=edge +pnpm build # Chrome (default) +pnpm build --target=firefox # or Firefox +pnpm build --target=edge # or Edge ``` -### 3. Load the Extension +Load the built extension in your browser's developer mode and configure it with your SurfSense API key. See the [Plasmo build docs](https://docs.plasmo.com/framework/workflows/build#with-a-specific-target) for details. -Load the extension in your browser's developer mode and configure it with your SurfSense API key. +## Verify Your Installation -## Verification - -To verify your installation: - -1. Open your browser and navigate to `http://localhost:3000` -2. Sign in with your Google account (or local credentials if `AUTH_TYPE=LOCAL`) -3. Create a workspace and try uploading a document -4. Watch the upload status update live without refreshing. This confirms zero-cache is wired up correctly -5. Test the chat functionality with your uploaded content +1. Open [http://localhost:3000](http://localhost:3000) and sign in. +2. Create a workspace and upload a document. +3. Watch the upload status update live without refreshing — this confirms zero-cache is wired up correctly. +4. Chat with your uploaded content. ## Troubleshooting -- **Database Connection Issues**: Verify your PostgreSQL server is running and pgvector is properly installed -- **Redis Connection Issues**: Ensure Redis server is running (`redis-cli ping` should return `PONG`). Check that `REDIS_URL` is correctly set in your `.env` file -- **Celery Worker Issues**: Make sure the Celery worker is running in a separate terminal. Check worker logs for any errors -- **Authentication Problems**: Check your Google OAuth configuration and ensure redirect URIs are set correctly -- **LLM Errors**: Confirm your LLM API keys are valid and the selected models are accessible -- **File Upload Failures**: Validate your ETL service API key (Unstructured.io or LlamaCloud) or ensure Docling is properly configured -- **Real-time updates not working / stale UI**: Verify zero-cache is running (`curl http://localhost:4848/keepalive` returns 200). Open browser DevTools → Console and look for WebSocket errors. In default Docker, confirm `/zero` is routed by Caddy. In manual local development, confirm `NEXT_PUBLIC_ZERO_CACHE_URL` in `surfsense_web/.env` matches the running zero-cache address. -- **Zero-cache stuck on `Unknown or invalid publications. Specified: [zero_publication]`**: You skipped (or never ran) `uv run alembic upgrade head` from `surfsense_backend/`. Run it, then restart the zero-cache container with `docker restart surfsense-zero-cache`. -- **Zero-cache crashes with `_zero.tableMetadata` errors**: A previous run left a half-built SQLite replica behind. Stop the container, remove the volume, and start fresh: `docker rm -f surfsense-zero-cache && docker volume rm surfsense-zero-cache && docker run -d ...` (re-run the command from [Zero-Cache Setup](#zero-cache-setup)). -- **`wal_level` is not set to `logical`**: zero-cache requires logical replication. Set `wal_level = logical` in `postgresql.conf`, restart PostgreSQL, and verify with `SHOW wal_level;` in psql. +- **Database connection issues**: Verify PostgreSQL is running and pgvector is installed. +- **Redis connection issues**: `redis-cli ping` should return `PONG`. Check the Redis URL in your backend `.env`. +- **Celery worker issues**: Make sure the worker is running in a separate terminal and check its logs. +- **File upload failures**: Validate your ETL service API key, or use Docling which needs none. +- **Real-time updates not working / stale UI**: Verify zero-cache is running (`curl http://localhost:4848/keepalive` returns 200). Open browser DevTools → Console and look for WebSocket errors. Confirm the zero-cache URL in `surfsense_web/.env` matches the running zero-cache address. +- **Zero-cache stuck on `Unknown or invalid publications. Specified: [zero_publication]`**: You skipped `uv run alembic upgrade head`. Run it from `surfsense_backend/`, then `docker restart surfsense-zero-cache`. +- **Zero-cache crashes with `_zero.tableMetadata` errors**: A previous run left a half-built SQLite replica behind. Start fresh: `docker rm -f surfsense-zero-cache && docker volume rm surfsense-zero-cache`, then re-run the command from [Zero-Cache Setup](#zero-cache-setup). +- **`wal_level` is not `logical`**: zero-cache requires logical replication. Set it in `postgresql.conf`, restart PostgreSQL, and verify with `SHOW wal_level;`. - **Backend `/ready` returns 503**: The readiness probe verifies `zero_publication` exists. Run `uv run alembic upgrade head` to create it. -- **Windows-specific**: If you encounter path issues, ensure you're using the correct path separator (`\` instead of `/`) -- **macOS-specific**: If you encounter permission issues, you may need to use `sudo` for some installation commands ## Next Steps -Now that you have SurfSense running locally, you can explore its features: - -- Create workspaces for organizing your content -- Upload documents or use the browser extension to save webpages -- Ask questions about your saved content -- Explore the advanced RAG capabilities - -For production deployments, consider setting up: - -- A reverse proxy like Nginx -- SSL certificates for secure connections -- Proper database backups -- User access controls +- Set up [connectors](/docs/connectors) to bring in your tools and services. +- Connect [local models](/docs/local-models) like Ollama or LM Studio. +- For production, put a reverse proxy in front, add SSL, and set up database backups. diff --git a/surfsense_web/content/docs/meta.json b/surfsense_web/content/docs/meta.json index 435e49f9f..20cd67f95 100644 --- a/surfsense_web/content/docs/meta.json +++ b/surfsense_web/content/docs/meta.json @@ -5,7 +5,6 @@ "pages": [ "---Guides---", "index", - "prerequisites", "installation", "manual-installation", "docker-installation", diff --git a/surfsense_web/content/docs/prerequisites.mdx b/surfsense_web/content/docs/prerequisites.mdx deleted file mode 100644 index 6c0450297..000000000 --- a/surfsense_web/content/docs/prerequisites.mdx +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Prerequisites -description: Required setup's before setting up SurfSense -icon: ClipboardCheck ---- - - -## Auth Setup - -SurfSense supports both Google OAuth and local email/password authentication. Google OAuth is optional - if you prefer local authentication, you can skip this section. - -**Note**: Google OAuth setup is **required** in your `.env` files if you want to use the Gmail and Google Calendar connectors in SurfSense. - -To set up Google OAuth: - -1. Login to your [Google Developer Console](https://console.cloud.google.com/) -2. Enable the required APIs: - - **People API** (required for basic Google OAuth) -![Google Developer Console People API](/docs/connectors/google/google_oauth_people_api.png) -3. Set up OAuth consent screen. -![Google Developer Console OAuth consent screen](/docs/connectors/google/google_oauth_screen.png) -4. Create OAuth client ID and secret. -![Google Developer Console OAuth client ID](/docs/connectors/google/google_oauth_client.png) -5. It should look like this. -![Google Developer Console Config](/docs/connectors/google/google_oauth_config.png) - ---- - -## File Upload's - -SurfSense supports three ETL (Extract, Transform, Load) services for converting files to LLM-friendly formats: - -### Option 1: Unstructured - -Files are converted using [Unstructured](https://github.com/Unstructured-IO/unstructured) - -1. Get an Unstructured.io API key from [Unstructured Platform](https://platform.unstructured.io/) -2. You should be able to generate API keys once registered -![Unstructured Dashboard](/docs/unstructured.png) - -### Option 2: LlamaIndex (LlamaCloud) - -Files are converted using [LlamaIndex](https://www.llamaindex.ai/) which offers 50+ file format support. - -1. Get a LlamaIndex API key from [LlamaCloud](https://cloud.llamaindex.ai/) -2. Sign up for a LlamaCloud account to access their parsing services -3. LlamaCloud provides enhanced parsing capabilities for complex documents - -### Option 3: Docling (Recommended for Privacy) - -Files are processed locally using [Docling](https://github.com/DS4SD/docling) - IBM's open-source document parsing library. - -1. **No API key required** - all processing happens locally -2. **Privacy-focused** - documents never leave your system -3. **Supported formats**: PDF, Office documents (Word, Excel, PowerPoint), images (PNG, JPEG, TIFF, BMP, WebP), HTML, CSV, AsciiDoc -4. **Enhanced features**: Advanced table detection, image extraction, and structured document parsing -5. **GPU acceleration** support for faster processing (when available) - -**Note**: You only need to set up one of these services. - ---- - -## LLM Observability (Optional) - -This is not required for SurfSense to work. But it is always a good idea to monitor LLM interactions. So we do not have those WTH moments. - -1. Get a LangSmith API key from [smith.langchain.com](https://smith.langchain.com/) -2. This helps in observing SurfSense Researcher Agent. -![LangSmith](/docs/langsmith.png) - ---- - -## Crawler - -SurfSense have 2 options for saving webpages: -- [SurfSense Extension](https://github.com/MODSetter/SurfSense/tree/main/surfsense_browser_extension) (Overall better experience & ability to save private webpages, recommended) -- Crawler (If you want to save public webpages) - -**NOTE:** SurfSense currently uses [Firecrawl.py](https://www.firecrawl.dev/) for web crawling. If you plan on using the crawler, you will need to create a Firecrawl account and get an API key. - - ---- - -## Next Steps - -Once you have all prerequisites in place, proceed to the [installation guide](/docs/installation) to set up SurfSense. \ No newline at end of file diff --git a/surfsense_web/contracts/types/scraper.types.ts b/surfsense_web/contracts/types/scraper.types.ts index 803ef4794..d31e9d4fb 100644 --- a/surfsense_web/contracts/types/scraper.types.ts +++ b/surfsense_web/contracts/types/scraper.types.ts @@ -6,6 +6,12 @@ import { z } from "zod"; * consumed by the playground's generic form renderer, the output schema by the * API reference docs — so they are intentionally untyped here. */ +/** One live per-item rate a verb charges on, e.g. 3500 micro-USD per place. */ +export const scraperPricingMeter = z.object({ + unit: z.string(), + micros_per_unit: z.number(), +}); + export const scraperCapability = z.object({ name: z.string(), description: z.string(), @@ -13,6 +19,8 @@ export const scraperCapability = z.object({ // Optional so a backend that predates output schemas degrades to just not // showing the output-schema block instead of failing the whole fetch. output_schema: z.record(z.string(), z.unknown()).optional(), + // Optional for the same backward-compat reason; empty array = free. + pricing: z.array(scraperPricingMeter).optional(), }); export const listCapabilitiesResponse = z.array(scraperCapability); @@ -50,6 +58,7 @@ export const startAsyncRunResponse = z.object({ status: z.string(), }); +export type ScraperPricingMeter = z.infer; export type ScraperCapability = z.infer; export type ScraperRunSummary = z.infer; export type ScraperRunDetail = z.infer; diff --git a/surfsense_web/lib/playground/catalog.ts b/surfsense_web/lib/playground/catalog.ts index a769c9e0c..9b508228a 100644 --- a/surfsense_web/lib/playground/catalog.ts +++ b/surfsense_web/lib/playground/catalog.ts @@ -1,11 +1,11 @@ -import { - IconBrandGoogle, - IconBrandGoogleMaps, - IconBrandReddit, - IconBrandYoutube, - IconWorldWww, -} from "@tabler/icons-react"; import type { ComponentType } from "react"; +import { + GoogleMapsIcon, + GoogleSearchIcon, + RedditIcon, + WebIcon, + YouTubeIcon, +} from "./platform-icons"; /** Icon component that accepts a ``className`` (Tabler + Lucide both satisfy this). */ export type PlatformIcon = ComponentType<{ className?: string }>; @@ -36,13 +36,13 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { id: "reddit", label: "Reddit", - icon: IconBrandReddit, + icon: RedditIcon, verbs: [{ name: "reddit.scrape", verb: "scrape", label: "Scrape" }], }, { id: "youtube", label: "YouTube", - icon: IconBrandYoutube, + icon: YouTubeIcon, verbs: [ { name: "youtube.scrape", verb: "scrape", label: "Scrape" }, { name: "youtube.comments", verb: "comments", label: "Comments" }, @@ -51,7 +51,7 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { id: "google_maps", label: "Google Maps", - icon: IconBrandGoogleMaps, + icon: GoogleMapsIcon, verbs: [ { name: "google_maps.scrape", verb: "scrape", label: "Scrape" }, { name: "google_maps.reviews", verb: "reviews", label: "Reviews" }, @@ -60,13 +60,13 @@ export const PLAYGROUND_PLATFORMS: PlaygroundPlatform[] = [ { id: "google_search", label: "Google Search", - icon: IconBrandGoogle, + icon: GoogleSearchIcon, verbs: [{ name: "google_search.scrape", verb: "scrape", label: "Scrape" }], }, { id: "web", label: "Web", - icon: IconWorldWww, + icon: WebIcon, verbs: [{ name: "web.crawl", verb: "crawl", label: "Crawl" }], }, ]; diff --git a/surfsense_web/lib/playground/format.ts b/surfsense_web/lib/playground/format.ts index fc722a1d6..0e9b75d24 100644 --- a/surfsense_web/lib/playground/format.ts +++ b/surfsense_web/lib/playground/format.ts @@ -1,11 +1,28 @@ /** Formatting helpers shared by the playground runs table and runner output. */ +import type { ScraperPricingMeter } from "@/contracts/types/scraper.types"; + export function formatCost(costMicros: number | null | undefined): string { if (costMicros == null) return "—"; if (costMicros === 0) return "Free"; return `$${(costMicros / 1_000_000).toFixed(4)}`; } +/** One meter as a per-1k rate, e.g. 3500 micros/place -> "$3.50 / 1k places". */ +export function formatRate(meter: ScraperPricingMeter): string { + const perThousand = (meter.micros_per_unit * 1000) / 1_000_000; + const dollars = Number.isInteger(perThousand) + ? perThousand.toString() + : perThousand.toFixed(2); + return `$${dollars} / 1k ${meter.unit}s`; +} + +/** A capability's full price line: "Free", one rate, or "rate + rate". */ +export function formatPricing(pricing: ScraperPricingMeter[] | undefined): string { + if (!pricing || pricing.length === 0) return "Free"; + return pricing.map(formatRate).join(" + "); +} + export function formatDuration(ms: number | null | undefined): string { if (ms == null) return "—"; if (ms < 1000) return `${ms}ms`; diff --git a/surfsense_web/lib/playground/platform-icons.tsx b/surfsense_web/lib/playground/platform-icons.tsx new file mode 100644 index 000000000..e7a443c2f --- /dev/null +++ b/surfsense_web/lib/playground/platform-icons.tsx @@ -0,0 +1,29 @@ +import Image from "next/image"; +import { cn } from "@/lib/utils"; + +/** + * Full-color brand marks for the platform-native scraper verbs, served from + * `/public/connectors/*.svg` (same asset library the connector UI uses). Each + * is a `ComponentType<{ className?: string }>` so it drops into the playground + * catalog and the composer badge exactly like a Lucide/Tabler icon. + */ +function brandIcon(src: string, alt: string) { + return function BrandIcon({ className }: { className?: string }) { + return ( + {alt} + ); + }; +} + +export const RedditIcon = brandIcon("/connectors/reddit.svg", "Reddit"); +export const YouTubeIcon = brandIcon("/connectors/youtube.svg", "YouTube"); +export const GoogleMapsIcon = brandIcon("/connectors/google-maps.svg", "Google Maps"); +export const GoogleSearchIcon = brandIcon("/connectors/google-search.svg", "Google Search"); +export const WebIcon = brandIcon("/connectors/web.svg", "Web"); diff --git a/surfsense_web/messages/en.json b/surfsense_web/messages/en.json index 130d89038..84a33a6b1 100644 --- a/surfsense_web/messages/en.json +++ b/surfsense_web/messages/en.json @@ -205,7 +205,7 @@ "manage_documents": "Manage Documents", "connectors": "Connectors", "add_connector": "Add Connector", - "manage_connectors": "Manage Connectors", + "manage_connectors": "Manage External MCP Connectors", "logs": "Logs", "all_workspaces": "All Workspaces", "team": "Team" diff --git a/surfsense_web/public/connectors/google-maps.svg b/surfsense_web/public/connectors/google-maps.svg new file mode 100644 index 000000000..5a5f8cbe9 --- /dev/null +++ b/surfsense_web/public/connectors/google-maps.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/google-search.svg b/surfsense_web/public/connectors/google-search.svg new file mode 100644 index 000000000..f4ae06ebd --- /dev/null +++ b/surfsense_web/public/connectors/google-search.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/public/connectors/reddit.svg b/surfsense_web/public/connectors/reddit.svg new file mode 100644 index 000000000..97b05bc57 --- /dev/null +++ b/surfsense_web/public/connectors/reddit.svg @@ -0,0 +1 @@ +Reddit \ No newline at end of file diff --git a/surfsense_web/public/connectors/web.svg b/surfsense_web/public/connectors/web.svg new file mode 100644 index 000000000..15a38b8c1 --- /dev/null +++ b/surfsense_web/public/connectors/web.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/surfsense_web/tests/connectors/clickup/journey.spec.ts b/surfsense_web/tests/connectors/clickup/journey.spec.ts index d82db3cb1..e24d69321 100644 --- a/surfsense_web/tests/connectors/clickup/journey.spec.ts +++ b/surfsense_web/tests/connectors/clickup/journey.spec.ts @@ -44,7 +44,7 @@ test.describe("ClickUp connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); await expect(connectorDialog.getByText("ClickUp")).toBeVisible(); diff --git a/surfsense_web/tests/connectors/composio/calendar/journey.spec.ts b/surfsense_web/tests/connectors/composio/calendar/journey.spec.ts index 36992dc40..a24fca929 100644 --- a/surfsense_web/tests/connectors/composio/calendar/journey.spec.ts +++ b/surfsense_web/tests/connectors/composio/calendar/journey.spec.ts @@ -27,7 +27,7 @@ test.describe("Composio Calendar journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); const beforeChatDocs = await listDocuments(request, apiToken, workspace.id); diff --git a/surfsense_web/tests/connectors/composio/drive/journey.spec.ts b/surfsense_web/tests/connectors/composio/drive/journey.spec.ts index fe2f6113a..db315caeb 100644 --- a/surfsense_web/tests/connectors/composio/drive/journey.spec.ts +++ b/surfsense_web/tests/connectors/composio/drive/journey.spec.ts @@ -7,7 +7,7 @@ import { } from "../../../helpers/api/connectors"; import { getEditorContent, listDocuments } from "../../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_DRIVE_FILES } from "../../../helpers/canary"; -import { openConnectorPopup } from "../../../helpers/ui/connector-popup"; +import { expectImportConnectorAvailable } from "../../../helpers/ui/connector-popup"; import { waitForDocumentByTitle, waitForIndexingComplete } from "../../../helpers/waits/indexing"; /** @@ -31,9 +31,7 @@ test.describe("Composio Drive journey", () => { await page.goto(`/dashboard/${workspace.id}/new-chat`, { waitUntil: "domcontentloaded", }); - await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); - await expect(connectorDialog).toBeVisible(); + await expectImportConnectorAvailable(page, "Google Drive"); const selectedFiles = [ { diff --git a/surfsense_web/tests/connectors/composio/gmail/journey.spec.ts b/surfsense_web/tests/connectors/composio/gmail/journey.spec.ts index 662864485..bbe0c8b12 100644 --- a/surfsense_web/tests/connectors/composio/gmail/journey.spec.ts +++ b/surfsense_web/tests/connectors/composio/gmail/journey.spec.ts @@ -27,7 +27,7 @@ test.describe("Composio Gmail journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); const beforeChatDocs = await listDocuments(request, apiToken, workspace.id); diff --git a/surfsense_web/tests/connectors/confluence/journey.spec.ts b/surfsense_web/tests/connectors/confluence/journey.spec.ts index 23fab968b..20b48fcd6 100644 --- a/surfsense_web/tests/connectors/confluence/journey.spec.ts +++ b/surfsense_web/tests/connectors/confluence/journey.spec.ts @@ -48,7 +48,7 @@ test.describe("Confluence connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); await connectorDialog.getByPlaceholder("Search").fill("Confluence"); await expect(connectorDialog.getByText("Confluence", { exact: true })).toBeVisible(); diff --git a/surfsense_web/tests/connectors/dropbox/journey.spec.ts b/surfsense_web/tests/connectors/dropbox/journey.spec.ts index b18771abb..d1aab19a8 100644 --- a/surfsense_web/tests/connectors/dropbox/journey.spec.ts +++ b/surfsense_web/tests/connectors/dropbox/journey.spec.ts @@ -3,7 +3,7 @@ import { streamChatToCompletion } from "../../helpers/api/chat"; import { listConnectors, triggerIndex, updateConnectorConfig } from "../../helpers/api/connectors"; import { getEditorContent, listDocuments } from "../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_DROPBOX_FILES } from "../../helpers/canary"; -import { openConnectorPopup } from "../../helpers/ui/connector-popup"; +import { expectImportConnectorAvailable } from "../../helpers/ui/connector-popup"; import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing"; /** @@ -32,9 +32,7 @@ test.describe("Native Dropbox journey", () => { await page.goto(`/dashboard/${workspace.id}/new-chat`, { waitUntil: "domcontentloaded", }); - await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); - await expect(connectorDialog).toBeVisible(); + await expectImportConnectorAvailable(page, "Dropbox"); const selectedFiles = [ { diff --git a/surfsense_web/tests/connectors/google/calendar/journey.spec.ts b/surfsense_web/tests/connectors/google/calendar/journey.spec.ts index d9a34649e..6874761f9 100644 --- a/surfsense_web/tests/connectors/google/calendar/journey.spec.ts +++ b/surfsense_web/tests/connectors/google/calendar/journey.spec.ts @@ -32,7 +32,7 @@ test.describe("Native Google Calendar journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); const beforeDocs = await listDocuments(request, apiToken, workspace.id); diff --git a/surfsense_web/tests/connectors/google/drive/journey.spec.ts b/surfsense_web/tests/connectors/google/drive/journey.spec.ts index e808cdb66..b9d6dc1d3 100644 --- a/surfsense_web/tests/connectors/google/drive/journey.spec.ts +++ b/surfsense_web/tests/connectors/google/drive/journey.spec.ts @@ -7,7 +7,7 @@ import { } from "../../../helpers/api/connectors"; import { getEditorContent, listDocuments } from "../../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_DRIVE_FILES } from "../../../helpers/canary"; -import { openConnectorPopup } from "../../../helpers/ui/connector-popup"; +import { expectImportConnectorAvailable } from "../../../helpers/ui/connector-popup"; import { waitForDocumentByTitle, waitForIndexingComplete } from "../../../helpers/waits/indexing"; /** @@ -36,9 +36,7 @@ test.describe("Native Google Drive journey", () => { await page.goto(`/dashboard/${workspace.id}/new-chat`, { waitUntil: "domcontentloaded", }); - await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); - await expect(connectorDialog).toBeVisible(); + await expectImportConnectorAvailable(page, "Google Drive"); const selectedFiles = [ { diff --git a/surfsense_web/tests/connectors/google/gmail/journey.spec.ts b/surfsense_web/tests/connectors/google/gmail/journey.spec.ts index 35e98aa09..67781f7f2 100644 --- a/surfsense_web/tests/connectors/google/gmail/journey.spec.ts +++ b/surfsense_web/tests/connectors/google/gmail/journey.spec.ts @@ -32,7 +32,7 @@ test.describe("Native Google Gmail journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); const beforeDocs = await listDocuments(request, apiToken, workspace.id); diff --git a/surfsense_web/tests/connectors/jira/journey.spec.ts b/surfsense_web/tests/connectors/jira/journey.spec.ts index 1a116e6a4..b6b561eac 100644 --- a/surfsense_web/tests/connectors/jira/journey.spec.ts +++ b/surfsense_web/tests/connectors/jira/journey.spec.ts @@ -42,7 +42,7 @@ test.describe("Jira connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); await connectorDialog.getByPlaceholder("Search").fill("Jira"); await expect(connectorDialog.getByText("Jira", { exact: true })).toBeVisible(); diff --git a/surfsense_web/tests/connectors/linear/journey.spec.ts b/surfsense_web/tests/connectors/linear/journey.spec.ts index 0787368a0..db1696999 100644 --- a/surfsense_web/tests/connectors/linear/journey.spec.ts +++ b/surfsense_web/tests/connectors/linear/journey.spec.ts @@ -42,7 +42,7 @@ test.describe("Linear connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); const beforeDocs = await listDocuments(request, apiToken, workspace.id); diff --git a/surfsense_web/tests/connectors/notion/journey.spec.ts b/surfsense_web/tests/connectors/notion/journey.spec.ts index d7f67c5e0..720fe4424 100644 --- a/surfsense_web/tests/connectors/notion/journey.spec.ts +++ b/surfsense_web/tests/connectors/notion/journey.spec.ts @@ -43,7 +43,7 @@ test.describe("Notion connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); await connectorDialog.getByPlaceholder("Search").fill("Notion"); await expect(connectorDialog.getByText("Notion", { exact: true })).toBeVisible(); diff --git a/surfsense_web/tests/connectors/onedrive/journey.spec.ts b/surfsense_web/tests/connectors/onedrive/journey.spec.ts index 73c671cae..1e24ebe9a 100644 --- a/surfsense_web/tests/connectors/onedrive/journey.spec.ts +++ b/surfsense_web/tests/connectors/onedrive/journey.spec.ts @@ -3,7 +3,7 @@ import { streamChatToCompletion } from "../../helpers/api/chat"; import { listConnectors, triggerIndex, updateConnectorConfig } from "../../helpers/api/connectors"; import { getEditorContent, listDocuments } from "../../helpers/api/documents"; import { CANARY_TOKENS, FAKE_ONEDRIVE_FILES } from "../../helpers/canary"; -import { openConnectorPopup } from "../../helpers/ui/connector-popup"; +import { expectImportConnectorAvailable } from "../../helpers/ui/connector-popup"; import { waitForDocumentByTitle, waitForIndexingComplete } from "../../helpers/waits/indexing"; /** @@ -32,9 +32,7 @@ test.describe("Native OneDrive journey", () => { await page.goto(`/dashboard/${workspace.id}/new-chat`, { waitUntil: "domcontentloaded", }); - await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); - await expect(connectorDialog).toBeVisible(); + await expectImportConnectorAvailable(page, "OneDrive"); const selectedFiles = [ { diff --git a/surfsense_web/tests/connectors/slack/journey.spec.ts b/surfsense_web/tests/connectors/slack/journey.spec.ts index ea5cc4b17..65283107d 100644 --- a/surfsense_web/tests/connectors/slack/journey.spec.ts +++ b/surfsense_web/tests/connectors/slack/journey.spec.ts @@ -44,7 +44,7 @@ test.describe("Slack connector journey", () => { waitUntil: "domcontentloaded", }); await openConnectorPopup(page); - const connectorDialog = page.getByRole("dialog", { name: "Manage Connectors" }); + const connectorDialog = page.getByRole("dialog", { name: "Manage External MCP Connectors" }); await expect(connectorDialog).toBeVisible(); await connectorDialog.getByPlaceholder("Search").fill("Slack"); await expect(connectorDialog.getByText("Slack", { exact: true })).toBeVisible(); diff --git a/surfsense_web/tests/helpers/ui/connector-popup.ts b/surfsense_web/tests/helpers/ui/connector-popup.ts index 19e271064..5cf8ba5d3 100644 --- a/surfsense_web/tests/helpers/ui/connector-popup.ts +++ b/surfsense_web/tests/helpers/ui/connector-popup.ts @@ -21,14 +21,27 @@ export async function openConnectorPopup(page: Page): Promise { await expect(trigger).toBeVisible({ timeout: 60_000 }); await trigger.click(); - await expect(page.getByRole("dialog", { name: "Manage Connectors" })).toBeVisible(); + await expect(page.getByRole("dialog", { name: "Manage External MCP Connectors" })).toBeVisible(); } -export async function clickComposioDriveCard(page: Page): Promise { - const composioDriveCard = page.getByText("Search your Drive files via Composio"); - await composioDriveCard.scrollIntoViewIfNeeded(); - const card = composioDriveCard - .locator("xpath=ancestor::*[self::article or self::div][1]") - .first(); - await card.getByRole("button", { name: "Connect" }).click(); +/** + * Opens the Documents sidebar "Import" menu. Import connectors + * (Google Drive / OneDrive / Dropbox) are surfaced here instead of in the + * external-MCP connector catalog. + */ +export async function openDocumentsImportMenu(page: Page): Promise { + const trigger = page.getByRole("button", { name: "Import documents" }).first(); + // Long timeout absorbs Next.js dev cold-compile of the dashboard route. + await expect(trigger).toBeVisible({ timeout: 60_000 }); + await trigger.click(); +} + +/** + * Opens the Import menu and asserts a cloud-drive import connector is offered. + * `label` is the visible menu label ("Google Drive", "OneDrive", "Dropbox"). + */ +export async function expectImportConnectorAvailable(page: Page, label: string): Promise { + await openDocumentsImportMenu(page); + await expect(page.getByRole("menuitem", { name: label })).toBeVisible(); + await page.keyboard.press("Escape"); }