From 1ea87694dab1e96686523e8aee1dcdacc0d90e19 Mon Sep 17 00:00:00 2001 From: Anish Sarkar <104695310+AnishSarkar22@users.noreply.github.com> Date: Mon, 12 Oct 2026 00:04:22 +0530 Subject: [PATCH] feat(plugins): list each scraper as its own plugin on the plugin host The first-party list holds ten scraper plugins, from Google Search to Indeed, each its own MCP server at https://plugins.surfsense.com//mcp. The host serves a folder under its id with underscores as hyphens, since a package cannot hold a hyphen, reserves scraping/ for the code they will share, and lints plugins/remote/ with ruff's ASYNC rules, since its plugins share each process's event loop. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/plugin-host.md | 13 +- docs/architecture/plugins.md | 6 +- plugins/remote/host/app.py | 2 + plugins/remote/host/mounted_plugins.py | 16 +- plugins/remote/host/tests/test_host.py | 12 +- plugins/remote/ruff.toml | 2 + .../plugins/lists/first_party/plugins.json | 160 +++++++++++++++++- .../integration/agent/test_plugin_tools.py | 10 +- .../routes/test_first_party_plugins.py | 39 +++-- .../backend/tests/mcp_test_server.py | 12 +- .../plugins/lists/test_first_party_list.py | 24 ++- .../plugins/plugins-settings.test.tsx | 24 ++- 12 files changed, 255 insertions(+), 65 deletions(-) diff --git a/docs/architecture/plugin-host.md b/docs/architecture/plugin-host.md index 6134ea026..7863f2c3a 100644 --- a/docs/architecture/plugin-host.md +++ b/docs/architecture/plugin-host.md @@ -22,9 +22,9 @@ The license check is `license_check/`, not `license/`: a `license/` folder besid [`host_app()`](../../plugins/remote/host/app.py) mounts each plugin and answers `GET /health` with `ok`. -- A plugin is a folder with a `server.py` whose `server()` returns its `MCPServer`, from the official MCP Python SDK. The folder's name is its id and its path: `proprietary/reddit/` would be served at `/reddit/mcp` ([`mounted_plugins.py`](../../plugins/remote/host/mounted_plugins.py)). +- A plugin is a folder with a `server.py` whose `server()` returns its `MCPServer`, from the official MCP Python SDK: its own MCP server at its own path. The folder's name, with underscores as hyphens, is its id and its path: `proprietary/google_search/` would be served at `/google-search/mcp`, since a Python package cannot hold a hyphen ([`mounted_plugins.py`](../../plugins/remote/host/mounted_plugins.py)). - Folders under the root and under `proprietary/` are both read. Without `proprietary/`, the open build, the host serves the Apache-2.0 plugins and has nothing else missing. -- `host`, `mcp_server`, `proprietary` and `license_check` are never plugin ids. Two plugins with one id stop the host from starting. +- `host`, `mcp_server`, `proprietary`, `license_check` and `scraping`, the code the scraper plugins will share, are never plugin ids. Two plugins with one id stop the host from starting. - A tool that raises returns an MCP error for that call; the other plugins keep answering. - The SDK's DNS-rebinding guard is off: it protects servers on a user's loopback, and the host is reached by its public name. @@ -32,7 +32,7 @@ The license check is `license_check/`, not `license/`: a `license/` folder besid ## The license check -A plugin's `server.py` may set `access`, an object with `refusal(headers)` ([`access.py`](../../plugins/remote/mcp_server/access.py)). The host asks it before every request reaches the plugin, and answers a refusal itself with its status and `{"reason": ...}` ([`refused_unless_allowed.py`](../../plugins/remote/host/refused_unless_allowed.py)). A plugin without `access` is open. +A plugin's `server.py` may set `access`, an object with `refusal(headers)` ([`access.py`](../../plugins/remote/mcp_server/access.py)). The host asks it before every request reaches the plugin, and answers a refusal itself with its status and `{"reason": ...}` ([`refused_unless_allowed.py`](../../plugins/remote/host/refused_unless_allowed.py)). A plugin without `access` is open: SurfSense offers no free plugin yet, and nothing limits an open one per IP address ([proposal](../proposals/plugins/remote/02-surfsense-servers.md#adding-a-plugin)). A paid plugin sets `access = LicenseAccess()` ([`license_access.py`](../../plugins/remote/proprietary/license_check/license_access.py)), which applies [contract 2](../contracts/02-scraper-api-auth.md): @@ -51,15 +51,16 @@ A paid plugin sets `access = LicenseAccess()` ([`license_access.py`](../../plugi ## Tests -From `plugins/remote/host`: `uv run pytest`, `uv run ruff check ..` and `uv run ruff format --check ..`. CI runs them in `plugins-pull-request-checks.yml` when `plugins/remote/` changes. +From `plugins/remote/host`: `uv run pytest`, `uv run ruff check ..` and `uv run ruff format --check ..`. The lint includes ruff's `ASYNC` rules, which fail a blocking call inside an `async def`, since every plugin answers from one process. CI runs them in `plugins-pull-request-checks.yml` when `plugins/remote/` changes. -- [`host/tests/`](../../plugins/remote/host/tests/) serve sample plugins on loopback and reach them with the official MCP client: the health check, each plugin at its path, the build without `proprietary/`, a failing tool, and reserved folders. +- [`host/tests/`](../../plugins/remote/host/tests/) serve sample plugins on loopback and reach them with the official MCP client: the health check, each plugin at its path, the build without `proprietary/`, a failing tool, an underscored folder at its hyphenated id, and reserved folders. - [`proprietary/license_check/tests/`](../../plugins/remote/proprietary/license_check/tests/) check the license against a stand-in Keygen on loopback, with the clock passed in: each answer above, the five-minute cache, the 24-hour grace, and a paid sample plugin through the host. ## Known gaps - `SURFSENSE_ACCOUNT_ID` in `keygen.py` is empty, so a paid plugin's `LicenseAccess()` stops the host from starting until it is set. -- No plugin exists: the scraper plugins, one per platform, are still in `surfsense_backend` ([proposal](../proposals/plugins/remote/02-surfsense-servers.md#the-scraper-plugins)). A folder's name is its id as-is, so a plugin such as `google_search/` would be served at `/google_search/mcp`, not `/google-search/mcp`, and `scraping` is not yet a reserved name. +- No plugin exists: the scraper plugins, one per platform, are still in `surfsense_backend` ([proposal](../proposals/plugins/remote/02-surfsense-servers.md#the-scraper-plugins)). +- No limit on threads or browsers per plugin, and no per-IP limit for a free plugin, since none exists. - Requests are not counted or rate-limited per license. License answers and the 24-hour grace are kept per replica, in memory, not in Redis, so a deploy during a Keygen outage refuses keys that were valid. - No deadline on a plugin's call and no MCP progress. Long calls and large results do not run as jobs, and there are no workers, Redis or Blob Storage ([proposal](../proposals/plugins/remote/02-surfsense-servers.md#long-calls-run-as-jobs)). - No registry files, Docker image or Azure deployment. diff --git a/docs/architecture/plugins.md b/docs/architecture/plugins.md index 2d44482e5..a1b3308a9 100644 --- a/docs/architecture/plugins.md +++ b/docs/architecture/plugins.md @@ -7,13 +7,13 @@ A plugin is a remote MCP server whose tools the agent can use. The user connects ## SurfSense's own plugins -[`lists/first_party/plugins.json`](../../surfsense_local/backend/modules/plugins/lists/first_party/plugins.json) lists them, packaged into the API (`bundling/api.spec`), so `GET /plugins/first-party` answers with no network or consent. It holds one entry, SurfSense Scrapers, served by the [plugin host](plugin-host.md). +[`lists/first_party/plugins.json`](../../surfsense_local/backend/modules/plugins/lists/first_party/plugins.json) lists them, packaged into the API (`bundling/api.spec`), so `GET /plugins/first-party` answers with no network or consent. It holds SurfSense's scraper plugins, one per platform: Google Search, Web Crawler, Reddit, YouTube, Instagram, TikTok, Google Maps, Amazon, Walmart and Indeed. Each is its own server at `https://plugins.surfsense.com//mcp`, served by the [plugin host](plugin-host.md), so the user connects, pauses and mentions each apart, and one consent for `plugins.surfsense.com` covers them all. - **The check.** Every entry follows the schema both lists share ([`list_entry.py`](../../surfsense_local/backend/modules/plugins/lists/list_entry.py)): its id, an `https` URL with no credentials, `hosts` naming the URL's host and no loopback, and an `access_note` when `access` is `external`. A first-party entry is published by `surfsense`, and one with `auth: license` sits on `https://plugins.surfsense.com` ([`first_party_list.py`](../../surfsense_local/backend/modules/plugins/lists/first_party/first_party_list.py)). A bad entry fails naming its field, and a unit test checks the packaged list, so it fails CI. - **Connecting.** `POST /plugins/first-party/{id}`: an id not listed is `404`; one connected already, or whose URL is, is `409 plugin_already_connected`; a host not allowed is `403 egress_disabled`, as below. The server's tools are listed with the license, and the plugin is stored with `source: first_party` and `credential_kind: license`. A `401` or `403` still connects it, its check saying why, since a missing license is the server's to report; any other failure is `502 plugin_unreachable`. - **The license.** Every request to a license plugin's server carries `Authorization: License `, the key read from the imported license each time and never stored on the plugin ([`plugin_headers.py`](../../surfsense_local/backend/modules/plugins/remote/sign_in/plugin_headers.py), [`license_key.py`](../../surfsense_local/backend/modules/license/license_key.py)). Its expiry is not checked here: the server decides. The key goes only to a URL on the plugin host this build reaches; a plugin connected elsewhere is sent no key. - **A developer's plugin host.** `SURFSENSE_LOCAL_PLUGIN_HOST_URL`, such as `http://127.0.0.1:8000`, moves every listed URL and host on `plugins.surfsense.com` to a plugin host run locally ([`plugin_host.py`](../../surfsense_local/backend/modules/plugins/lists/plugin_host.py)), and the license key with them. -- A custom plugin never takes a first-party id: one named Scrapers is `scrapers-plugin`. +- A custom plugin never takes a first-party id: one named Reddit is `reddit-plugin`. ## Connecting a server @@ -78,7 +78,7 @@ A tool switched off by a check, because it is new or its description or annotati ## Known gaps - The third-party registry is not listed yet ([proposal](../proposals/plugins/core/02-registry.md)). -- SurfSense Scrapers' entry points at `plugins.surfsense.com`, which is not deployed, so connecting it fails as `502 plugin_unreachable` outside a developer's local plugin host. +- The scraper plugins' entries point at `plugins.surfsense.com`, which is not deployed, so connecting one fails as `502 plugin_unreachable` outside a developer's local plugin host. - An agent turn names a license plugin it could not use as needing sign-in, not as needing a license ([agent](agent.md#plugins)). - Only remote MCP servers, reached over HTTP. A local server started by a command, such as `npx` or `uvx`, cannot be added; whether it should be is an [open question](../proposals/plugins/README.md#open-questions). - OAuth sign-in does not exist; a server that needs it cannot be connected ([proposal](../proposals/plugins/remote/01-mcp-client.md)). diff --git a/plugins/remote/host/app.py b/plugins/remote/host/app.py index 1fd3accd3..fb5f308fd 100644 --- a/plugins/remote/host/app.py +++ b/plugins/remote/host/app.py @@ -36,6 +36,8 @@ def _served(plugin: MountedPlugin) -> ASGIApp: # Reached by its public hostname, not on a user's loopback, so the SDK's # DNS-rebinding guard for local servers does not apply. app = plugin.server.streamable_http_app(host="0.0.0.0") + # A plugin without `access` is open to anyone. SurfSense offers no free plugin + # yet; one will be limited per IP address here, as 02-surfsense-servers.md says. return app if plugin.access is None else RefusedUnlessAllowed(app, plugin.access) diff --git a/plugins/remote/host/mounted_plugins.py b/plugins/remote/host/mounted_plugins.py index 42a422897..a7177d09e 100644 --- a/plugins/remote/host/mounted_plugins.py +++ b/plugins/remote/host/mounted_plugins.py @@ -1,4 +1,7 @@ -"""The plugins the host serves: each folder with a `server.py`, under the root or `proprietary/`.""" +"""The plugins the host serves: each folder with a `server.py`, under the root or `proprietary/`. + +Each is its own MCP server at `//mcp`; one image and one host serve them all. +""" import hashlib import importlib.util @@ -12,8 +15,8 @@ from mcp.server.mcpserver import MCPServer from mcp_server.access import Access -# The host's own folders, beside the plugins; never a plugin's id. -RESERVED = frozenset({"host", "mcp_server", "proprietary", "license_check"}) +# The host's own folders and the code plugins share, beside the plugins; never a plugin. +RESERVED = frozenset({"host", "mcp_server", "proprietary", "license_check", "scraping"}) @dataclass(frozen=True) @@ -42,7 +45,12 @@ def mounted_plugins(root: Path) -> list[MountedPlugin]: def _mounted(folder: Path) -> MountedPlugin: module = _loaded(folder / "server.py") build: Callable[[], MCPServer] = module.server - return MountedPlugin(folder.name, build(), getattr(module, "access", None)) + return MountedPlugin(plugin_id(folder), build(), getattr(module, "access", None)) + + +def plugin_id(folder: Path) -> str: + """`google_search/` is `google-search`: a package cannot hold a hyphen, nor an id an underscore.""" + return folder.name.replace("_", "-") def _loaded(path: Path) -> ModuleType: diff --git a/plugins/remote/host/tests/test_host.py b/plugins/remote/host/tests/test_host.py index 227f280b6..9b773b454 100644 --- a/plugins/remote/host/tests/test_host.py +++ b/plugins/remote/host/tests/test_host.py @@ -34,6 +34,16 @@ async def test_without_proprietary_the_free_plugins_still_run(tmp_path: Path) -> assert httpx2.post(f"{base}/paid/mcp").status_code == 404 +async def test_a_folder_s_underscores_are_its_id_s_hyphens(tmp_path: Path) -> None: + """A Python package cannot hold a hyphen, and a plugin's id cannot hold an underscore.""" + root = tmp_path / "plugins" + shutil.copytree(SAMPLES / "notes", root / "google_search") + + with serving(host_app(root)) as base: + assert await tool_names(f"{base}/google-search/mcp") == ["search"] + assert httpx2.post(f"{base}/google_search/mcp").status_code == 404 + + async def test_a_plugin_that_fails_leaves_the_others_answering() -> None: with serving(host_app(SAMPLES)) as base: failed = await call(f"{base}/failing/mcp", "explode", {}) @@ -43,7 +53,7 @@ async def test_a_plugin_that_fails_leaves_the_others_answering() -> None: assert found.structured_content == {"result": "found x"} -@pytest.mark.parametrize("folder", ["host", "mcp_server", "license_check"]) +@pytest.mark.parametrize("folder", ["host", "mcp_server", "license_check", "scraping"]) def test_a_reserved_folder_is_never_a_plugin(tmp_path: Path, folder: str) -> None: """The host's own folders sit beside the plugins and must not be served.""" root = tmp_path / "plugins" diff --git a/plugins/remote/ruff.toml b/plugins/remote/ruff.toml index 564bede39..4109ca714 100644 --- a/plugins/remote/ruff.toml +++ b/plugins/remote/ruff.toml @@ -16,6 +16,8 @@ select = [ "T20", # flake8-print "SIM", # flake8-simplify "RUF", + # Every plugin answers from one process: a blocking call in a coroutine stalls them all. + "ASYNC", ] [lint.isort] diff --git a/surfsense_local/backend/modules/plugins/lists/first_party/plugins.json b/surfsense_local/backend/modules/plugins/lists/first_party/plugins.json index cb0ec3b19..bc830bc95 100644 --- a/surfsense_local/backend/modules/plugins/lists/first_party/plugins.json +++ b/surfsense_local/backend/modules/plugins/lists/first_party/plugins.json @@ -2,15 +2,163 @@ "schema_version": 1, "plugins": [ { - "id": "scrapers", + "id": "google-search", "kind": "remote", "publisher": "surfsense", - "name": "SurfSense Scrapers", - "description": "Search Google, crawl sites, and scrape Reddit, YouTube, TikTok, Instagram, Google Maps, Amazon, Walmart and Indeed.", + "name": "Google Search", + "description": "Search Google and read its results, ads and related questions.", "author": "SurfSense", - "url": "https://plugins.surfsense.com/scrapers/mcp", - "auth": { "type": "license" }, - "hosts": ["plugins.surfsense.com"], + "url": "https://plugins.surfsense.com/google-search/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "web-crawler", + "kind": "remote", + "publisher": "surfsense", + "name": "Web Crawler", + "description": "Read a web page, or crawl a whole site into clean text.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/web-crawler/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "reddit", + "kind": "remote", + "publisher": "surfsense", + "name": "Reddit", + "description": "Read Reddit posts, their comments and whole subreddits.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/reddit/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "youtube", + "kind": "remote", + "publisher": "surfsense", + "name": "YouTube", + "description": "Read YouTube videos, channels and their comments.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/youtube/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "instagram", + "kind": "remote", + "publisher": "surfsense", + "name": "Instagram", + "description": "Read Instagram profiles and their posts.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/instagram/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "tiktok", + "kind": "remote", + "publisher": "surfsense", + "name": "TikTok", + "description": "Read TikTok videos and their comments, search users, and see what is trending.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/tiktok/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "google-maps", + "kind": "remote", + "publisher": "surfsense", + "name": "Google Maps", + "description": "Find places on Google Maps and read their reviews.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/google-maps/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "amazon", + "kind": "remote", + "publisher": "surfsense", + "name": "Amazon", + "description": "Read Amazon products and search results.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/amazon/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "walmart", + "kind": "remote", + "publisher": "surfsense", + "name": "Walmart", + "description": "Read Walmart products, search results and reviews.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/walmart/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], + "access": "license" + }, + { + "id": "indeed", + "kind": "remote", + "publisher": "surfsense", + "name": "Indeed", + "description": "Read job listings on Indeed.", + "author": "SurfSense", + "url": "https://plugins.surfsense.com/indeed/mcp", + "auth": { + "type": "license" + }, + "hosts": [ + "plugins.surfsense.com" + ], "access": "license" } ] diff --git a/surfsense_local/backend/tests/integration/agent/test_plugin_tools.py b/surfsense_local/backend/tests/integration/agent/test_plugin_tools.py index a92354e43..1c9195f60 100644 --- a/surfsense_local/backend/tests/integration/agent/test_plugin_tools.py +++ b/surfsense_local/backend/tests/integration/agent/test_plugin_tools.py @@ -26,7 +26,7 @@ from tests.mcp_test_server import ( RequiringToken, notes_server, plugin_host, - scrapers_server, + reddit_server, serving, ) @@ -201,13 +201,13 @@ async def test_surfsense_s_plugin_gets_the_license_key_through_opencode( guarded.append(RequiringLicense(app)) return guarded[0] - with serving(plugin_host("scrapers", scrapers_server(), around)) as base: + with serving(plugin_host("reddit", reddit_server(), around)) as base: monkeypatch.setattr(get_plugin_host_settings(), "plugin_host_url", base) - connected = await agent_api.http.post("/plugins/first-party/scrapers") + connected = await agent_api.http.post("/plugins/first-party/reddit") assert connected.status_code == 201, connected.text guarded[0].authorizations.clear() agent_api.model.replies = [ - call("scrapers_google_search", query="ship"), + call("reddit_search_posts", query="ship"), ("text", "Done."), ] thread = await open_thread(agent_api) @@ -215,7 +215,7 @@ async def test_surfsense_s_plugin_gets_the_license_key_through_opencode( frames = await send(agent_api, thread["id"], "Search the web") step = of_type(frames, "agent-step")[-1] - assert (step["tool"], step["status"]) == ("scrapers_google_search", "completed") + assert (step["tool"], step["status"]) == ("reddit_search_posts", "completed") assert set(guarded[0].authorizations) == {f"License {INDIVIDUAL_KEY}"} diff --git a/surfsense_local/backend/tests/integration/plugins/routes/test_first_party_plugins.py b/surfsense_local/backend/tests/integration/plugins/routes/test_first_party_plugins.py index 3326adf87..b14ca0cd7 100644 --- a/surfsense_local/backend/tests/integration/plugins/routes/test_first_party_plugins.py +++ b/surfsense_local/backend/tests/integration/plugins/routes/test_first_party_plugins.py @@ -13,7 +13,7 @@ from tests.license_samples import ( from tests.mcp_test_server import ( RequiringLicense, plugin_host, - scrapers_server, + reddit_server, serving, ) @@ -35,7 +35,7 @@ def host(monkeypatch: pytest.MonkeyPatch) -> Iterator[RequiringLicense]: guarded.append(RequiringLicense(app, {TRIAL_KEY: "expired"})) return guarded[0] - with serving(plugin_host("scrapers", scrapers_server(), around)) as base: + with serving(plugin_host("reddit", reddit_server(), around)) as base: monkeypatch.setattr(get_plugin_host_settings(), "plugin_host_url", base) yield guarded[0] @@ -47,11 +47,10 @@ async def test_surfsense_s_plugins_are_listed_before_any_is_connected( listed = await client.get("/plugins/first-party") assert listed.status_code == 200 - (scrapers,) = listed.json() - assert scrapers["id"] == "scrapers" - assert scrapers["name"] == "SurfSense Scrapers" - assert scrapers["access"] == "license" - assert scrapers["connected"] is False + reddit = next(entry for entry in listed.json() if entry["id"] == "reddit") + assert reddit["name"] == "Reddit" + assert reddit["access"] == "license" + assert reddit["connected"] is False async def test_connecting_with_a_license_sends_its_key( @@ -60,23 +59,23 @@ async def test_connecting_with_a_license_sends_its_key( """Every request to SurfSense's plugin host carries the imported license's key.""" await import_license(client, "individual.lic") - connected = await client.post("/plugins/first-party/scrapers") + connected = await client.post("/plugins/first-party/reddit") assert connected.status_code == 201, connected.text plugin = connected.json() assert (plugin["source"], plugin["publisher"]) == ("first_party", "surfsense") assert plugin["check_failure"] is None - assert [tool["name"] for tool in plugin["tools"]] == ["google_search"] + assert [tool["name"] for tool in plugin["tools"]] == ["search_posts"] assert set(host.authorizations) == {f"License {INDIVIDUAL_KEY}"} listed = (await client.get("/plugins/first-party")).json() - assert listed[0]["connected"] is True + assert [entry["id"] for entry in listed if entry["connected"]] == ["reddit"] async def test_connecting_without_a_license_says_one_is_needed( client: AsyncClient, host: RequiringLicense ) -> None: """Connect is never blocked: the server's refusal says what is missing.""" - connected = await client.post("/plugins/first-party/scrapers") + connected = await client.post("/plugins/first-party/reddit") assert connected.status_code == 201, connected.text assert connected.json()["check_failure"] == "needs_license" @@ -89,7 +88,7 @@ async def test_an_expired_license_reads_as_expired( """The server's reason, not a generic sign-in failure.""" await import_license(client, "trial.lic") - connected = await client.post("/plugins/first-party/scrapers") + connected = await client.post("/plugins/first-party/reddit") assert connected.json()["check_failure"] == "license_expired" @@ -98,10 +97,10 @@ async def test_a_new_license_is_used_at_the_next_check( client: AsyncClient, host: RequiringLicense ) -> None: """The key is read from the license each time, never copied onto the plugin.""" - await client.post("/plugins/first-party/scrapers") + await client.post("/plugins/first-party/reddit") await import_license(client, "individual.lic") - checked = await client.post("/plugins/scrapers/check") + checked = await client.post("/plugins/reddit/check") assert checked.json()["check_failure"] is None @@ -111,13 +110,13 @@ async def test_the_key_never_leaves_for_another_host( ) -> None: """A plugin connected on one plugin host, checked after the build points elsewhere.""" await import_license(client, "individual.lic") - await client.post("/plugins/first-party/scrapers") + await client.post("/plugins/first-party/reddit") monkeypatch.setattr( get_plugin_host_settings(), "plugin_host_url", "http://127.0.0.1:9" ) host.authorizations.clear() - await client.post("/plugins/scrapers/check") + await client.post("/plugins/reddit/check") assert set(host.authorizations) == {None} @@ -126,9 +125,9 @@ async def test_connecting_twice_is_refused( client: AsyncClient, host: RequiringLicense ) -> None: """A stale screen gets a clear answer, not a second copy.""" - await client.post("/plugins/first-party/scrapers") + await client.post("/plugins/first-party/reddit") - again = await client.post("/plugins/first-party/scrapers") + again = await client.post("/plugins/first-party/reddit") assert again.status_code == 409 assert again.json()["detail"]["code"] == "plugin_already_connected" @@ -144,7 +143,7 @@ async def test_a_custom_plugin_never_takes_a_first_party_id( ) -> None: """So SurfSense's own plugin can still be connected under its id.""" added = await client.post( - "/plugins/custom", json={"url": notes_url, "name": "Scrapers"} + "/plugins/custom", json={"url": notes_url, "name": "Reddit"} ) - assert added.json()["id"] == "scrapers-plugin" + assert added.json()["id"] == "reddit-plugin" diff --git a/surfsense_local/backend/tests/mcp_test_server.py b/surfsense_local/backend/tests/mcp_test_server.py index 9706dda88..9dcbe843a 100644 --- a/surfsense_local/backend/tests/mcp_test_server.py +++ b/surfsense_local/backend/tests/mcp_test_server.py @@ -216,14 +216,14 @@ class RequiringToken: await self.app(scope, receive, send) -def scrapers_server() -> MCPServer: - """A stand-in for SurfSense Scrapers: one read-only search.""" - server = MCPServer("scrapers") +def reddit_server() -> MCPServer: + """A stand-in for SurfSense's Reddit plugin: one read-only search.""" + server = MCPServer("reddit") @server.tool(annotations=ToolAnnotations(readOnlyHint=True, openWorldHint=True)) - def google_search(query: str) -> str: - """Search Google.""" - return f"results for {query}" + def search_posts(query: str) -> str: + """Search Reddit's posts.""" + return f"posts about {query}" return server diff --git a/surfsense_local/backend/tests/unit/plugins/lists/test_first_party_list.py b/surfsense_local/backend/tests/unit/plugins/lists/test_first_party_list.py index 71798b7da..82c5c29a1 100644 --- a/surfsense_local/backend/tests/unit/plugins/lists/test_first_party_list.py +++ b/surfsense_local/backend/tests/unit/plugins/lists/test_first_party_list.py @@ -38,7 +38,29 @@ def listed(tmp_path: Path, *entries: dict[str, Any]) -> Path: def test_the_packaged_list_passes_its_check() -> None: """A bad entry fails CI here, not on a user's machine.""" - assert [entry.id for entry in checked_entries(FIRST_PARTY_LIST)] == ["scrapers"] + assert [entry.id for entry in checked_entries(FIRST_PARTY_LIST)] == [ + "google-search", + "web-crawler", + "reddit", + "youtube", + "instagram", + "tiktok", + "google-maps", + "amazon", + "walmart", + "indeed", + ] + + +def test_each_scraper_plugin_is_its_own_server_on_the_plugin_host() -> None: + """One host, so one consent; a path each, so a server each.""" + entries = checked_entries(FIRST_PARTY_LIST) + + assert {entry.url for entry in entries} == { + f"https://plugins.surfsense.com/{entry.id}/mcp" for entry in entries + } + assert {tuple(entry.hosts) for entry in entries} == {("plugins.surfsense.com",)} + assert {entry.access for entry in entries} == {"license"} def test_a_newer_field_is_ignored(tmp_path: Path) -> None: diff --git a/surfsense_local/frontend/src/features/plugins/plugins-settings.test.tsx b/surfsense_local/frontend/src/features/plugins/plugins-settings.test.tsx index 2670e7761..07712120d 100644 --- a/surfsense_local/frontend/src/features/plugins/plugins-settings.test.tsx +++ b/surfsense_local/frontend/src/features/plugins/plugins-settings.test.tsx @@ -43,10 +43,10 @@ function notes(extra: Partial = {}): Plugin { } } -const SCRAPERS: FirstPartyEntry = { - id: "scrapers", - name: "SurfSense Scrapers", - description: "Search Google and crawl sites.", +const REDDIT: FirstPartyEntry = { + id: "reddit", + name: "Reddit", + description: "Read Reddit posts, their comments and whole subreddits.", author: "SurfSense", access: "license", hosts: ["plugins.surfsense.com"], @@ -78,16 +78,16 @@ function api({ const [, , id, , tool] = url.pathname.split("/") if (url.pathname === "/plugins/first-party") { return Response.json( - [SCRAPERS].map((entry) => ({ + [REDDIT].map((entry) => ({ ...entry, connected: connected.some((plugin) => plugin.id === entry.id), })) ) } - if (url.pathname === "/plugins/first-party/scrapers") { + if (url.pathname === "/plugins/first-party/reddit") { const added = notes({ - id: "scrapers", - name: "SurfSense Scrapers", + id: "reddit", + name: "Reddit", source: "first_party", publisher: "surfsense", hosts: ["plugins.surfsense.com"], @@ -433,16 +433,14 @@ describe("plugins settings", () => { const surfsense = await screen.findByRole("region", { name: "By SurfSense", }) - expect(within(surfsense).getByText("SurfSense Scrapers")).toBeTruthy() + expect(within(surfsense).getByText("Reddit")).toBeTruthy() expect(within(surfsense).getByText("License required")).toBeTruthy() await user.click(within(surfsense).getByRole("button", { name: "Connect" })) - expect( - await screen.findByRole("region", { name: "SurfSense Scrapers" }) - ).toBeTruthy() + expect(await screen.findByRole("region", { name: "Reddit" })).toBeTruthy() expect(await within(surfsense).findByText("Connected")).toBeTruthy() expect(changes(calls)).toEqual([ - { method: "POST", path: "/plugins/first-party/scrapers", body: null }, + { method: "POST", path: "/plugins/first-party/reddit", body: null }, ]) })