Merge pull request #1672 from Yigtwxx/fix/backend-ruff-imports

fix(backend): restore ruff compliance across surfsense_backend

surfsense_backend/alembic/env.py

@@ -5,12 +5,12 @@ import sys
 from logging.config import fileConfig
 
 import sqlalchemy as sa
-from alembic.script import ScriptDirectory
 from sqlalchemy import pool
 from sqlalchemy.engine import Connection
 from sqlalchemy.ext.asyncio import async_engine_from_config
 
 from alembic import context
+from alembic.script import ScriptDirectory
 
 # Ensure the app directory is in the Python path
 # This allows Alembic to find your models

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/kb_persistence/middleware.py

@@ -41,12 +41,6 @@ from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import (
     SurfSenseFilesystemState,
 )
 from app.agents.chat.multi_agent_chat.shared.state.reducers import _CLEAR
-from app.knowledge_store.paths import (
-    DOCUMENTS_ROOT,
-    parse_documents_path,
-    safe_folder_segment,
-    virtual_path_to_doc,
-)
 from app.db import (
     AgentActionLog,
     Chunk,
@@ -58,6 +52,12 @@ from app.db import (
     shielded_async_session,
 )
 from app.indexing_pipeline.document_chunker import chunk_text
+from app.knowledge_store.paths import (
+    DOCUMENTS_ROOT,
+    parse_documents_path,
+    safe_folder_segment,
+    virtual_path_to_doc,
+)
 from app.services.folder_service import ensure_folder_hierarchy
 from app.utils.document_converters import (
     embed_texts,

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_store_persistence/commit_turn.py

@@ -128,7 +128,9 @@ async def _failed_receipts(
 ) -> list[Receipt]:
     """One failed receipt per leftover change, read back off the kept copy."""
     try:
-        writes, removes = await store.diff_working_copy(thread_working_copy_id(thread_id))
+        writes, removes = await store.diff_working_copy(
+            thread_working_copy_id(thread_id)
+        )
     except FileNotFoundError:
         writes, removes = {}, []
     return [

surfsense_backend/app/agents/chat/multi_agent_chat/main_agent/middleware/knowledge_tree/middleware.py

@@ -37,13 +37,13 @@ from app.agents.chat.multi_agent_chat.shared.filesystem_selection import Filesys
 from app.agents.chat.multi_agent_chat.shared.state.filesystem_state import (
     SurfSenseFilesystemState,
 )
+from app.db import Document, shielded_async_session
 from app.knowledge_store.paths import (
     DOCUMENTS_ROOT,
     PathIndex,
     build_path_index,
     virtual_path_of,
 )
-from app.db import Document, shielded_async_session
 from app.utils.perf import get_perf_logger
 
 _perf_log = get_perf_logger()

surfsense_backend/app/agents/chat/multi_agent_chat/shared/middleware/filesystem/backends/kb_postgres.py

@@ -52,13 +52,13 @@ from app.agents.chat.multi_agent_chat.shared.document_render import (
     render_document,
     source_label,
 )
+from app.db import Chunk, Document, shielded_async_session
 from app.knowledge_store.paths import (
     DOCUMENTS_ROOT,
     build_path_index,
     doc_to_virtual_path,
     virtual_path_to_doc,
 )
-from app.db import Chunk, Document, shielded_async_session
 
 logger = logging.getLogger(__name__)
 

surfsense_backend/app/agents/chat/multi_agent_chat/subagents/shared/run_reader.py

@@ -166,8 +166,8 @@ async def _save_export_document(
     from app.agents.chat.multi_agent_chat.main_agent.middleware.kb_persistence.middleware import (
         _create_document,
     )
-    from app.knowledge_store.paths import DOCUMENTS_ROOT
     from app.db import async_session_maker
+    from app.knowledge_store.paths import DOCUMENTS_ROOT
 
     path = virtual_path.strip()
     if not path.startswith("/"):

surfsense_backend/app/agents/chat/runtime/mention_resolver.py

@@ -36,12 +36,12 @@ from dataclasses import dataclass, field
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.db import Document, Folder
 from app.knowledge_store.paths import (
     DOCUMENTS_ROOT,
     build_path_index,
     virtual_path_of,
 )
-from app.db import Document, Folder
 from app.schemas.new_chat import MentionedDocumentInfo
 
 logger = logging.getLogger(__name__)

surfsense_backend/app/knowledge_store/index/converge.py

@@ -19,10 +19,10 @@ from dataclasses import dataclass, field
 
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.knowledge_store.paths import PATH_MARKER, to_virtual_path
 from app.db import Document, DocumentStatus, Workspace
 from app.indexing_pipeline.connector_document import ConnectorDocument
 from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService
+from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.engines.base import Change
 from app.knowledge_store.index.folders import reconcile_tree_folders
 from app.knowledge_store.index.rows import (
@@ -34,8 +34,8 @@ from app.knowledge_store.index.rows import (
     revision_author_id,
     upsert_row,
 )
-from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.locks import workspace_index_lock
+from app.knowledge_store.paths import PATH_MARKER, to_virtual_path
 from app.utils.document_converters import generate_content_hash
 
 logger = logging.getLogger(__name__)

surfsense_backend/app/knowledge_store/index/folders.py

@@ -64,7 +64,11 @@ async def reconcile_folders(
             folder_parts=list(chain),
         )
     rows = (
-        (await session.execute(select(Folder).where(Folder.workspace_id == workspace_id)))
+        (
+            await session.execute(
+                select(Folder).where(Folder.workspace_id == workspace_id)
+            )
+        )
         .scalars()
         .all()
     )
@@ -119,7 +123,11 @@ async def reparent_folder(
     if not source_chain or not destination_chain:
         return False
     rows = (
-        (await session.execute(select(Folder).where(Folder.workspace_id == workspace_id)))
+        (
+            await session.execute(
+                select(Folder).where(Folder.workspace_id == workspace_id)
+            )
+        )
         .scalars()
         .all()
     )

surfsense_backend/app/knowledge_store/index/project.py

@@ -19,8 +19,8 @@ from dataclasses import dataclass, field
 
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.knowledge_store.paths import to_virtual_path
 from app.db import Workspace
+from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.index.folders import reconcile_tree_folders
 from app.knowledge_store.index.rows import (
     delete_row,
@@ -30,11 +30,11 @@ from app.knowledge_store.index.rows import (
     revision_author_id,
     upsert_row,
 )
-from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.locks import (
     KnowledgeStoreLockError,
     workspace_index_lock,
 )
+from app.knowledge_store.paths import to_virtual_path
 
 logger = logging.getLogger(__name__)
 

surfsense_backend/app/knowledge_store/index/rows.py

@@ -21,13 +21,13 @@ from datetime import UTC, datetime
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.db import Document, DocumentStatus, DocumentType, Workspace
+from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.paths import (
     PATH_MARKER,
     parse_documents_path,
     virtual_path_to_doc,
 )
-from app.db import Document, DocumentStatus, DocumentType, Workspace
-from app.knowledge_store import KnowledgeStore
 from app.services.folder_service import ensure_folder_hierarchy
 from app.utils.document_converters import (
     generate_content_hash,

surfsense_backend/app/knowledge_store/paths/__init__.py

@@ -62,7 +62,7 @@ __all__ = [
     "to_virtual_path",
     "virtual_path_of",
     "virtual_path_to_doc",
+    "working_copies_root",
     "workspace_store_path",
     "workspace_working_copies_path",
-    "working_copies_root",
 ]

surfsense_backend/app/knowledge_store/paths/resolve.py

@@ -101,10 +101,9 @@ async def _resolve_by_title(
 
     from app.db import Document
 
-    unlocated = (
-        Document.path.is_(None)
-        & Document.document_metadata[PATH_MARKER].as_string().is_(None)
-    )
+    unlocated = Document.path.is_(None) & Document.document_metadata[
+        PATH_MARKER
+    ].as_string().is_(None)
     folder_id = await _resolve_folder_id(
         session, workspace_id=workspace_id, folder_parts=list(path.folder_parts)
     )

surfsense_backend/app/knowledge_store/service.py

@@ -203,11 +203,15 @@ class KnowledgeStore:
     async def write(self, path: str, content: str | bytes) -> Outcome:
         """Create or replace one path as a single-verb revision."""
         data = content.encode() if isinstance(content, str) else content
-        return await self._single(lambda tx: tx.write(path, data), f"docs: write {_leaf(path)}")
+        return await self._single(
+            lambda tx: tx.write(path, data), f"docs: write {_leaf(path)}"
+        )
 
     async def remove(self, path: str) -> Outcome:
         """Delete one path as a single-verb revision."""
-        return await self._single(lambda tx: tx.remove(path), f"docs: delete {_leaf(path)}")
+        return await self._single(
+            lambda tx: tx.remove(path), f"docs: delete {_leaf(path)}"
+        )
 
     async def move(self, source: str, destination: str) -> Outcome:
         """Relocate one path as a single-verb revision."""
@@ -269,7 +273,9 @@ class KnowledgeStore:
             enqueue_index(self._workspace_id)
         return tx.revision
 
-    async def _taken_virtual_paths(self, *, exclude: set[str] | None = None) -> set[str]:
+    async def _taken_virtual_paths(
+        self, *, exclude: set[str] | None = None
+    ) -> set[str]:
         """The document paths git already holds, so a fresh name skips them.
 
         Occupancy comes from the tree, the one authority on which files exist;
@@ -293,7 +299,9 @@ class KnowledgeStore:
                 taken.add(virtual)
         return taken
 
-    def _author_path(self, *, title: str, folder_id: int | None, index, taken: set[str]) -> str:
+    def _author_path(
+        self, *, title: str, folder_id: int | None, index, taken: set[str]
+    ) -> str:
         """A fresh ``.md`` path under the row's folder, breaking a name clash.
 
         The naming law, not the legacy ``.xml`` derivation: this is the one place
@@ -348,7 +356,8 @@ class KnowledgeStore:
             previous = (metadata or {}).get(PATH_MARKER)
             recorded = (
                 previous
-                if isinstance(previous, str) and previous.startswith(f"{DOCUMENTS_ROOT}/")
+                if isinstance(previous, str)
+                and previous.startswith(f"{DOCUMENTS_ROOT}/")
                 else None
             )
             # A recorded path stays put; only an explicit title, or a first write,
@@ -366,9 +375,7 @@ class KnowledgeStore:
                     and document.path.startswith(f"{DOCUMENTS_ROOT}/")
                     else None
                 )
-                taken = await self._taken_virtual_paths(
-                    exclude={own} if own else set()
-                )
+                taken = await self._taken_virtual_paths(exclude={own} if own else set())
                 virtual_path = self._author_path(
                     title=title, folder_id=folder_id, index=index, taken=taken
                 )
@@ -589,7 +596,9 @@ class KnowledgeStore:
             return Outcome(revision=None)
         src = self._folder_store_path(source)
         dst = self._folder_store_path(destination)
-        moves = [(p, f"{dst}{p[len(src):]}") for p in await self._subtree_paths(source)]
+        moves = [
+            (p, f"{dst}{p[len(src) :]}") for p in await self._subtree_paths(source)
+        ]
         revision = await self._commit_files(
             files={}, moves=moves, message=f"docs: move folder {_leaf(destination)}"
         )

surfsense_backend/app/proprietary/platforms/reddit/fetch.py

@@ -156,7 +156,7 @@ def _browser_cookie_jar(page: Any) -> dict[str, str]:
     if isinstance(cookies, dict):
         return {str(k): str(v) for k, v in cookies.items()}
     jar: dict[str, str] = {}
-    if isinstance(cookies, (list, tuple)):
+    if isinstance(cookies, list | tuple):
         for item in cookies:
             if isinstance(item, dict) and "name" in item and "value" in item:
                 jar[str(item["name"])] = str(item["value"])

surfsense_backend/app/routes/documents_routes.py

@@ -8,7 +8,6 @@ from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 from sqlalchemy.orm import selectinload
 
-from app.knowledge_store.paths import virtual_path_to_doc
 from app.auth.context import AuthContext
 from app.db import (
     Chunk,
@@ -21,6 +20,7 @@ from app.db import (
     WorkspaceMembership,
     get_async_session,
 )
+from app.knowledge_store.paths import virtual_path_to_doc
 from app.knowledge_store.service import record_deleted_documents
 from app.knowledge_store.settings import knowledge_store_enabled_for
 from app.schemas import (

surfsense_backend/app/services/folder_service.py

@@ -8,8 +8,8 @@ from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.future import select
 
-from app.knowledge_store.paths import safe_folder_segment
 from app.db import Folder
+from app.knowledge_store.paths import safe_folder_segment
 
 MAX_FOLDER_DEPTH = 8
 

surfsense_backend/app/services/revert_service.py

@@ -38,11 +38,6 @@ from typing import Any, Literal
 from sqlalchemy import delete, select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from app.knowledge_store.paths import (
-    DOCUMENTS_ROOT,
-    safe_filename,
-    safe_folder_segment,
-)
 from app.db import (
     AgentActionLog,
     Chunk,
@@ -53,6 +48,11 @@ from app.db import (
     FolderRevision,
     NewChatThread,
 )
+from app.knowledge_store.paths import (
+    DOCUMENTS_ROOT,
+    safe_filename,
+    safe_folder_segment,
+)
 from app.utils.document_converters import (
     embed_texts,
     generate_content_hash,

surfsense_backend/app/tasks/celery_tasks/knowledge_store/index_tasks.py

@@ -12,13 +12,13 @@ from sqlalchemy import select
 
 from app.celery_app import celery_app
 from app.db import Workspace
+from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.index.converge import index_changes, index_tree
+from app.knowledge_store.locks import KnowledgeStoreLockError
 from app.knowledge_store.settings import (
     knowledge_store_enabled_for,
     load_knowledge_store_settings,
 )
-from app.knowledge_store import KnowledgeStore
-from app.knowledge_store.locks import KnowledgeStoreLockError
 from app.tasks.celery_tasks import get_celery_session_maker, run_async_celery_task
 
 logger = logging.getLogger(__name__)

surfsense_backend/app/tasks/connector_indexers/local_folder_indexer.py

@@ -843,9 +843,7 @@ async def index_local_folder(
                     failed_count += 1
                     continue
 
-                result = await pipeline.index_unless_store_owns(
-                    document, connector_doc
-                )
+                result = await pipeline.index_unless_store_owns(document, connector_doc)
 
                 # A deferral (``None``) recorded the row for the store's indexer to
                 # chunk; still stamp mtime so the next sweep skips an unchanged file.

surfsense_backend/scripts/check_migration_flow.py

@@ -23,7 +23,8 @@ BACKEND_DIR = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(BACKEND_DIR))
 
 ADMIN_URL = os.getenv(
-    "ADMIN_DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/postgres"
+    "ADMIN_DATABASE_URL",
+    "postgresql://postgres:postgres@localhost:5432/postgres",  # pragma: allowlist secret
 )
 SCRATCH_DB = "surfsense_check_migration_flow"
 SCRATCH_URL = ADMIN_URL.rsplit("/", 1)[0] + f"/{SCRATCH_DB}"
@@ -94,6 +95,7 @@ async def set_version(version: str | None) -> None:
 
 async def assert_at_head() -> None:
     import asyncpg
+
     from alembic.script import ScriptDirectory
 
     head = ScriptDirectory(str(BACKEND_DIR / "alembic")).get_current_head()

surfsense_backend/scripts/migrate_knowledge_store.py

@@ -18,8 +18,8 @@ from datetime import UTC, datetime
 from sqlalchemy import select, update
 
 from app.db import Workspace, async_session_maker
-from app.knowledge_store.migrate import migrate_workspace
 from app.knowledge_store import KnowledgeStore
+from app.knowledge_store.migrate import migrate_workspace
 
 
 async def _workspace_ids(only: list[int]) -> list[int]:

surfsense_backend/tests/integration/knowledge_store/index/test_converge.py

@@ -16,7 +16,14 @@ import pytest
 from sqlalchemy import func, select
 
 from app.config import config as app_config
-from app.db import Chunk, Document, DocumentStatus, DocumentType, DocumentVersion, Folder
+from app.db import (
+    Chunk,
+    Document,
+    DocumentStatus,
+    DocumentType,
+    DocumentVersion,
+    Folder,
+)
 from app.indexing_pipeline.connector_document import ConnectorDocument
 from app.indexing_pipeline.indexing_pipeline_service import IndexingPipelineService
 from app.knowledge_store import KnowledgeStore

surfsense_backend/tests/integration/knowledge_store/index/test_project.py

@@ -18,8 +18,8 @@ from app.knowledge_store import KnowledgeStore
 from app.knowledge_store.identities import user_identity
 from app.knowledge_store.index.converge import index_changes
 from app.knowledge_store.index.project import project_revision
-from app.knowledge_store.paths import PATH_MARKER
 from app.knowledge_store.locks import workspace_index_lock
+from app.knowledge_store.paths import PATH_MARKER
 
 pytestmark = pytest.mark.integration
 

surfsense_backend/tests/integration/knowledge_store/test_migrate.py

@@ -122,7 +122,9 @@ async def test_a_leftover_keep_in_a_populated_folder_is_not_drift(
         tx.write("notes/plan.md", b"# Plan")
         tx.write("notes/.keep", b"")  # the empty-folder marker left behind
 
-    report = await seed_workspace(workspace_id, {"notes/plan.md": "# Plan"}, dry_run=True)
+    report = await seed_workspace(
+        workspace_id, {"notes/plan.md": "# Plan"}, dry_run=True
+    )
 
     assert report.ok
     assert report.extra == []

surfsense_backend/tests/integration/knowledge_store/test_migrate_placement.py

@@ -24,7 +24,9 @@ def knowledge_root(tmp_path, monkeypatch):
     return tmp_path
 
 
-async def _add_document(session, workspace, *, title, markdown, marker=None, folder_id=None):
+async def _add_document(
+    session, workspace, *, title, markdown, marker=None, folder_id=None
+):
     document = Document(
         title=title,
         document_type=DocumentType.NOTE,

surfsense_backend/tests/integration/knowledge_store/test_path_column.py

@@ -48,7 +48,9 @@ async def test_the_healed_column_resolves_without_a_marker(
         marker=None,
     )
     resolved = await virtual_path_to_doc(
-        db_session, workspace_id=db_workspace.id, virtual_path="/documents/notes/plan.md"
+        db_session,
+        workspace_id=db_workspace.id,
+        virtual_path="/documents/notes/plan.md",
     )
     assert resolved is doc
 

surfsense_backend/tests/integration/knowledge_store/test_service.py

@@ -6,10 +6,10 @@ import uuid
 
 import pytest
 
-from app.knowledge_store.paths import PATH_MARKER
 from app.config import config as app_config
 from app.db import Document, DocumentStatus, DocumentType
 from app.knowledge_store import KnowledgeStore
+from app.knowledge_store.paths import PATH_MARKER
 from app.knowledge_store.service import (
     drop_workspace_store,
     record_deleted_documents,

surfsense_backend/tests/integration/test_okf_path_identity.py

@@ -7,12 +7,12 @@ import pytest
 import pytest_asyncio
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.db import Document, DocumentType, Folder, User, Workspace
 from app.knowledge_store.paths import (
     build_path_index,
     doc_to_virtual_path,
     virtual_path_to_doc,
 )
-from app.db import Document, DocumentType, Folder, User, Workspace
 
 pytestmark = pytest.mark.integration
 

surfsense_backend/tests/unit/knowledge_store/test_recorded_virtual_path.py

@@ -17,7 +17,10 @@ def test_marker_is_preferred():
         metadata={PATH_MARKER: f"{DOCUMENTS_ROOT}/from-marker.md"},
         path=f"{DOCUMENTS_ROOT}/from-column.md",
     )
-    assert _recorded_virtual_path(doc, DOCUMENTS_ROOT) == f"{DOCUMENTS_ROOT}/from-marker.md"
+    assert (
+        _recorded_virtual_path(doc, DOCUMENTS_ROOT)
+        == f"{DOCUMENTS_ROOT}/from-marker.md"
+    )
 
 
 def test_column_is_the_fallback_when_marker_was_wiped():

surfsense_backend/tests/unit/middleware/test_git_tree_backend.py

@@ -116,9 +116,7 @@ class TestGitTreeBackend:
         write = await backend.awrite("/documents/research/a.md", "x")
         assert write.error is None
 
-    async def test_mkdir_materializes_an_empty_folder_as_its_keep(
-        self, knowledge_root
-    ):
+    async def test_mkdir_materializes_an_empty_folder_as_its_keep(self, knowledge_root):
         # Git drops empty directories; the .keep marker is what carries an
         # agent-created empty folder into the turn's diff, same as the facade.
         backend = GitTreeBackend(WORKSPACE_ID, _RuntimeStub())

surfsense_backend/tests/unit/services/test_model_connections.py

@@ -183,10 +183,7 @@ async def test_ollama_discovery_merges_details_from_both_endpoints(
     assert details["quantization_level"] == "Q4_K_M"
     # The reported maximum survives in the metadata the UI reads, while the
     # seeded budget stays at the fallback the host is likely to have allocated.
-    assert (
-        results[0]["max_input_tokens"]
-        == SURFSENSE_UNKNOWN_MODEL_MAX_INPUT_TOKENS
-    )
+    assert results[0]["max_input_tokens"] == SURFSENSE_UNKNOWN_MODEL_MAX_INPUT_TOKENS
 
 
 def test_anthropic_resolver_strips_trailing_v1_from_api_base() -> None: