mirror of
https://github.com/MODSetter/SurfSense.git
synced 2025-09-10 22:34:39 +00:00
feat: Updated the extension for SurfSense v0.0.6
This commit is contained in:
parent
24fd873ca7
commit
8cd1264d3f
4 changed files with 74 additions and 22 deletions
|
@ -1 +1 @@
|
|||
Subproject commit 8b1635dc2756f86f0a24df6eeaa94519e0518aae
|
||||
Subproject commit b52f4666ce6c484b7acab9c88746c4f7ff5ef112
|
|
@ -67,14 +67,6 @@ app.include_router(
|
|||
app.include_router(crud_router, prefix="/api/v1", tags=["crud"])
|
||||
|
||||
|
||||
@app.get("/authenticated-route")
|
||||
@app.get("/verify-token")
|
||||
async def authenticated_route(user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session)):
|
||||
retriever = ChucksHybridSearchRetriever(session)
|
||||
results = await retriever.hybrid_search(
|
||||
query_text="SurfSense",
|
||||
top_k=1,
|
||||
user_id=user.id,
|
||||
search_space_id=1,
|
||||
document_type="CRAWLED_URL"
|
||||
)
|
||||
return results
|
||||
return {"message": "Token is valid"}
|
||||
|
|
|
@ -35,16 +35,14 @@ async def create_documents(
|
|||
if request.document_type == DocumentType.EXTENSION:
|
||||
for individual_document in request.content:
|
||||
fastapi_background_tasks.add_task(
|
||||
add_extension_received_document,
|
||||
session,
|
||||
process_extension_document_with_new_session,
|
||||
individual_document,
|
||||
request.search_space_id
|
||||
)
|
||||
elif request.document_type == DocumentType.CRAWLED_URL:
|
||||
for url in request.content:
|
||||
fastapi_background_tasks.add_task(
|
||||
add_crawled_url_document,
|
||||
session,
|
||||
process_crawled_url_with_new_session,
|
||||
url,
|
||||
request.search_space_id
|
||||
)
|
||||
|
@ -97,11 +95,10 @@ async def create_documents(
|
|||
|
||||
# Process in background to avoid uvloop conflicts
|
||||
fastapi_background_tasks.add_task(
|
||||
process_file_in_background,
|
||||
process_file_in_background_with_new_session,
|
||||
temp_path,
|
||||
file.filename,
|
||||
search_space_id,
|
||||
session
|
||||
search_space_id
|
||||
)
|
||||
except Exception as e:
|
||||
raise HTTPException(
|
||||
|
@ -120,6 +117,45 @@ async def create_documents(
|
|||
detail=f"Failed to upload files: {str(e)}"
|
||||
)
|
||||
|
||||
async def process_extension_document_with_new_session(
|
||||
individual_document,
|
||||
search_space_id: int
|
||||
):
|
||||
"""Create a new session and process extension document."""
|
||||
from app.db import async_session_maker
|
||||
|
||||
async with async_session_maker() as session:
|
||||
try:
|
||||
await add_extension_received_document(session, individual_document, search_space_id)
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.error(f"Error processing extension document: {str(e)}")
|
||||
|
||||
async def process_crawled_url_with_new_session(
|
||||
url: str,
|
||||
search_space_id: int
|
||||
):
|
||||
"""Create a new session and process crawled URL."""
|
||||
from app.db import async_session_maker
|
||||
|
||||
async with async_session_maker() as session:
|
||||
try:
|
||||
await add_crawled_url_document(session, url, search_space_id)
|
||||
except Exception as e:
|
||||
import logging
|
||||
logging.error(f"Error processing crawled URL: {str(e)}")
|
||||
|
||||
async def process_file_in_background_with_new_session(
|
||||
file_path: str,
|
||||
filename: str,
|
||||
search_space_id: int
|
||||
):
|
||||
"""Create a new session and process file."""
|
||||
from app.db import async_session_maker
|
||||
|
||||
async with async_session_maker() as session:
|
||||
await process_file_in_background(file_path, filename, search_space_id, session)
|
||||
|
||||
async def process_file_in_background(
|
||||
file_path: str,
|
||||
filename: str,
|
||||
|
|
|
@ -255,8 +255,7 @@ async def index_connector_content(
|
|||
# Add the indexing task to background tasks
|
||||
if background_tasks:
|
||||
background_tasks.add_task(
|
||||
run_slack_indexing,
|
||||
session,
|
||||
run_slack_indexing_with_new_session,
|
||||
connector_id,
|
||||
search_space_id
|
||||
)
|
||||
|
@ -292,8 +291,7 @@ async def index_connector_content(
|
|||
# Add the indexing task to background tasks
|
||||
if background_tasks:
|
||||
background_tasks.add_task(
|
||||
run_notion_indexing,
|
||||
session,
|
||||
run_notion_indexing_with_new_session,
|
||||
connector_id,
|
||||
search_space_id
|
||||
)
|
||||
|
@ -355,6 +353,19 @@ async def update_connector_last_indexed(
|
|||
logger.error(f"Failed to update last_indexed_at for connector {connector_id}: {str(e)}")
|
||||
await session.rollback()
|
||||
|
||||
async def run_slack_indexing_with_new_session(
|
||||
connector_id: int,
|
||||
search_space_id: int
|
||||
):
|
||||
"""
|
||||
Create a new session and run the Slack indexing task.
|
||||
This prevents session leaks by creating a dedicated session for the background task.
|
||||
"""
|
||||
from app.db import async_session_maker
|
||||
|
||||
async with async_session_maker() as session:
|
||||
await run_slack_indexing(session, connector_id, search_space_id)
|
||||
|
||||
async def run_slack_indexing(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
|
@ -386,6 +397,19 @@ async def run_slack_indexing(
|
|||
except Exception as e:
|
||||
logger.error(f"Error in background Slack indexing task: {str(e)}")
|
||||
|
||||
async def run_notion_indexing_with_new_session(
|
||||
connector_id: int,
|
||||
search_space_id: int
|
||||
):
|
||||
"""
|
||||
Create a new session and run the Notion indexing task.
|
||||
This prevents session leaks by creating a dedicated session for the background task.
|
||||
"""
|
||||
from app.db import async_session_maker
|
||||
|
||||
async with async_session_maker() as session:
|
||||
await run_notion_indexing(session, connector_id, search_space_id)
|
||||
|
||||
async def run_notion_indexing(
|
||||
session: AsyncSession,
|
||||
connector_id: int,
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue