mirror of
https://github.com/MODSetter/SurfSense.git
synced 2025-09-11 06:44:40 +00:00
feat: Updated the extension for SurfSense v0.0.6
This commit is contained in:
parent
24fd873ca7
commit
8cd1264d3f
4 changed files with 74 additions and 22 deletions
|
@ -1 +1 @@
|
||||||
Subproject commit 8b1635dc2756f86f0a24df6eeaa94519e0518aae
|
Subproject commit b52f4666ce6c484b7acab9c88746c4f7ff5ef112
|
|
@ -67,14 +67,6 @@ app.include_router(
|
||||||
app.include_router(crud_router, prefix="/api/v1", tags=["crud"])
|
app.include_router(crud_router, prefix="/api/v1", tags=["crud"])
|
||||||
|
|
||||||
|
|
||||||
@app.get("/authenticated-route")
|
@app.get("/verify-token")
|
||||||
async def authenticated_route(user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session)):
|
async def authenticated_route(user: User = Depends(current_active_user), session: AsyncSession = Depends(get_async_session)):
|
||||||
retriever = ChucksHybridSearchRetriever(session)
|
return {"message": "Token is valid"}
|
||||||
results = await retriever.hybrid_search(
|
|
||||||
query_text="SurfSense",
|
|
||||||
top_k=1,
|
|
||||||
user_id=user.id,
|
|
||||||
search_space_id=1,
|
|
||||||
document_type="CRAWLED_URL"
|
|
||||||
)
|
|
||||||
return results
|
|
||||||
|
|
|
@ -35,16 +35,14 @@ async def create_documents(
|
||||||
if request.document_type == DocumentType.EXTENSION:
|
if request.document_type == DocumentType.EXTENSION:
|
||||||
for individual_document in request.content:
|
for individual_document in request.content:
|
||||||
fastapi_background_tasks.add_task(
|
fastapi_background_tasks.add_task(
|
||||||
add_extension_received_document,
|
process_extension_document_with_new_session,
|
||||||
session,
|
|
||||||
individual_document,
|
individual_document,
|
||||||
request.search_space_id
|
request.search_space_id
|
||||||
)
|
)
|
||||||
elif request.document_type == DocumentType.CRAWLED_URL:
|
elif request.document_type == DocumentType.CRAWLED_URL:
|
||||||
for url in request.content:
|
for url in request.content:
|
||||||
fastapi_background_tasks.add_task(
|
fastapi_background_tasks.add_task(
|
||||||
add_crawled_url_document,
|
process_crawled_url_with_new_session,
|
||||||
session,
|
|
||||||
url,
|
url,
|
||||||
request.search_space_id
|
request.search_space_id
|
||||||
)
|
)
|
||||||
|
@ -97,11 +95,10 @@ async def create_documents(
|
||||||
|
|
||||||
# Process in background to avoid uvloop conflicts
|
# Process in background to avoid uvloop conflicts
|
||||||
fastapi_background_tasks.add_task(
|
fastapi_background_tasks.add_task(
|
||||||
process_file_in_background,
|
process_file_in_background_with_new_session,
|
||||||
temp_path,
|
temp_path,
|
||||||
file.filename,
|
file.filename,
|
||||||
search_space_id,
|
search_space_id
|
||||||
session
|
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise HTTPException(
|
raise HTTPException(
|
||||||
|
@ -120,6 +117,45 @@ async def create_documents(
|
||||||
detail=f"Failed to upload files: {str(e)}"
|
detail=f"Failed to upload files: {str(e)}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
async def process_extension_document_with_new_session(
|
||||||
|
individual_document,
|
||||||
|
search_space_id: int
|
||||||
|
):
|
||||||
|
"""Create a new session and process extension document."""
|
||||||
|
from app.db import async_session_maker
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
try:
|
||||||
|
await add_extension_received_document(session, individual_document, search_space_id)
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
logging.error(f"Error processing extension document: {str(e)}")
|
||||||
|
|
||||||
|
async def process_crawled_url_with_new_session(
|
||||||
|
url: str,
|
||||||
|
search_space_id: int
|
||||||
|
):
|
||||||
|
"""Create a new session and process crawled URL."""
|
||||||
|
from app.db import async_session_maker
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
try:
|
||||||
|
await add_crawled_url_document(session, url, search_space_id)
|
||||||
|
except Exception as e:
|
||||||
|
import logging
|
||||||
|
logging.error(f"Error processing crawled URL: {str(e)}")
|
||||||
|
|
||||||
|
async def process_file_in_background_with_new_session(
|
||||||
|
file_path: str,
|
||||||
|
filename: str,
|
||||||
|
search_space_id: int
|
||||||
|
):
|
||||||
|
"""Create a new session and process file."""
|
||||||
|
from app.db import async_session_maker
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
await process_file_in_background(file_path, filename, search_space_id, session)
|
||||||
|
|
||||||
async def process_file_in_background(
|
async def process_file_in_background(
|
||||||
file_path: str,
|
file_path: str,
|
||||||
filename: str,
|
filename: str,
|
||||||
|
|
|
@ -255,8 +255,7 @@ async def index_connector_content(
|
||||||
# Add the indexing task to background tasks
|
# Add the indexing task to background tasks
|
||||||
if background_tasks:
|
if background_tasks:
|
||||||
background_tasks.add_task(
|
background_tasks.add_task(
|
||||||
run_slack_indexing,
|
run_slack_indexing_with_new_session,
|
||||||
session,
|
|
||||||
connector_id,
|
connector_id,
|
||||||
search_space_id
|
search_space_id
|
||||||
)
|
)
|
||||||
|
@ -292,8 +291,7 @@ async def index_connector_content(
|
||||||
# Add the indexing task to background tasks
|
# Add the indexing task to background tasks
|
||||||
if background_tasks:
|
if background_tasks:
|
||||||
background_tasks.add_task(
|
background_tasks.add_task(
|
||||||
run_notion_indexing,
|
run_notion_indexing_with_new_session,
|
||||||
session,
|
|
||||||
connector_id,
|
connector_id,
|
||||||
search_space_id
|
search_space_id
|
||||||
)
|
)
|
||||||
|
@ -355,6 +353,19 @@ async def update_connector_last_indexed(
|
||||||
logger.error(f"Failed to update last_indexed_at for connector {connector_id}: {str(e)}")
|
logger.error(f"Failed to update last_indexed_at for connector {connector_id}: {str(e)}")
|
||||||
await session.rollback()
|
await session.rollback()
|
||||||
|
|
||||||
|
async def run_slack_indexing_with_new_session(
|
||||||
|
connector_id: int,
|
||||||
|
search_space_id: int
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Create a new session and run the Slack indexing task.
|
||||||
|
This prevents session leaks by creating a dedicated session for the background task.
|
||||||
|
"""
|
||||||
|
from app.db import async_session_maker
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
await run_slack_indexing(session, connector_id, search_space_id)
|
||||||
|
|
||||||
async def run_slack_indexing(
|
async def run_slack_indexing(
|
||||||
session: AsyncSession,
|
session: AsyncSession,
|
||||||
connector_id: int,
|
connector_id: int,
|
||||||
|
@ -386,6 +397,19 @@ async def run_slack_indexing(
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"Error in background Slack indexing task: {str(e)}")
|
logger.error(f"Error in background Slack indexing task: {str(e)}")
|
||||||
|
|
||||||
|
async def run_notion_indexing_with_new_session(
|
||||||
|
connector_id: int,
|
||||||
|
search_space_id: int
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
Create a new session and run the Notion indexing task.
|
||||||
|
This prevents session leaks by creating a dedicated session for the background task.
|
||||||
|
"""
|
||||||
|
from app.db import async_session_maker
|
||||||
|
|
||||||
|
async with async_session_maker() as session:
|
||||||
|
await run_notion_indexing(session, connector_id, search_space_id)
|
||||||
|
|
||||||
async def run_notion_indexing(
|
async def run_notion_indexing(
|
||||||
session: AsyncSession,
|
session: AsyncSession,
|
||||||
connector_id: int,
|
connector_id: int,
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue