vrr/SurfSense
2
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2025-09-02 18:49:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

sync with main and address comments

Browse source
This commit is contained in:
Adamsmith6300 2025-04-14 17:04:43 -07:00
parent a69bbb32f7
commit 3e472c124f
1 changed files with 5 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
surfsense_backend/app/utils/connector_service.py
View file

@ -529,8 +529,8 @@ class ConnectorService:
document_type="GITHUB_CONNECTOR" document_type="GITHUB_CONNECTOR"
) )
# Map github_chunks to the required format # Process each chunk and create sources directly without deduplication
mapped_sources = {} sources_list = []
for i, chunk in enumerate(github_chunks): for i, chunk in enumerate(github_chunks):
# Fix for UI - assign a unique ID for citation/source tracking # Fix for UI - assign a unique ID for citation/source tracking
github_chunks[i]['document']['id'] = self.source_id_counter github_chunks[i]['document']['id'] = self.source_id_counter
@ -539,7 +539,7 @@ class ConnectorService:
document = chunk.get('document', {}) document = chunk.get('document', {})
metadata = document.get('metadata', {}) metadata = document.get('metadata', {})
# Create a mapped source entry # Create a source entry
source = { source = {
"id": self.source_id_counter, "id": self.source_id_counter,
"title": document.get('title', 'GitHub Document'), # Use specific title if available "title": document.get('title', 'GitHub Document'), # Use specific title if available
@ -548,18 +548,11 @@ class ConnectorService:
} }
self.source_id_counter += 1 self.source_id_counter += 1
sources_list.append(source)
# Use a unique identifier for tracking unique sources (URL preferred)
source_key = source.get("url") or source.get("title")
if source_key and source_key not in mapped_sources:
mapped_sources[source_key] = source
# Convert to list of sources
sources_list = list(mapped_sources.values())
# Create result object # Create result object
result_object = { result_object = {
"id": 7, # Assuming 7 is the next available ID "id": 8,
"name": "GitHub", "name": "GitHub",
"type": "GITHUB_CONNECTOR", "type": "GITHUB_CONNECTOR",
"sources": sources_list, "sources": sources_list,