vrr/SurfSense
2
0
Fork 0
mirror of https://github.com/MODSetter/SurfSense.git synced 2025-09-04 19:49:09 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Refactor: Cleanup DOCLING PR

Browse source
This commit is contained in:
MSI\ModSetter 2025-07-21 06:12:11 -07:00
parent 641f784f77
commit 621590c049
5 changed files with 2331 additions and 2326 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,5 +1,3 @@
.flashrank_cache* .flashrank_cache*
podcasts/ podcasts/
reports/
SURFSENSE_CRITICAL_FIXES_REPORT.md
.env .env

7
surfsense_backend/Dockerfile
View file

@ -11,10 +11,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
wget \ wget \
unzip \ unzip \
gnupg2 \ gnupg2 \
tesseract-ocr \
tesseract-ocr-eng \
libtesseract-dev \
libleptonica-dev \
&& rm -rf /var/lib/apt/lists/* && rm -rf /var/lib/apt/lists/*
# Update certificates and install SSL tools # Update certificates and install SSL tools
@ -60,9 +56,6 @@ COPY . .
ENV PYTHONPATH=/app ENV PYTHONPATH=/app
ENV UVICORN_LOOP=asyncio ENV UVICORN_LOOP=asyncio
# Set Tesseract data path
ENV TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata/
# Run # Run
EXPOSE 8000 EXPOSE 8000
CMD ["python", "main.py"] CMD ["python", "main.py"]

1
surfsense_backend/pyproject.toml
View file

@ -30,7 +30,6 @@ dependencies = [
"slack-sdk>=3.34.0", "slack-sdk>=3.34.0",
"static-ffmpeg>=2.13", "static-ffmpeg>=2.13",
"tavily-python>=0.3.2", "tavily-python>=0.3.2",
"tesserocr>=2.8.0",
"unstructured-client>=0.30.0", "unstructured-client>=0.30.0",
"unstructured[all-docs]>=0.16.25", "unstructured[all-docs]>=0.16.25",
"uvicorn[standard]>=0.34.0", "uvicorn[standard]>=0.34.0",

4480
surfsense_backend/uv.lock generated
View file

File diff suppressed because it is too large Load diff

23
surfsense_web/app/dashboard/[search_space_id]/documents/upload/page.tsx
View file

@ -53,8 +53,11 @@ export default function FileUploader() {
}; };
// Conditionally set accepted file types based on ETL service // Conditionally set accepted file types based on ETL service
const acceptedFileTypes = process.env.NEXT_PUBLIC_ETL_SERVICE === 'LLAMACLOUD' const getAcceptedFileTypes = () => {
? { const etlService = process.env.NEXT_PUBLIC_ETL_SERVICE;
if (etlService === 'LLAMACLOUD') {
return {
// LlamaCloud supported file types // LlamaCloud supported file types
'application/pdf': ['.pdf'], 'application/pdf': ['.pdf'],
'application/msword': ['.doc'], 'application/msword': ['.doc'],
@ -100,8 +103,16 @@ export default function FileUploader() {
'text/x-spreadsheet': ['.dif', '.sylk', '.slk', '.prn', '.et', '.uos1', '.uos2', '.wk1', '.wk2', '.wk3', '.wk4', '.wks', '.wq1', '.wq2', '.wb1', '.wb2', '.wb3', '.qpw', '.xlr', '.eth'], 'text/x-spreadsheet': ['.dif', '.sylk', '.slk', '.prn', '.et', '.uos1', '.uos2', '.wk1', '.wk2', '.wk3', '.wk4', '.wks', '.wq1', '.wq2', '.wb1', '.wb2', '.wb3', '.qpw', '.xlr', '.eth'],
// Audio files (always supported) // Audio files (always supported)
...audioFileTypes, ...audioFileTypes,
} };
: { } else if (etlService === 'DOCLING') {
return {
// Docling supported file types (currently only PDF)
'application/pdf': ['.pdf'],
// Audio files (always supported)
...audioFileTypes,
};
} else {
return {
// Unstructured supported file types // Unstructured supported file types
'image/bmp': ['.bmp'], 'image/bmp': ['.bmp'],
'text/csv': ['.csv'], 'text/csv': ['.csv'],
@ -130,6 +141,10 @@ export default function FileUploader() {
// Audio files (always supported) // Audio files (always supported)
...audioFileTypes, ...audioFileTypes,
}; };
}
};
const acceptedFileTypes = getAcceptedFileTypes();
const supportedExtensions = Array.from(new Set(Object.values(acceptedFileTypes).flat())).sort() const supportedExtensions = Array.from(new Set(Object.values(acceptedFileTypes).flat())).sort()