From f875ba88aca14af5eaebfee853fccdd20404147d Mon Sep 17 00:00:00 2001 From: xijibomi-coffee <121471102+Whitestar14@users.noreply.github.com> Date: Tue, 20 Jan 2026 06:22:04 +0100 Subject: [PATCH] perf: use native walkdir for recursive imports from directory (#2993) * fix(perf): replace JS recursion with native Rust walkdir for imports * fix: implement security scope check in rust recursive scanner * refactor and format code --------- Co-authored-by: Huang Xin --- Cargo.lock | 1 + apps/readest-app/src-tauri/Cargo.toml | 1 + apps/readest-app/src-tauri/src/dir_scanner.rs | 96 +++++++++++++++++++ apps/readest-app/src-tauri/src/lib.rs | 2 + apps/readest-app/src/app/library/page.tsx | 27 +++--- apps/readest-app/src/libs/document.ts | 84 ++++++++-------- .../src/services/nativeAppService.ts | 38 +++++++- apps/readest-app/src/store/libraryStore.ts | 12 +++ 8 files changed, 208 insertions(+), 53 deletions(-) create mode 100644 apps/readest-app/src-tauri/src/dir_scanner.rs diff --git a/Cargo.lock b/Cargo.lock index f1cb86b6..41913216 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,6 +55,7 @@ dependencies = [ "thiserror 2.0.17", "tokio", "tokio-util", + "walkdir", ] [[package]] diff --git a/apps/readest-app/src-tauri/Cargo.toml b/apps/readest-app/src-tauri/Cargo.toml index e9adebc1..e6373c31 100644 --- a/apps/readest-app/src-tauri/Cargo.toml +++ b/apps/readest-app/src-tauri/Cargo.toml @@ -26,6 +26,7 @@ serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } log = "0.4" thiserror = "2" +walkdir = "2" tokio = { version = "1", features = ["fs"] } tokio-util = { version = "0.7", features = ["codec"] } futures-util = "0.3" diff --git a/apps/readest-app/src-tauri/src/dir_scanner.rs b/apps/readest-app/src-tauri/src/dir_scanner.rs new file mode 100644 index 00000000..858efb1a --- /dev/null +++ b/apps/readest-app/src-tauri/src/dir_scanner.rs @@ -0,0 +1,96 @@ +use std::path::Path; +use tauri::AppHandle; +use tauri_plugin_fs::FsExt; +use walkdir::WalkDir; + +#[derive(serde::Serialize)] +pub struct ScannedFile { + pub path: String, + pub size: u64, +} + +#[tauri::command] +pub fn read_dir( + app: AppHandle, + path: String, + recursive: bool, + extensions: Vec, +) -> Result, String> { + let scope = app.fs_scope(); + let path_buf = std::path::PathBuf::from(&path); + + if !scope.is_allowed(&path_buf) { + return Err("Permission denied: Path not in filesystem scope".to_string()); + } + + let mut files = Vec::new(); + + let normalized_extensions: Vec = + extensions.iter().map(|ext| ext.to_lowercase()).collect(); + + if recursive { + for entry_result in WalkDir::new(&path).into_iter() { + match entry_result { + Ok(entry) => { + if entry.file_type().is_file() { + if let Some(scanned_file) = + process_file_entry(entry.path(), &normalized_extensions) + { + files.push(scanned_file); + } + } + } + Err(e) => { + log::warn!("RUST: Skipping file due to error: {}", e); + } + } + } + } else { + match std::fs::read_dir(&path_buf) { + Ok(entries) => { + for entry_result in entries { + match entry_result { + Ok(entry) => { + let path = entry.path(); + if path.is_file() { + if let Some(scanned_file) = + process_file_entry(&path, &normalized_extensions) + { + files.push(scanned_file); + } + } + } + Err(e) => { + log::warn!("RUST: Skipping entry due to error: {}", e); + } + } + } + } + Err(e) => { + return Err(format!("Failed to read directory: {}", e)); + } + } + } + + Ok(files) +} + +fn process_file_entry(path: &Path, extensions: &[String]) -> Option { + if extensions.is_empty() || extensions.contains(&"*".to_string()) { + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + return Some(ScannedFile { + path: path.to_string_lossy().to_string(), + size, + }); + } else if let Some(ext) = path.extension() { + let ext_str = ext.to_string_lossy().to_lowercase(); + if extensions.contains(&ext_str) { + let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + return Some(ScannedFile { + path: path.to_string_lossy().to_string(), + size, + }); + } + } + None +} diff --git a/apps/readest-app/src-tauri/src/lib.rs b/apps/readest-app/src-tauri/src/lib.rs index a2dd53c0..50bff75c 100644 --- a/apps/readest-app/src-tauri/src/lib.rs +++ b/apps/readest-app/src-tauri/src/lib.rs @@ -22,6 +22,7 @@ use tauri_plugin_fs::FsExt; #[cfg(desktop)] use tauri::{Listener, Url}; +mod dir_scanner; #[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))] mod discord_rpc; #[cfg(target_os = "macos")] @@ -163,6 +164,7 @@ pub fn run() { upload_file, get_environment_variable, get_executable_dir, + dir_scanner::read_dir, #[cfg(target_os = "macos")] macos::safari_auth::auth_with_safari, #[cfg(target_os = "macos")] diff --git a/apps/readest-app/src/app/library/page.tsx b/apps/readest-app/src/app/library/page.tsx index 583209ec..5f7a9cb8 100644 --- a/apps/readest-app/src/app/library/page.tsx +++ b/apps/readest-app/src/app/library/page.tsx @@ -83,6 +83,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP isSyncing, syncProgress, updateBook, + updateBooks, setLibrary, getGroupId, getGroupName, @@ -405,30 +406,29 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP ['Unsupported or corrupted book file', _('The book file is corrupted')], ]; - const processFile = async (selectedFile: SelectedFile) => { + const processFile = async (selectedFile: SelectedFile): Promise => { const file = selectedFile.file || selectedFile.path; - if (!file) return; + if (!file) return null; try { const book = await appService?.importBook(file, library); + if (!book) return null; const { path, basePath } = selectedFile; - if (book && groupId) { + if (groupId) { book.groupId = groupId; book.groupName = getGroupName(groupId); - await updateBook(envConfig, book); - } else if (book && path && basePath) { + } else if (path && basePath) { const rootPath = getDirPath(basePath); const groupName = getDirPath(path).replace(rootPath, '').replace(/^\//, ''); book.groupName = groupName; book.groupId = getGroupId(groupName); - await updateBook(envConfig, book); } - if (user && book && !book.uploadedAt && settings.autoUpload) { + + if (user && !book.uploadedAt && settings.autoUpload) { console.log('Queueing upload for book:', book.title); transferManager.queueUpload(book); } - if (book) { - successfulImports.push(book.title); - } + successfulImports.push(book.title); + return book; } catch (error) { const filename = typeof file === 'string' ? file : file.name; const baseFilename = getFilename(filename); @@ -438,14 +438,17 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP : ''; failedImports.push({ filename: baseFilename, errorMessage }); console.error('Failed to import book:', filename, error); + return null; } }; const concurrency = 4; for (let i = 0; i < files.length; i += concurrency) { const batch = files.slice(i, i + concurrency); - await Promise.all(batch.map(processFile)); + const importedBooks = (await Promise.all(batch.map(processFile))).filter((book) => !!book); + await updateBooks(envConfig, importedBooks); } + pushLibrary(); if (failedImports.length > 0) { @@ -470,8 +473,6 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP }); } - setLibrary([...library]); - appService?.saveLibraryBooks(library); setLoading(false); }; diff --git a/apps/readest-app/src/libs/document.ts b/apps/readest-app/src/libs/document.ts index a38fd27f..d0d87595 100644 --- a/apps/readest-app/src/libs/document.ts +++ b/apps/readest-app/src/libs/document.ts @@ -192,48 +192,56 @@ export class DocumentLoader { if (!this.file.size) { throw new Error('File is empty'); } - if (await this.isZip()) { - const loader = await this.makeZipLoader(); - const { entries } = loader; + try { + if (await this.isZip()) { + const loader = await this.makeZipLoader(); + const { entries } = loader; - if (this.isCBZ()) { - const { makeComicBook } = await import('foliate-js/comic-book.js'); - book = await makeComicBook(loader, this.file); - format = 'CBZ'; - } else if (this.isFBZ()) { - const entry = entries.find((entry) => entry.filename.endsWith(`.${EXTS.FB2}`)); - const blob = await loader.loadBlob((entry ?? entries[0]!).filename); + if (this.isCBZ()) { + const { makeComicBook } = await import('foliate-js/comic-book.js'); + book = await makeComicBook(loader, this.file); + format = 'CBZ'; + } else if (this.isFBZ()) { + const entry = entries.find((entry) => entry.filename.endsWith(`.${EXTS.FB2}`)); + const blob = await loader.loadBlob((entry ?? entries[0]!).filename); + const { makeFB2 } = await import('foliate-js/fb2.js'); + book = await makeFB2(blob); + format = 'FBZ'; + } else { + const { EPUB } = await import('foliate-js/epub.js'); + book = await new EPUB(loader).init(); + format = 'EPUB'; + } + } else if (await this.isPDF()) { + const { makePDF } = await import('foliate-js/pdf.js'); + book = await makePDF(this.file); + format = 'PDF'; + } else if (await (await import('foliate-js/mobi.js')).isMOBI(this.file)) { + const fflate = await import('foliate-js/vendor/fflate.js'); + const { MOBI } = await import('foliate-js/mobi.js'); + book = await new MOBI({ unzlib: fflate.unzlibSync }).open(this.file); + const ext = this.file.name.split('.').pop()?.toLowerCase(); + switch (ext) { + case 'azw': + format = 'AZW'; + break; + case 'azw3': + format = 'AZW3'; + break; + default: + format = 'MOBI'; + } + } else if (this.isFB2()) { const { makeFB2 } = await import('foliate-js/fb2.js'); - book = await makeFB2(blob); - format = 'FBZ'; - } else { - const { EPUB } = await import('foliate-js/epub.js'); - book = await new EPUB(loader).init(); - format = 'EPUB'; + book = await makeFB2(this.file); + format = 'FB2'; } - } else if (await this.isPDF()) { - const { makePDF } = await import('foliate-js/pdf.js'); - book = await makePDF(this.file); - format = 'PDF'; - } else if (await (await import('foliate-js/mobi.js')).isMOBI(this.file)) { - const fflate = await import('foliate-js/vendor/fflate.js'); - const { MOBI } = await import('foliate-js/mobi.js'); - book = await new MOBI({ unzlib: fflate.unzlibSync }).open(this.file); - const ext = this.file.name.split('.').pop()?.toLowerCase(); - switch (ext) { - case 'azw': - format = 'AZW'; - break; - case 'azw3': - format = 'AZW3'; - break; - default: - format = 'MOBI'; + } catch (e: unknown) { + console.error('Failed to open document:', e); + if (e instanceof Error && e.message?.includes('not a valid zip')) { + throw new Error('Unsupported or corrupted book file'); } - } else if (this.isFB2()) { - const { makeFB2 } = await import('foliate-js/fb2.js'); - book = await makeFB2(this.file); - format = 'FB2'; + throw e; } return { book, format } as { book: BookDoc; format: BookFormat }; } diff --git a/apps/readest-app/src/services/nativeAppService.ts b/apps/readest-app/src/services/nativeAppService.ts index 9edbac1d..43f16743 100644 --- a/apps/readest-app/src/services/nativeAppService.ts +++ b/apps/readest-app/src/services/nativeAppService.ts @@ -304,6 +304,36 @@ export const nativeFileSystem: FileSystem = { async readDir(path: string, base: BaseDir) { const { fp, baseDir } = this.resolvePath(path, base); + const getRelativePath = (filePath: string, basePath: string): string => { + let relativePath = filePath; + if (filePath.toLowerCase().startsWith(basePath.toLowerCase())) { + relativePath = filePath.substring(basePath.length); + } + if (relativePath.startsWith('\\') || relativePath.startsWith('/')) { + relativePath = relativePath.substring(1); + } + return relativePath; + }; + + // Use Rust WalkDir for massive performance gain on absolute paths + if (!baseDir || baseDir === 0) { + try { + const files = await invoke<{ path: string; size: number }[]>('read_dir', { + path: fp, + recursive: true, + extensions: ['*'], + }); + + return files.map((file) => ({ + path: getRelativePath(file.path, fp), + size: file.size, + })); + } catch (e) { + console.error('Rust read_dir failed, falling back to JS recursion', e); + } + } + + // Fallback to readDir for non-absolute paths or on error const entries = await readDir(fp, baseDir ? { baseDir } : undefined); const fileList: FileItem[] = []; const readDirRecursively = async ( @@ -316,8 +346,12 @@ export const nativeFileSystem: FileSystem = { if (entry.isDirectory) { const dir = await join(parent, entry.name); const relativeDir = relative ? await join(relative, entry.name) : entry.name; - const entries = await readDir(dir, baseDir ? { baseDir } : undefined); - await readDirRecursively(dir, relativeDir, entries, fileList); + try { + const entries = await readDir(dir, baseDir ? { baseDir } : undefined); + await readDirRecursively(dir, relativeDir, entries, fileList); + } catch { + console.warn(`Skipping unreadable dir: ${dir}`); + } } else { const filePath = await join(parent, entry.name); const relativePath = relative ? await join(relative, entry.name) : entry.name; diff --git a/apps/readest-app/src/store/libraryStore.ts b/apps/readest-app/src/store/libraryStore.ts index 5d19cce7..1a8e98cd 100644 --- a/apps/readest-app/src/store/libraryStore.ts +++ b/apps/readest-app/src/store/libraryStore.ts @@ -23,6 +23,7 @@ interface LibraryState { setCheckLastOpenBooks: (check: boolean) => void; setLibrary: (books: Book[]) => void; updateBook: (envConfig: EnvConfigType, book: Book) => void; + updateBooks: (envConfig: EnvConfigType, books: Book[]) => void; setCurrentBookshelf: (bookshelf: (Book | BooksGroup)[]) => void; refreshGroups: () => void; addGroup: (name: string) => BookGroupType; @@ -68,6 +69,17 @@ export const useLibraryStore = create((set, get) => ({ set({ library: [...library] }); await appService.saveLibraryBooks(library); }, + updateBooks: async (envConfig: EnvConfigType, books: Book[]) => { + if (!books?.length) return; + + const appService = await envConfig.getAppService(); + const { library, refreshGroups } = get(); + + const newLibrary = Array.from(new Map([...library, ...books].map((b) => [b.hash, b])).values()); + set({ library: newLibrary }); + refreshGroups(); + await appService.saveLibraryBooks(newLibrary); + }, setSelectedBooks: (ids: string[]) => { set({ selectedBooks: new Set(ids) });