perf: use native walkdir for recursive imports from directory (#2993)

* fix(perf): replace JS recursion with native Rust walkdir for imports

* fix: implement security scope check in rust recursive scanner

* refactor and format code

---------

Co-authored-by: Huang Xin <chrox.huang@gmail.com>

Cargo.lock

@@ -55,6 +55,7 @@ dependencies = [
  "thiserror 2.0.17",
  "tokio",
  "tokio-util",
+ "walkdir",
 ]
 
 [[package]]

apps/readest-app/src-tauri/Cargo.toml

@@ -26,6 +26,7 @@ serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 log = "0.4"
 thiserror = "2"
+walkdir = "2"
 tokio = { version = "1", features = ["fs"] }
 tokio-util = { version = "0.7", features = ["codec"] }
 futures-util = "0.3"

apps/readest-app/src-tauri/src/dir_scanner.rs

@@ -0,0 +1,96 @@
+use std::path::Path;
+use tauri::AppHandle;
+use tauri_plugin_fs::FsExt;
+use walkdir::WalkDir;
+
+#[derive(serde::Serialize)]
+pub struct ScannedFile {
+    pub path: String,
+    pub size: u64,
+}
+
+#[tauri::command]
+pub fn read_dir(
+    app: AppHandle,
+    path: String,
+    recursive: bool,
+    extensions: Vec<String>,
+) -> Result<Vec<ScannedFile>, String> {
+    let scope = app.fs_scope();
+    let path_buf = std::path::PathBuf::from(&path);
+
+    if !scope.is_allowed(&path_buf) {
+        return Err("Permission denied: Path not in filesystem scope".to_string());
+    }
+
+    let mut files = Vec::new();
+
+    let normalized_extensions: Vec<String> =
+        extensions.iter().map(|ext| ext.to_lowercase()).collect();
+
+    if recursive {
+        for entry_result in WalkDir::new(&path).into_iter() {
+            match entry_result {
+                Ok(entry) => {
+                    if entry.file_type().is_file() {
+                        if let Some(scanned_file) =
+                            process_file_entry(entry.path(), &normalized_extensions)
+                        {
+                            files.push(scanned_file);
+                        }
+                    }
+                }
+                Err(e) => {
+                    log::warn!("RUST: Skipping file due to error: {}", e);
+                }
+            }
+        }
+    } else {
+        match std::fs::read_dir(&path_buf) {
+            Ok(entries) => {
+                for entry_result in entries {
+                    match entry_result {
+                        Ok(entry) => {
+                            let path = entry.path();
+                            if path.is_file() {
+                                if let Some(scanned_file) =
+                                    process_file_entry(&path, &normalized_extensions)
+                                {
+                                    files.push(scanned_file);
+                                }
+                            }
+                        }
+                        Err(e) => {
+                            log::warn!("RUST: Skipping entry due to error: {}", e);
+                        }
+                    }
+                }
+            }
+            Err(e) => {
+                return Err(format!("Failed to read directory: {}", e));
+            }
+        }
+    }
+
+    Ok(files)
+}
+
+fn process_file_entry(path: &Path, extensions: &[String]) -> Option<ScannedFile> {
+    if extensions.is_empty() || extensions.contains(&"*".to_string()) {
+        let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+        return Some(ScannedFile {
+            path: path.to_string_lossy().to_string(),
+            size,
+        });
+    } else if let Some(ext) = path.extension() {
+        let ext_str = ext.to_string_lossy().to_lowercase();
+        if extensions.contains(&ext_str) {
+            let size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0);
+            return Some(ScannedFile {
+                path: path.to_string_lossy().to_string(),
+                size,
+            });
+        }
+    }
+    None
+}

apps/readest-app/src-tauri/src/lib.rs

@@ -22,6 +22,7 @@ use tauri_plugin_fs::FsExt;
 
 #[cfg(desktop)]
 use tauri::{Listener, Url};
+mod dir_scanner;
 #[cfg(any(target_os = "macos", target_os = "windows", target_os = "linux"))]
 mod discord_rpc;
 #[cfg(target_os = "macos")]
@@ -163,6 +164,7 @@ pub fn run() {
             upload_file,
             get_environment_variable,
             get_executable_dir,
+            dir_scanner::read_dir,
             #[cfg(target_os = "macos")]
             macos::safari_auth::auth_with_safari,
             #[cfg(target_os = "macos")]

apps/readest-app/src/app/library/page.tsx

@@ -83,6 +83,7 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
     isSyncing,
     syncProgress,
     updateBook,
+    updateBooks,
     setLibrary,
     getGroupId,
     getGroupName,
@@ -405,30 +406,29 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
       ['Unsupported or corrupted book file', _('The book file is corrupted')],
     ];
 
-    const processFile = async (selectedFile: SelectedFile) => {
+    const processFile = async (selectedFile: SelectedFile): Promise<Book | null> => {
       const file = selectedFile.file || selectedFile.path;
-      if (!file) return;
+      if (!file) return null;
       try {
         const book = await appService?.importBook(file, library);
+        if (!book) return null;
         const { path, basePath } = selectedFile;
-        if (book && groupId) {
+        if (groupId) {
           book.groupId = groupId;
           book.groupName = getGroupName(groupId);
-          await updateBook(envConfig, book);
-        } else if (book && path && basePath) {
+        } else if (path && basePath) {
           const rootPath = getDirPath(basePath);
           const groupName = getDirPath(path).replace(rootPath, '').replace(/^\//, '');
           book.groupName = groupName;
           book.groupId = getGroupId(groupName);
-          await updateBook(envConfig, book);
         }
-        if (user && book && !book.uploadedAt && settings.autoUpload) {
+
+        if (user && !book.uploadedAt && settings.autoUpload) {
           console.log('Queueing upload for book:', book.title);
           transferManager.queueUpload(book);
         }
-        if (book) {
-          successfulImports.push(book.title);
-        }
+        successfulImports.push(book.title);
+        return book;
       } catch (error) {
         const filename = typeof file === 'string' ? file : file.name;
         const baseFilename = getFilename(filename);
@@ -438,14 +438,17 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
             : '';
         failedImports.push({ filename: baseFilename, errorMessage });
         console.error('Failed to import book:', filename, error);
+        return null;
       }
     };
 
     const concurrency = 4;
     for (let i = 0; i < files.length; i += concurrency) {
       const batch = files.slice(i, i + concurrency);
-      await Promise.all(batch.map(processFile));
+      const importedBooks = (await Promise.all(batch.map(processFile))).filter((book) => !!book);
+      await updateBooks(envConfig, importedBooks);
     }
+
     pushLibrary();
 
     if (failedImports.length > 0) {
@@ -470,8 +473,6 @@ const LibraryPageContent = ({ searchParams }: { searchParams: ReadonlyURLSearchP
       });
     }
 
-    setLibrary([...library]);
-    appService?.saveLibraryBooks(library);
     setLoading(false);
   };
 

apps/readest-app/src/libs/document.ts

@@ -192,48 +192,56 @@ export class DocumentLoader {
     if (!this.file.size) {
       throw new Error('File is empty');
     }
-    if (await this.isZip()) {
-      const loader = await this.makeZipLoader();
-      const { entries } = loader;
+    try {
+      if (await this.isZip()) {
+        const loader = await this.makeZipLoader();
+        const { entries } = loader;
 
-      if (this.isCBZ()) {
-        const { makeComicBook } = await import('foliate-js/comic-book.js');
-        book = await makeComicBook(loader, this.file);
-        format = 'CBZ';
-      } else if (this.isFBZ()) {
-        const entry = entries.find((entry) => entry.filename.endsWith(`.${EXTS.FB2}`));
-        const blob = await loader.loadBlob((entry ?? entries[0]!).filename);
+        if (this.isCBZ()) {
+          const { makeComicBook } = await import('foliate-js/comic-book.js');
+          book = await makeComicBook(loader, this.file);
+          format = 'CBZ';
+        } else if (this.isFBZ()) {
+          const entry = entries.find((entry) => entry.filename.endsWith(`.${EXTS.FB2}`));
+          const blob = await loader.loadBlob((entry ?? entries[0]!).filename);
+          const { makeFB2 } = await import('foliate-js/fb2.js');
+          book = await makeFB2(blob);
+          format = 'FBZ';
+        } else {
+          const { EPUB } = await import('foliate-js/epub.js');
+          book = await new EPUB(loader).init();
+          format = 'EPUB';
+        }
+      } else if (await this.isPDF()) {
+        const { makePDF } = await import('foliate-js/pdf.js');
+        book = await makePDF(this.file);
+        format = 'PDF';
+      } else if (await (await import('foliate-js/mobi.js')).isMOBI(this.file)) {
+        const fflate = await import('foliate-js/vendor/fflate.js');
+        const { MOBI } = await import('foliate-js/mobi.js');
+        book = await new MOBI({ unzlib: fflate.unzlibSync }).open(this.file);
+        const ext = this.file.name.split('.').pop()?.toLowerCase();
+        switch (ext) {
+          case 'azw':
+            format = 'AZW';
+            break;
+          case 'azw3':
+            format = 'AZW3';
+            break;
+          default:
+            format = 'MOBI';
+        }
+      } else if (this.isFB2()) {
         const { makeFB2 } = await import('foliate-js/fb2.js');
-        book = await makeFB2(blob);
-        format = 'FBZ';
-      } else {
-        const { EPUB } = await import('foliate-js/epub.js');
-        book = await new EPUB(loader).init();
-        format = 'EPUB';
+        book = await makeFB2(this.file);
+        format = 'FB2';
       }
-    } else if (await this.isPDF()) {
-      const { makePDF } = await import('foliate-js/pdf.js');
-      book = await makePDF(this.file);
-      format = 'PDF';
-    } else if (await (await import('foliate-js/mobi.js')).isMOBI(this.file)) {
-      const fflate = await import('foliate-js/vendor/fflate.js');
-      const { MOBI } = await import('foliate-js/mobi.js');
-      book = await new MOBI({ unzlib: fflate.unzlibSync }).open(this.file);
-      const ext = this.file.name.split('.').pop()?.toLowerCase();
-      switch (ext) {
-        case 'azw':
-          format = 'AZW';
-          break;
-        case 'azw3':
-          format = 'AZW3';
-          break;
-        default:
-          format = 'MOBI';
+    } catch (e: unknown) {
+      console.error('Failed to open document:', e);
+      if (e instanceof Error && e.message?.includes('not a valid zip')) {
+        throw new Error('Unsupported or corrupted book file');
       }
-    } else if (this.isFB2()) {
-      const { makeFB2 } = await import('foliate-js/fb2.js');
-      book = await makeFB2(this.file);
-      format = 'FB2';
+      throw e;
     }
     return { book, format } as { book: BookDoc; format: BookFormat };
   }

apps/readest-app/src/services/nativeAppService.ts

@@ -304,6 +304,36 @@ export const nativeFileSystem: FileSystem = {
   async readDir(path: string, base: BaseDir) {
     const { fp, baseDir } = this.resolvePath(path, base);
 
+    const getRelativePath = (filePath: string, basePath: string): string => {
+      let relativePath = filePath;
+      if (filePath.toLowerCase().startsWith(basePath.toLowerCase())) {
+        relativePath = filePath.substring(basePath.length);
+      }
+      if (relativePath.startsWith('\\') || relativePath.startsWith('/')) {
+        relativePath = relativePath.substring(1);
+      }
+      return relativePath;
+    };
+
+    // Use Rust WalkDir for massive performance gain on absolute paths
+    if (!baseDir || baseDir === 0) {
+      try {
+        const files = await invoke<{ path: string; size: number }[]>('read_dir', {
+          path: fp,
+          recursive: true,
+          extensions: ['*'],
+        });
+
+        return files.map((file) => ({
+          path: getRelativePath(file.path, fp),
+          size: file.size,
+        }));
+      } catch (e) {
+        console.error('Rust read_dir failed, falling back to JS recursion', e);
+      }
+    }
+
+    // Fallback to readDir for non-absolute paths or on error
     const entries = await readDir(fp, baseDir ? { baseDir } : undefined);
     const fileList: FileItem[] = [];
     const readDirRecursively = async (
@@ -316,8 +346,12 @@ export const nativeFileSystem: FileSystem = {
         if (entry.isDirectory) {
           const dir = await join(parent, entry.name);
           const relativeDir = relative ? await join(relative, entry.name) : entry.name;
-          const entries = await readDir(dir, baseDir ? { baseDir } : undefined);
-          await readDirRecursively(dir, relativeDir, entries, fileList);
+          try {
+            const entries = await readDir(dir, baseDir ? { baseDir } : undefined);
+            await readDirRecursively(dir, relativeDir, entries, fileList);
+          } catch {
+            console.warn(`Skipping unreadable dir: ${dir}`);
+          }
         } else {
           const filePath = await join(parent, entry.name);
           const relativePath = relative ? await join(relative, entry.name) : entry.name;

apps/readest-app/src/store/libraryStore.ts

@@ -23,6 +23,7 @@ interface LibraryState {
   setCheckLastOpenBooks: (check: boolean) => void;
   setLibrary: (books: Book[]) => void;
   updateBook: (envConfig: EnvConfigType, book: Book) => void;
+  updateBooks: (envConfig: EnvConfigType, books: Book[]) => void;
   setCurrentBookshelf: (bookshelf: (Book | BooksGroup)[]) => void;
   refreshGroups: () => void;
   addGroup: (name: string) => BookGroupType;
@@ -68,6 +69,17 @@ export const useLibraryStore = create<LibraryState>((set, get) => ({
     set({ library: [...library] });
     await appService.saveLibraryBooks(library);
   },
+  updateBooks: async (envConfig: EnvConfigType, books: Book[]) => {
+    if (!books?.length) return;
+
+    const appService = await envConfig.getAppService();
+    const { library, refreshGroups } = get();
+
+    const newLibrary = Array.from(new Map([...library, ...books].map((b) => [b.hash, b])).values());
+    set({ library: newLibrary });
+    refreshGroups();
+    await appService.saveLibraryBooks(newLibrary);
+  },
 
   setSelectedBooks: (ids: string[]) => {
     set({ selectedBooks: new Set(ids) });