From f9a6d6b6d421659657f46171613de167c3b246e5 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 17:39:13 -0800 Subject: [PATCH] Parallelize work during the clean step --- Cargo.lock | 13 +++++++------ Cargo.toml | 1 + src/index/update.rs | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d4d4a0..4462df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1004,7 +1004,7 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1539,6 +1539,7 @@ dependencies = [ "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1799,17 +1800,17 @@ dependencies = [ [[package]] name = "rayon" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rayon-core" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2926,8 +2927,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" "checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "43739f8831493b276363637423d3622d4bd6394ab6f0a9c4a552e208aeb7fddd" -"checksum rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bf17de6f23b05473c437eb958b9c850bfc8af0961fe17b4cc92d5a627b4791" +"checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +"checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecedbca3bf205f8d8f5c2b44d83cd0690e39ee84b951ed649e9f1841132b66d" diff --git a/Cargo.toml b/Cargo.toml index d4afb38..8c1d855 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ metaflac = "0.2" mp3-duration = "0.1" pbkdf2 = "0.3" rand = "0.7" +rayon = "1.3" regex = "1.2" reqwest = "0.9.2" rocket = { version = "0.4.2", optional = true } diff --git a/src/index/update.rs b/src/index/update.rs index 3ba30f6..d47b376 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -4,6 +4,7 @@ use diesel::prelude::*; #[cfg(feature = "profile-index")] use flame; use log::{error, info}; +use rayon::prelude::*; use regex::Regex; use std::fs; use std::path::Path; @@ -261,7 +262,7 @@ pub fn clean(db: &DB) -> Result<()> { } let missing_songs = all_songs - .into_iter() + .par_iter() .filter(|ref song_path| { let path = Path::new(&song_path); !path.exists() || vfs.real_to_virtual(path).is_err() @@ -287,7 +288,7 @@ pub fn clean(db: &DB) -> Result<()> { } let missing_directories = all_directories - .into_iter() + .par_iter() .filter(|ref directory_path| { let path = Path::new(&directory_path); !path.exists() || vfs.real_to_virtual(path).is_err()