Merge pull request #86 from lnicola/parallel-index

Fix build and populate index in parallel

Cargo.lock

@@ -325,6 +325,15 @@ dependencies = [
  "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+dependencies = [
+ "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
+]
+
 [[package]]
 name = "crossbeam-deque"
 version = "0.7.3"
@@ -1419,22 +1428,22 @@ dependencies = [
 
 [[package]]
 name = "pear"
-version = "0.1.2"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "pear_codegen 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pear_codegen 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
 name = "pear_codegen"
-version = "0.1.2"
+version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)",
  "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)",
  "syn 0.15.44 (registry+https://github.com/rust-lang/crates.io-index)",
- "version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
+ "version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -1472,6 +1481,7 @@ dependencies = [
  "app_dirs 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "cookie 0.14.0 (registry+https://github.com/rust-lang/crates.io-index)",
+ "crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
  "diesel 1.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "diesel_migrations 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "flame 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1856,7 +1866,7 @@ dependencies = [
  "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
  "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)",
- "pear 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pear 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocket_codegen 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "rocket_http 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)",
  "state 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1900,7 +1910,7 @@ dependencies = [
  "cookie 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
  "hyper 0.10.16 (registry+https://github.com/rust-lang/crates.io-index)",
  "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
- "pear 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
+ "pear 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "smallvec 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "state 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -2790,11 +2800,6 @@ name = "xdg"
 version = "2.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 
-[[package]]
-name = "yansi"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-
 [[package]]
 name = "yansi"
 version = "0.5.0"
@@ -2847,6 +2852,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum core-foundation 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171"
 "checksum core-foundation-sys 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac"
 "checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1"
+"checksum crossbeam-channel 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cced8691919c02aac3cb0a1bc2e9b73d89e832bf9a06fc579d4e71b68a2da061"
 "checksum crossbeam-deque 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285"
 "checksum crossbeam-epoch 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
 "checksum crossbeam-queue 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ab6bffe714b6bb07e42f201352c34f51fefd355ace793f9e638ebd52d23f98d2"
@@ -2969,8 +2975,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum parking_lot_core 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b"
 "checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3"
 "checksum pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "006c038a43a45995a9670da19e67600114740e8511d4333bf97a56e66a7542d9"
-"checksum pear 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c26d2b92e47063ffce70d3e3b1bd097af121a9e0db07ca38a6cc1cf0cc85ff25"
+"checksum pear 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5320f212db967792b67cfe12bd469d08afd6318a249bd917d5c19bc92200ab8a"
-"checksum pear_codegen 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "336db4a192cc7f54efeb0c4e11a9245394824cc3bcbd37ba3ff51240c35d7a6e"
+"checksum pear_codegen 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bfc1c836fdc3d1ef87c348b237b5b5c4dff922156fb2d968f57734f9669768ca"
 "checksum percent-encoding 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "31010dd2e1ac33d5b46a5b413495239882813e0369f8ed8a5e266f173602f831"
 "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 "checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
@@ -3120,6 +3126,5 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum wrapped-vec 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "06c29bb4abe93d1c8ef79b60f270d0efcaa6c5c97aaaaaaa0d477ea72f5f9e45"
 "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
 "checksum xdg 2.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d089681aa106a86fade1b0128fb5daf07d5867a509ab036d99988dec80429a57"
-"checksum yansi 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d60c3b48c9cdec42fb06b3b84b5b087405e1fa1c644a1af3930e4dfafe93de48"
 "checksum yansi 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9fc79f4a1e39857fc00c3f662cbf2651c771f00e9c15fe2abc341806bd46bd71"
 "checksum zeroize 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3cbac2ed2ba24cc90f5e06485ac8c7c1e5449fe8911aef4d8877218af021a5b8"

Cargo.toml

@@ -15,6 +15,7 @@ anyhow = "1.0.31"
 ape = "0.3.0"
 app_dirs = "1.1.1"
 base64 = "0.12.1"
+crossbeam-channel = "0.4"
 diesel = { version = "1.4.4", features = ["sqlite", "r2d2"] }
 diesel_migrations = { version = "1.4", features = ["sqlite"] }
 flame = { version = "0.2.2", optional = true }

src/index/update.rs

@@ -1,4 +1,5 @@
 use anyhow::*;
+use crossbeam_channel::{Receiver, Sender};
 use diesel;
 use diesel::prelude::*;
 #[cfg(feature = "profile-index")]
@@ -7,14 +8,14 @@ use log::{error, info};
 use rayon::prelude::*;
 use regex::Regex;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
-use std::sync::mpsc::*;
 use std::time;
 
 use crate::config::MiscSettings;
 use crate::db::{directories, misc_settings, songs, DB};
 use crate::index::metadata;
 use crate::vfs::VFSSource;
+use metadata::SongTags;
 
 const INDEX_BUILDING_INSERT_BUFFER_SIZE: usize = 1000; // Insertions in each transaction
 const INDEX_BUILDING_CLEAN_BUFFER_SIZE: usize = 500; // Insertions in each transaction
@@ -82,12 +83,12 @@ impl IndexUpdater {
 	}
 
 	#[cfg_attr(feature = "profile-index", flame)]
-	fn push_song(&mut self, song: NewSong) -> Result<()> {
+	fn push_song(&self, song: NewSong) -> Result<()> {
 		self.song_sender.send(song).map_err(Error::new)
 	}
 
 	#[cfg_attr(feature = "profile-index", flame)]
-	fn push_directory(&mut self, directory: NewDirectory) -> Result<()> {
+	fn push_directory(&self, directory: NewDirectory) -> Result<()> {
 		self.directory_sender.send(directory).map_err(Error::new)
 	}
 
@@ -103,7 +104,7 @@ impl IndexUpdater {
 		Ok(None)
 	}
 
-	fn populate_directory(&mut self, parent: Option<&Path>, path: &Path) -> Result<()> {
+	fn populate_directory(&self, parent: Option<&Path>, path: &Path) -> Result<()> {
 		#[cfg(feature = "profile-index")]
 		let _guard = flame::start_guard(format!(
 			"dir: {}",
@@ -148,13 +149,22 @@ impl IndexUpdater {
 
 		// Sub directories
 		let mut sub_directories = Vec::new();
+		let mut song_files = Vec::new();
+
+		let files = match fs::read_dir(path) {
+			Ok(files) => files,
+			Err(e) => {
+				error!("Directory read error for `{}`: {}", path.display(), e);
+				return Err(e.into());
+			}
+		};
 
 		// Insert content
-		for file in fs::read_dir(path)? {
+		for file in files {
 			let file_path = match file {
 				Ok(ref f) => f.path(),
-				_ => {
+				Err(e) => {
-					error!("File read error within {}", path_string);
+					error!("File read error within `{}`: {}", path_string, e);
 					break;
 				}
 			};
@@ -174,54 +184,64 @@ impl IndexUpdater {
 				continue;
 			}
 
-			if let Some(file_path_string) = file_path.to_str() {
+			song_files.push(file_path);
-				if let Some(tags) = metadata::read(file_path.as_path()) {
+		}
-					if tags.year.is_some() {
-						inconsistent_directory_year |=
-							directory_year.is_some() && directory_year != tags.year;
-						directory_year = tags.year;
-					}
 
-					if tags.album.is_some() {
+		let song_metadata = |path: PathBuf| -> Option<(String, SongTags)> {
-						inconsistent_directory_album |=
+			#[cfg(feature = "profile-index")]
-							directory_album.is_some() && directory_album != tags.album;
+			let _guard = flame::start_guard("song_metadata");
-						directory_album = tags.album.as_ref().cloned();
-					}
 
-					if tags.album_artist.is_some() {
+			path.to_str().and_then(|file_path_string| {
-						inconsistent_directory_artist |=
+				metadata::read(&path).map(|m| (file_path_string.to_owned(), m))
-							directory_artist.is_some() && directory_artist != tags.album_artist;
+			})
-						directory_artist = tags.album_artist.as_ref().cloned();
+		};
-					} else if tags.artist.is_some() {
+		let song_tags = song_files
-						inconsistent_directory_artist |=
+			.into_par_iter()
-							directory_artist.is_some() && directory_artist != tags.artist;
+			.filter_map(song_metadata)
-						directory_artist = tags.artist.as_ref().cloned();
+			.collect::<Vec<_>>();
-					}
 
-					let song = NewSong {
+		for (file_path_string, tags) in song_tags {
-						path: file_path_string.to_owned(),
+			if tags.year.is_some() {
-						parent: path_string.to_owned(),
+				inconsistent_directory_year |=
-						disc_number: tags.disc_number.map(|n| n as i32),
+					directory_year.is_some() && directory_year != tags.year;
-						track_number: tags.track_number.map(|n| n as i32),
+				directory_year = tags.year;
-						title: tags.title,
-						duration: tags.duration.map(|n| n as i32),
-						artist: tags.artist,
-						album_artist: tags.album_artist,
-						album: tags.album,
-						year: tags.year,
-						artwork: artwork.as_ref().cloned(),
-					};
-
-					self.push_song(song)?;
-				}
 			}
+
+			if tags.album.is_some() {
+				inconsistent_directory_album |=
+					directory_album.is_some() && directory_album != tags.album;
+				directory_album = tags.album.as_ref().cloned();
+			}
+
+			if tags.album_artist.is_some() {
+				inconsistent_directory_artist |=
+					directory_artist.is_some() && directory_artist != tags.album_artist;
+				directory_artist = tags.album_artist.as_ref().cloned();
+			} else if tags.artist.is_some() {
+				inconsistent_directory_artist |=
+					directory_artist.is_some() && directory_artist != tags.artist;
+				directory_artist = tags.artist.as_ref().cloned();
+			}
+
+			let song = NewSong {
+				path: file_path_string.to_owned(),
+				parent: path_string.to_owned(),
+				disc_number: tags.disc_number.map(|n| n as i32),
+				track_number: tags.track_number.map(|n| n as i32),
+				title: tags.title,
+				duration: tags.duration.map(|n| n as i32),
+				artist: tags.artist,
+				album_artist: tags.album_artist,
+				album: tags.album,
+				year: tags.year,
+				artwork: artwork.as_ref().cloned(),
+			};
+
+			self.push_song(song)?;
 		}
 
 		// Insert directory
 		let directory = {
-			#[cfg(feature = "profile-index")]
-			let _guard = flame::start_guard("create_directory");
-
 			if inconsistent_directory_year {
 				directory_year = None;
 			}
@@ -246,11 +266,10 @@ impl IndexUpdater {
 		self.push_directory(directory)?;
 
 		// Populate subdirectories
-		for sub_directory in sub_directories {
+		sub_directories
-			self.populate_directory(Some(path), &sub_directory)?;
+			.into_par_iter()
-		}
+			.map(|sub_directory| self.populate_directory(Some(path), &sub_directory))
-
+			.collect() // propagate an error to the caller if one of them failed
-		Ok(())
 	}
 }
 
@@ -322,8 +341,8 @@ pub fn populate(db: &DB) -> Result<()> {
 		Regex::new(&settings.index_album_art_pattern)?
 	};
 
-	let (directory_sender, directory_receiver) = channel();
+	let (directory_sender, directory_receiver) = crossbeam_channel::unbounded();
-	let (song_sender, song_receiver) = channel();
+	let (song_sender, song_receiver) = crossbeam_channel::unbounded();
 
 	let songs_db = db.clone();
 	let directories_db = db.clone();
@@ -337,10 +356,13 @@ pub fn populate(db: &DB) -> Result<()> {
 	});
 
 	{
-		let mut updater = IndexUpdater::new(album_art_pattern, directory_sender, song_sender)?;
+		let updater = IndexUpdater::new(album_art_pattern, directory_sender, song_sender)?;
-		for target in mount_points.values() {
+		let mount_points = mount_points.values().collect::<Vec<_>>();
-			updater.populate_directory(None, target.as_path())?;
+		mount_points
-		}
+			.iter()
+			.par_bridge()
+			.map(|target| updater.populate_directory(None, target.as_path()))
+			.collect::<Result<()>>()?;
 	}
 
 	match directories_thread.join() {