From f9a6d6b6d421659657f46171613de167c3b246e5 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 17:39:13 -0800 Subject: [PATCH 01/11] Parallelize work during the clean step --- Cargo.lock | 13 +++++++------ Cargo.toml | 1 + src/index/update.rs | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d4d4a0..4462df9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1004,7 +1004,7 @@ version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1539,6 +1539,7 @@ dependencies = [ "pbkdf2 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "regex 1.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.9.24 (registry+https://github.com/rust-lang/crates.io-index)", "rocket 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1799,17 +1800,17 @@ dependencies = [ [[package]] name = "rayon" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "either 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)", - "rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rayon-core" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "crossbeam-deque 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -2926,8 +2927,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum rand_os 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" "checksum rand_pcg 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" "checksum rand_xorshift 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -"checksum rayon 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "43739f8831493b276363637423d3622d4bd6394ab6f0a9c4a552e208aeb7fddd" -"checksum rayon-core 1.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f8bf17de6f23b05473c437eb958b9c850bfc8af0961fe17b4cc92d5a627b4791" +"checksum rayon 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +"checksum rayon-core 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" "checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" "checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" "checksum redox_users 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecedbca3bf205f8d8f5c2b44d83cd0690e39ee84b951ed649e9f1841132b66d" diff --git a/Cargo.toml b/Cargo.toml index d4afb38..8c1d855 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ metaflac = "0.2" mp3-duration = "0.1" pbkdf2 = "0.3" rand = "0.7" +rayon = "1.3" regex = "1.2" reqwest = "0.9.2" rocket = { version = "0.4.2", optional = true } diff --git a/src/index/update.rs b/src/index/update.rs index 3ba30f6..d47b376 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -4,6 +4,7 @@ use diesel::prelude::*; #[cfg(feature = "profile-index")] use flame; use log::{error, info}; +use rayon::prelude::*; use regex::Regex; use std::fs; use std::path::Path; @@ -261,7 +262,7 @@ pub fn clean(db: &DB) -> Result<()> { } let missing_songs = all_songs - .into_iter() + .par_iter() .filter(|ref song_path| { let path = Path::new(&song_path); !path.exists() || vfs.real_to_virtual(path).is_err() @@ -287,7 +288,7 @@ pub fn clean(db: &DB) -> Result<()> { } let missing_directories = all_directories - .into_iter() + .par_iter() .filter(|ref directory_path| { let path = Path::new(&directory_path); !path.exists() || vfs.real_to_virtual(path).is_err() From 25b36be0732872e61da8058ff9ee1276fc279a28 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 17:41:57 -0800 Subject: [PATCH 02/11] More accurate index duration display --- src/index/update.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index/update.rs b/src/index/update.rs index d47b376..3ae790c 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -25,7 +25,7 @@ pub fn update(db: &DB) -> Result<()> { populate(db)?; info!( "Library index update took {} seconds", - start.elapsed().as_secs() + start.elapsed().as_millis() as f32 / 1000.0 ); #[cfg(feature = "profile-index")] flame::dump_html(&mut fs::File::create("index-flame-graph.html").unwrap()).unwrap(); From b6d985859c7ef2e3fbb09c2d64463f357927106e Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 19:40:46 -0800 Subject: [PATCH 03/11] Renamed IndexBuilder --- src/index/update.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index 3ae790c..de54610 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -60,21 +60,21 @@ struct NewDirectory { date_added: i32, } -struct IndexBuilder { +struct IndexUpdater { new_songs: Vec, new_directories: Vec, db: DB, album_art_pattern: Regex, } -impl IndexBuilder { +impl IndexUpdater { #[cfg_attr(feature = "profile-index", flame)] - fn new(db: DB, album_art_pattern: Regex) -> Result { + fn new(db: DB, album_art_pattern: Regex) -> Result { let mut new_songs = Vec::new(); let mut new_directories = Vec::new(); new_songs.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); new_directories.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); - Ok(IndexBuilder { + Ok(IndexUpdater { new_songs, new_directories, db, @@ -319,7 +319,7 @@ pub fn populate(db: &DB) -> Result<()> { album_art_pattern = Regex::new(&settings.index_album_art_pattern)?; } - let mut builder = IndexBuilder::new(db.clone(), album_art_pattern)?; + let mut builder = IndexUpdater::new(db.clone(), album_art_pattern)?; for target in mount_points.values() { builder.populate_directory(None, target.as_path())?; } From 18bc9594a44478e0f00ad5ac275bbc88c5348641 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 19:41:07 -0800 Subject: [PATCH 04/11] Local variable rename --- src/index/update.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index de54610..7a22f19 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -319,11 +319,11 @@ pub fn populate(db: &DB) -> Result<()> { album_art_pattern = Regex::new(&settings.index_album_art_pattern)?; } - let mut builder = IndexUpdater::new(db.clone(), album_art_pattern)?; + let mut updater = IndexUpdater::new(db.clone(), album_art_pattern)?; for target in mount_points.values() { - builder.populate_directory(None, target.as_path())?; + updater.populate_directory(None, target.as_path())?; } - builder.flush_songs()?; - builder.flush_directories()?; + updater.flush_songs()?; + updater.flush_directories()?; Ok(()) } From 1764f3da4d8f0070eb1bdee3334f72bc0729a6fb Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 21:57:44 -0800 Subject: [PATCH 05/11] Moved database insertions to separate threads from the file crawl --- src/index/update.rs | 122 ++++++++++++++++++++++++++++---------------- 1 file changed, 78 insertions(+), 44 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index 7a22f19..f207b1b 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -9,6 +9,7 @@ use regex::Regex; use std::fs; use std::path::Path; use std::time; +use std::sync::mpsc::*; use crate::config::MiscSettings; use crate::db::{directories, misc_settings, songs, DB}; @@ -61,63 +62,29 @@ struct NewDirectory { } struct IndexUpdater { - new_songs: Vec, - new_directories: Vec, - db: DB, + directory_sender: Sender, + song_sender: Sender, album_art_pattern: Regex, } impl IndexUpdater { #[cfg_attr(feature = "profile-index", flame)] - fn new(db: DB, album_art_pattern: Regex) -> Result { - let mut new_songs = Vec::new(); - let mut new_directories = Vec::new(); - new_songs.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); - new_directories.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); + fn new(album_art_pattern: Regex, directory_sender: Sender, song_sender: Sender) -> Result { Ok(IndexUpdater { - new_songs, - new_directories, - db, + directory_sender, + song_sender, album_art_pattern, }) } - #[cfg_attr(feature = "profile-index", flame)] - fn flush_songs(&mut self) -> Result<()> { - let connection = self.db.connect()?; - diesel::insert_into(songs::table) - .values(&self.new_songs) - .execute(&*connection)?; // TODO https://github.com/diesel-rs/diesel/issues/1822 - self.new_songs.clear(); - Ok(()) - } - - #[cfg_attr(feature = "profile-index", flame)] - fn flush_directories(&mut self) -> Result<()> { - let connection = self.db.connect()?; - diesel::insert_into(directories::table) - .values(&self.new_directories) - .execute(&*connection)?; // TODO https://github.com/diesel-rs/diesel/issues/1822 - self.new_directories.clear(); - Ok(()) - } - #[cfg_attr(feature = "profile-index", flame)] fn push_song(&mut self, song: NewSong) -> Result<()> { - if self.new_songs.len() >= self.new_songs.capacity() { - self.flush_songs()?; - } - self.new_songs.push(song); - Ok(()) + self.song_sender.send(song).map_err(Error::new) } #[cfg_attr(feature = "profile-index", flame)] fn push_directory(&mut self, directory: NewDirectory) -> Result<()> { - if self.new_directories.len() >= self.new_directories.capacity() { - self.flush_directories()?; - } - self.new_directories.push(directory); - Ok(()) + self.directory_sender.send(directory).map_err(Error::new) } fn get_artwork(&self, dir: &Path) -> Result> { @@ -319,11 +286,78 @@ pub fn populate(db: &DB) -> Result<()> { album_art_pattern = Regex::new(&settings.index_album_art_pattern)?; } - let mut updater = IndexUpdater::new(db.clone(), album_art_pattern)?; + let (directory_sender, directory_receiver) = channel(); + let directories_db = db.clone(); + std::thread::spawn(move || { + insert_directories(directory_receiver, directories_db); + }); + + let (song_sender, song_receiver) = channel(); + let songs_db = db.clone(); + std::thread::spawn(move || { + insert_songs(song_receiver, songs_db); + }); + + let mut updater = IndexUpdater::new(album_art_pattern, directory_sender, song_sender)?; for target in mount_points.values() { updater.populate_directory(None, target.as_path())?; } - updater.flush_songs()?; - updater.flush_directories()?; + Ok(()) } + +fn insert_songs(receiver: Receiver, db: DB) { + let mut new_entries = Vec::new(); + new_entries.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); + + loop { + match receiver.recv() { + Ok(s) => { + new_entries.push(s); + if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { + let connection = db.connect().unwrap(); + diesel::insert_into(songs::table) + .values(&new_entries) + .execute(&*connection).unwrap(); // TODO https://github.com/diesel-rs/diesel/issues/1822 + new_entries.clear(); + } + }, + Err(_) => break, + } + } + + if new_entries.len() > 0 { + let connection = db.connect().unwrap(); + diesel::insert_into(songs::table) + .values(&new_entries) + .execute(&*connection).unwrap(); + } +} + +fn insert_directories(receiver: Receiver, db: DB) { + let mut new_entries = Vec::new(); + new_entries.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); + + loop { + match receiver.recv() { + Ok(s) => { + new_entries.push(s); + if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { + let connection = db.connect().unwrap(); + diesel::insert_into(directories::table) + .values(&new_entries) + .execute(&*connection).unwrap(); // TODO https://github.com/diesel-rs/diesel/issues/1822 + new_entries.clear(); + } + }, + Err(_) => break, + } + } + + if new_entries.len() > 0 { + let connection = db.connect().unwrap(); + diesel::insert_into(directories::table) + .values(&new_entries) + .execute(&*connection).unwrap(); + } +} From b8b3c80be9f31e7230b0fc95f83bfee92eb41078 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 22:20:17 -0800 Subject: [PATCH 06/11] Don't emit errors on critical path --- src/index/update.rs | 2 +- src/metadata.rs | 23 ++++++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index f207b1b..35ee181 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -144,7 +144,7 @@ impl IndexUpdater { } if let Some(file_path_string) = file_path.to_str() { - if let Ok(tags) = metadata::read(file_path.as_path()) { + if let Some(tags) = metadata::read(file_path.as_path()) { if tags.year.is_some() { inconsistent_directory_year |= directory_year.is_some() && directory_year != tags.year; diff --git a/src/metadata.rs b/src/metadata.rs index 033530a..b31794b 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -2,6 +2,7 @@ use anyhow::*; use ape; use id3; use lewton::inside_ogg::OggStreamReader; +use log::error; use metaflac; use mp3_duration; use regex::Regex; @@ -24,13 +25,21 @@ pub struct SongTags { } #[cfg_attr(feature = "profile-index", flame)] -pub fn read(path: &Path) -> Result { - match utils::get_audio_format(path) { - Some(AudioFormat::FLAC) => read_flac(path), - Some(AudioFormat::MP3) => read_id3(path), - Some(AudioFormat::MPC) => read_ape(path), - Some(AudioFormat::OGG) => read_vorbis(path), - _ => bail!("Unsupported file format for reading metadata"), +pub fn read(path: &Path) -> Option { + let data = match utils::get_audio_format(path) { + Some(AudioFormat::FLAC) => Some(read_flac(path)), + Some(AudioFormat::MP3) => Some(read_id3(path)), + Some(AudioFormat::MPC) => Some(read_ape(path)), + Some(AudioFormat::OGG) => Some(read_vorbis(path)), + _ => None, + }; + match data { + Some(Ok(d)) => Some(d), + Some(Err(e)) => { + error!("Error while reading file metadata for '{:?}': {}", path, e); + None + } + None => None, } } From f6b9e67d4e53f1323588ae6358e460d4d8df1a24 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 22:20:59 -0800 Subject: [PATCH 07/11] Error handling for index insertions --- src/index/update.rs | 56 ++++++++++++++++++++++++++++++++------------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index 35ee181..c117db7 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -315,10 +315,16 @@ fn insert_songs(receiver: Receiver, db: DB) { Ok(s) => { new_entries.push(s); if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { - let connection = db.connect().unwrap(); - diesel::insert_into(songs::table) - .values(&new_entries) - .execute(&*connection).unwrap(); // TODO https://github.com/diesel-rs/diesel/issues/1822 + if db.connect() + .and_then(|connection|{ + diesel::insert_into(songs::table) + .values(&new_entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new songs in database"); + } new_entries.clear(); } }, @@ -327,10 +333,16 @@ fn insert_songs(receiver: Receiver, db: DB) { } if new_entries.len() > 0 { - let connection = db.connect().unwrap(); - diesel::insert_into(songs::table) - .values(&new_entries) - .execute(&*connection).unwrap(); + if db.connect() + .and_then(|connection|{ + diesel::insert_into(songs::table) + .values(&new_entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new songs in database"); + } } } @@ -343,10 +355,16 @@ fn insert_directories(receiver: Receiver, db: DB) { Ok(s) => { new_entries.push(s); if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { - let connection = db.connect().unwrap(); - diesel::insert_into(directories::table) - .values(&new_entries) - .execute(&*connection).unwrap(); // TODO https://github.com/diesel-rs/diesel/issues/1822 + if db.connect() + .and_then(|connection|{ + diesel::insert_into(directories::table) + .values(&new_entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new directories in database"); + } new_entries.clear(); } }, @@ -355,9 +373,15 @@ fn insert_directories(receiver: Receiver, db: DB) { } if new_entries.len() > 0 { - let connection = db.connect().unwrap(); - diesel::insert_into(directories::table) - .values(&new_entries) - .execute(&*connection).unwrap(); + if db.connect() + .and_then(|connection|{ + diesel::insert_into(directories::table) + .values(&new_entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new directories in database"); + } } } From d1bb60a1c7235fbd1c4d6913bc0ae48b8a4fcc72 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 23:21:45 -0800 Subject: [PATCH 08/11] Moved metadata module under index --- src/{ => index}/metadata.rs | 0 src/index/mod.rs | 1 + src/index/update.rs | 2 +- src/main.rs | 1 - 4 files changed, 2 insertions(+), 2 deletions(-) rename src/{ => index}/metadata.rs (100%) diff --git a/src/metadata.rs b/src/index/metadata.rs similarity index 100% rename from src/metadata.rs rename to src/index/metadata.rs diff --git a/src/index/mod.rs b/src/index/mod.rs index dcd146b..5c5e99b 100644 --- a/src/index/mod.rs +++ b/src/index/mod.rs @@ -11,6 +11,7 @@ use crate::db::{misc_settings, DB}; use crate::config::MiscSettings; use crate::vfs::VFS; +mod metadata; mod query; #[cfg(test)] mod test; diff --git a/src/index/update.rs b/src/index/update.rs index c117db7..51dccd7 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -13,7 +13,7 @@ use std::sync::mpsc::*; use crate::config::MiscSettings; use crate::db::{directories, misc_settings, songs, DB}; -use crate::metadata; +use crate::index::metadata; use crate::vfs::VFSSource; const INDEX_BUILDING_INSERT_BUFFER_SIZE: usize = 1000; // Insertions in each transaction diff --git a/src/main.rs b/src/main.rs index 5548708..495d401 100644 --- a/src/main.rs +++ b/src/main.rs @@ -31,7 +31,6 @@ mod db; mod ddns; mod index; mod lastfm; -mod metadata; mod playlist; mod service; From f71a8320e92c1fdda679efd55427374956531259 Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sat, 18 Jan 2020 23:38:18 -0800 Subject: [PATCH 09/11] Don't return from populate() while still writing to database --- src/index/update.rs | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index 51dccd7..0baeea5 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -279,28 +279,41 @@ pub fn populate(db: &DB) -> Result<()> { let vfs = db.get_vfs()?; let mount_points = vfs.get_mount_points(); - let album_art_pattern; - { + let album_art_pattern = { let connection = db.connect()?; let settings: MiscSettings = misc_settings::table.get_result(&connection)?; - album_art_pattern = Regex::new(&settings.index_album_art_pattern)?; - } + Regex::new(&settings.index_album_art_pattern)? + }; let (directory_sender, directory_receiver) = channel(); + let (song_sender, song_receiver) = channel(); + + let songs_db = db.clone(); let directories_db = db.clone(); - std::thread::spawn(move || { + + let directories_thread = std::thread::spawn(move || { insert_directories(directory_receiver, directories_db); }); - let (song_sender, song_receiver) = channel(); - let songs_db = db.clone(); - std::thread::spawn(move || { + let songs_thread = std::thread::spawn(move || { insert_songs(song_receiver, songs_db); }); - let mut updater = IndexUpdater::new(album_art_pattern, directory_sender, song_sender)?; - for target in mount_points.values() { - updater.populate_directory(None, target.as_path())?; + { + let mut updater = IndexUpdater::new(album_art_pattern, directory_sender, song_sender)?; + for target in mount_points.values() { + updater.populate_directory(None, target.as_path())?; + } + } + + match directories_thread.join() { + Err(e) => error!("Error while waiting for directory insertions to complete: {:?}", e), + _ => (), + } + + match songs_thread.join() { + Err(e) => error!("Error while waiting for song insertions to complete: {:?}", e), + _ => (), } Ok(()) From 2b30307488e08c98970906286c9d0acf3c2506cc Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sun, 19 Jan 2020 00:44:24 -0800 Subject: [PATCH 10/11] Added more profiling markers --- src/index/metadata.rs | 14 ++++++++-- src/index/update.rs | 64 +++++++++++++++++++++++++++++-------------- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/index/metadata.rs b/src/index/metadata.rs index b31794b..3642c67 100644 --- a/src/index/metadata.rs +++ b/src/index/metadata.rs @@ -45,10 +45,18 @@ pub fn read(path: &Path) -> Option { #[cfg_attr(feature = "profile-index", flame)] fn read_id3(path: &Path) -> Result { - let tag = id3::Tag::read_from_path(&path)?; - let duration = mp3_duration::from_path(&path) + let tag = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("id3_tag_read"); + id3::Tag::read_from_path(&path)? + }; + let duration = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("mp3_duration"); + mp3_duration::from_path(&path) .map(|d| d.as_secs() as u32) - .ok(); + .ok() + }; let artist = tag.artist().map(|s| s.to_string()); let album_artist = tag.album_artist().map(|s| s.to_string()); diff --git a/src/index/update.rs b/src/index/update.rs index 0baeea5..d9aa7cc 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -102,19 +102,31 @@ impl IndexUpdater { #[cfg_attr(feature = "profile-index", flame)] fn populate_directory(&mut self, parent: Option<&Path>, path: &Path) -> Result<()> { // Find artwork - let artwork = self.get_artwork(path).unwrap_or(None); + let artwork = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("artwork"); + self.get_artwork(path).unwrap_or(None) + }; // Extract path and parent path let parent_string = parent.and_then(|p| p.to_str()).map(|s| s.to_owned()); let path_string = path.to_str().ok_or(anyhow!("Invalid directory path"))?; // Find date added - let metadata = fs::metadata(path_string)?; - let created = metadata + let metadata = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("metadata"); + fs::metadata(path_string)? + }; + let created = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("created_date"); + metadata .created() .or_else(|_| metadata.modified())? .duration_since(time::UNIX_EPOCH)? - .as_secs() as i32; + .as_secs() as i32 + }; let mut directory_album = None; let mut directory_year = None; @@ -145,6 +157,10 @@ impl IndexUpdater { if let Some(file_path_string) = file_path.to_str() { if let Some(tags) = metadata::read(file_path.as_path()) { + + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("process_song"); + if tags.year.is_some() { inconsistent_directory_year |= directory_year.is_some() && directory_year != tags.year; @@ -187,25 +203,31 @@ impl IndexUpdater { } // Insert directory - if inconsistent_directory_year { - directory_year = None; - } - if inconsistent_directory_album { - directory_album = None; - } - if inconsistent_directory_artist { - directory_artist = None; - } + let directory = { + #[cfg(feature = "profile-index")] + let _guard = flame::start_guard("create_directory"); - let directory = NewDirectory { - path: path_string.to_owned(), - parent: parent_string, - artwork, - album: directory_album, - artist: directory_artist, - year: directory_year, - date_added: created, + if inconsistent_directory_year { + directory_year = None; + } + if inconsistent_directory_album { + directory_album = None; + } + if inconsistent_directory_artist { + directory_artist = None; + } + + NewDirectory { + path: path_string.to_owned(), + parent: parent_string, + artwork, + album: directory_album, + artist: directory_artist, + year: directory_year, + date_added: created, + } }; + self.push_directory(directory)?; // Populate subdirectories From 028633d0e67b7287d4ef9de060759f4fe4c8ed2e Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Sun, 19 Jan 2020 01:01:43 -0800 Subject: [PATCH 11/11] Re-factored duplicated code --- src/index/update.rs | 100 ++++++++++++++++++++------------------------ 1 file changed, 45 insertions(+), 55 deletions(-) diff --git a/src/index/update.rs b/src/index/update.rs index d9aa7cc..1d08dd7 100644 --- a/src/index/update.rs +++ b/src/index/update.rs @@ -341,43 +341,29 @@ pub fn populate(db: &DB) -> Result<()> { Ok(()) } -fn insert_songs(receiver: Receiver, db: DB) { - let mut new_entries = Vec::new(); - new_entries.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); - - loop { - match receiver.recv() { - Ok(s) => { - new_entries.push(s); - if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { - if db.connect() - .and_then(|connection|{ - diesel::insert_into(songs::table) - .values(&new_entries) - .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 - .map_err(Error::new) - }) - .is_err() { - error!("Could not insert new songs in database"); - } - new_entries.clear(); - } - }, - Err(_) => break, - } +fn flush_directories(db: &DB, entries: &Vec) { + if db.connect() + .and_then(|connection|{ + diesel::insert_into(directories::table) + .values(entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new directories in database"); } +} - if new_entries.len() > 0 { - if db.connect() - .and_then(|connection|{ - diesel::insert_into(songs::table) - .values(&new_entries) - .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 - .map_err(Error::new) - }) - .is_err() { - error!("Could not insert new songs in database"); - } +fn flush_songs(db: &DB, entries: &Vec) { + if db.connect() + .and_then(|connection|{ + diesel::insert_into(songs::table) + .values(entries) + .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 + .map_err(Error::new) + }) + .is_err() { + error!("Could not insert new songs in database"); } } @@ -390,16 +376,7 @@ fn insert_directories(receiver: Receiver, db: DB) { Ok(s) => { new_entries.push(s); if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { - if db.connect() - .and_then(|connection|{ - diesel::insert_into(directories::table) - .values(&new_entries) - .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 - .map_err(Error::new) - }) - .is_err() { - error!("Could not insert new directories in database"); - } + flush_directories(&db, &new_entries); new_entries.clear(); } }, @@ -408,15 +385,28 @@ fn insert_directories(receiver: Receiver, db: DB) { } if new_entries.len() > 0 { - if db.connect() - .and_then(|connection|{ - diesel::insert_into(directories::table) - .values(&new_entries) - .execute(&*connection) // TODO https://github.com/diesel-rs/diesel/issues/1822 - .map_err(Error::new) - }) - .is_err() { - error!("Could not insert new directories in database"); - } + flush_directories(&db, &new_entries); + } +} + +fn insert_songs(receiver: Receiver, db: DB) { + let mut new_entries = Vec::new(); + new_entries.reserve_exact(INDEX_BUILDING_INSERT_BUFFER_SIZE); + + loop { + match receiver.recv() { + Ok(s) => { + new_entries.push(s); + if new_entries.len() >= INDEX_BUILDING_INSERT_BUFFER_SIZE { + flush_songs(&db, &new_entries); + new_entries.clear(); + } + }, + Err(_) => break, + } + } + + if new_entries.len() > 0 { + flush_songs(&db, &new_entries); } }