diff --git a/src/app.rs b/src/app.rs index e6be2b4..e34bfb9 100644 --- a/src/app.rs +++ b/src/app.rs @@ -4,12 +4,13 @@ use std::path::PathBuf; use crate::db::{self, DB}; use crate::paths::Paths; -pub mod collection; pub mod config; pub mod ddns; pub mod formats; +pub mod index; pub mod lastfm; pub mod playlist; +pub mod scanner; pub mod settings; pub mod thumbnail; pub mod user; @@ -21,7 +22,7 @@ pub mod test; #[derive(thiserror::Error, Debug)] pub enum Error { #[error(transparent)] - Collection(#[from] collection::Error), + Collection(#[from] index::Error), #[error(transparent)] Config(#[from] config::Error), #[error(transparent)] @@ -37,9 +38,8 @@ pub struct App { pub port: u16, pub web_dir_path: PathBuf, pub swagger_dir_path: PathBuf, - pub updater: collection::Updater, - pub browser: collection::Browser, - pub index_manager: collection::IndexManager, + pub scanner: scanner::Scanner, + pub index_manager: index::Manager, pub config_manager: config::Manager, pub ddns_manager: ddns::Manager, pub lastfm_manager: lastfm::Manager, @@ -67,9 +67,8 @@ impl App { let auth_secret = settings_manager.get_auth_secret().await?; let ddns_manager = ddns::Manager::new(db.clone()); let user_manager = user::Manager::new(db.clone(), auth_secret); - let index_manager = collection::IndexManager::new(db.clone()).await; - let browser = collection::Browser::new(db.clone(), vfs_manager.clone()); - let updater = collection::Updater::new( + let index_manager = index::Manager::new(db.clone()).await; + let scanner = scanner::Scanner::new( index_manager.clone(), settings_manager.clone(), vfs_manager.clone(), @@ -83,7 +82,7 @@ impl App { ); let playlist_manager = playlist::Manager::new(db.clone(), vfs_manager.clone()); let thumbnail_manager = thumbnail::Manager::new(thumbnails_dir_path); - let lastfm_manager = lastfm::Manager::new(browser.clone(), user_manager.clone()); + let lastfm_manager = lastfm::Manager::new(index_manager.clone(), user_manager.clone()); if let Some(config_path) = paths.config_file_path { let config = config::Config::from_path(&config_path)?; @@ -94,8 +93,7 @@ impl App { port, web_dir_path: paths.web_dir_path, swagger_dir_path: paths.swagger_dir_path, - updater, - browser, + scanner, index_manager, config_manager, ddns_manager, diff --git a/src/app/collection.rs b/src/app/collection.rs deleted file mode 100644 index 84460a9..0000000 --- a/src/app/collection.rs +++ /dev/null @@ -1,11 +0,0 @@ -mod browser; -mod index; -mod scanner; -mod types; -mod updater; - -pub use browser::*; -pub use index::*; -pub use scanner::*; -pub use types::*; -pub use updater::*; diff --git a/src/app/collection/browser.rs b/src/app/collection/browser.rs deleted file mode 100644 index add8c51..0000000 --- a/src/app/collection/browser.rs +++ /dev/null @@ -1,152 +0,0 @@ -use std::path::Path; - -use crate::app::{collection, vfs}; -use crate::db::DB; - -#[derive(Clone)] -pub struct Browser { - db: DB, - vfs_manager: vfs::Manager, -} - -impl Browser { - pub fn new(db: DB, vfs_manager: vfs::Manager) -> Self { - Self { db, vfs_manager } - } - - pub async fn search(&self, query: &str) -> Result, collection::Error> { - todo!(); - } - - pub async fn get_song(&self, path: &Path) -> Result { - todo!(); - } -} - -#[cfg(test)] -mod test { - use std::path::{Path, PathBuf}; - - use super::*; - use crate::app::test; - use crate::test_name; - - const TEST_MOUNT_NAME: &str = "root"; - - #[tokio::test] - async fn can_browse_top_level() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - - let root_path = Path::new(TEST_MOUNT_NAME); - let files = ctx.browser.browse(Path::new("")).await.unwrap(); - assert_eq!(files.len(), 1); - match files[0] { - collection::File::Directory(ref d) => { - assert_eq!(d, &root_path) - } - _ => panic!("Expected directory"), - } - } - - #[tokio::test] - async fn can_browse_directory() { - let khemmis_path: PathBuf = [TEST_MOUNT_NAME, "Khemmis"].iter().collect(); - let tobokegao_path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect(); - - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - - let files = ctx - .browser - .browse(Path::new(TEST_MOUNT_NAME)) - .await - .unwrap(); - - assert_eq!(files.len(), 2); - match files[0] { - collection::File::Directory(ref d) => { - assert_eq!(d, &khemmis_path) - } - _ => panic!("Expected directory"), - } - - match files[1] { - collection::File::Directory(ref d) => { - assert_eq!(d, &tobokegao_path) - } - _ => panic!("Expected directory"), - } - } - - #[tokio::test] - async fn can_flatten_root() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let songs = ctx - .browser - .flatten(Path::new(TEST_MOUNT_NAME)) - .await - .unwrap(); - assert_eq!(songs.len(), 13); - assert_eq!(songs[0].title, Some("Above The Water".to_owned())); - } - - #[tokio::test] - async fn can_flatten_directory() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect(); - let songs = ctx.browser.flatten(path).await.unwrap(); - assert_eq!(songs.len(), 8); - } - - #[tokio::test] - async fn can_flatten_directory_with_shared_prefix() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); // Prefix of '(Picnic Remixes)' - let songs = ctx.browser.flatten(path).await.unwrap(); - assert_eq!(songs.len(), 7); - } - - #[tokio::test] - async fn can_get_a_song() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - - ctx.updater.update().await.unwrap(); - - let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); - let song_virtual_path = picnic_virtual_dir.join("05 - シャーベット (Sherbet).mp3"); - let artwork_virtual_path = picnic_virtual_dir.join("Folder.png"); - - let song = ctx.browser.get_song(&song_virtual_path).await.unwrap(); - assert_eq!(song.virtual_path, song_virtual_path); - assert_eq!(song.track_number, Some(5)); - assert_eq!(song.disc_number, None); - assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned())); - assert_eq!(song.artists, vec!["Tobokegao".to_owned()]); - assert_eq!(song.album_artists, Vec::::new()); - assert_eq!(song.album, Some("Picnic".to_owned())); - assert_eq!(song.year, Some(2016)); - assert_eq!(song.artwork, Some(artwork_virtual_path)); - } -} diff --git a/src/app/collection/index.rs b/src/app/collection/index.rs deleted file mode 100644 index eae8a0e..0000000 --- a/src/app/collection/index.rs +++ /dev/null @@ -1,556 +0,0 @@ -use std::{ - borrow::BorrowMut, - collections::{HashMap, HashSet}, - hash::{DefaultHasher, Hash, Hasher}, - path::{Path, PathBuf}, - sync::{Arc, RwLock}, -}; - -use log::{error, info}; -use rand::{rngs::ThreadRng, seq::IteratorRandom}; -use serde::{Deserialize, Serialize}; -use tokio::task::spawn_blocking; -use trie_rs::{Trie, TrieBuilder}; - -use crate::{app::collection, db::DB}; - -#[derive(Clone)] -pub struct IndexManager { - db: DB, - index: Arc>, // Not a tokio RwLock as we want to do CPU-bound work with Index -} - -impl IndexManager { - pub async fn new(db: DB) -> Self { - let mut index_manager = Self { - db, - index: Arc::new(RwLock::new(Index::new())), - }; - if let Err(e) = index_manager.try_restore_index().await { - error!("Failed to restore index: {}", e); - } - index_manager - } - - pub(super) async fn replace_index(&mut self, new_index: Index) { - spawn_blocking({ - let index_manager = self.clone(); - move || { - let mut lock = index_manager.index.write().unwrap(); - *lock = new_index; - } - }) - .await - .unwrap() - } - - pub(super) async fn persist_index(&mut self, index: &Index) -> Result<(), collection::Error> { - let serialized = match bitcode::serialize(index) { - Ok(s) => s, - Err(_) => return Err(collection::Error::IndexSerializationError), - }; - sqlx::query!("UPDATE collection_index SET content = $1", serialized) - .execute(self.db.connect().await?.as_mut()) - .await?; - Ok(()) - } - - async fn try_restore_index(&mut self) -> Result { - let serialized = sqlx::query_scalar!("SELECT content FROM collection_index") - .fetch_one(self.db.connect().await?.as_mut()) - .await?; - - let Some(serialized) = serialized else { - info!("Database did not contain a collection to restore"); - return Ok(false); - }; - - let index = match bitcode::deserialize(&serialized[..]) { - Ok(i) => i, - Err(_) => return Err(collection::Error::IndexDeserializationError), - }; - - self.replace_index(index).await; - - Ok(true) - } - - pub async fn browse( - &self, - virtual_path: PathBuf, - ) -> Result, collection::Error> { - spawn_blocking({ - let index_manager = self.clone(); - move || { - let index = index_manager.index.read().unwrap(); - index.browse(virtual_path) - } - }) - .await - .unwrap() - } - - pub async fn flatten(&self, virtual_path: PathBuf) -> Result, collection::Error> { - spawn_blocking({ - let index_manager = self.clone(); - move || { - let index = index_manager.index.read().unwrap(); - index.flatten(virtual_path) - } - }) - .await - .unwrap() - } - - pub async fn get_artist( - &self, - artist_key: &ArtistKey, - ) -> Result { - spawn_blocking({ - let index_manager = self.clone(); - let artist_id = artist_key.into(); - move || { - let index = index_manager.index.read().unwrap(); - index - .get_artist(artist_id) - .ok_or_else(|| collection::Error::ArtistNotFound) - } - }) - .await - .unwrap() - } - - pub async fn get_album( - &self, - album_key: &AlbumKey, - ) -> Result { - spawn_blocking({ - let index_manager = self.clone(); - let album_id = album_key.into(); - move || { - let index = index_manager.index.read().unwrap(); - index - .get_album(album_id) - .ok_or_else(|| collection::Error::AlbumNotFound) - } - }) - .await - .unwrap() - } - - pub async fn get_random_albums( - &self, - count: usize, - ) -> Result, collection::Error> { - spawn_blocking({ - let index_manager = self.clone(); - move || { - let index = index_manager.index.read().unwrap(); - Ok(index - .albums - .keys() - .choose_multiple(&mut ThreadRng::default(), count) - .into_iter() - .filter_map(|k| index.get_album(*k)) - .collect()) - } - }) - .await - .unwrap() - } - - pub async fn get_recent_albums( - &self, - count: usize, - ) -> Result, collection::Error> { - spawn_blocking({ - let index_manager = self.clone(); - move || { - let index = index_manager.index.read().unwrap(); - Ok(index - .recent_albums - .iter() - .take(count) - .filter_map(|k| index.get_album(*k)) - .collect()) - } - }) - .await - .unwrap() - } -} - -#[derive(Default)] -pub(super) struct IndexBuilder { - directories: HashMap>, - flattened: TrieBuilder, - songs: HashMap, - artists: HashMap, - albums: HashMap, -} - -impl IndexBuilder { - pub fn add_directory(&mut self, directory: collection::Directory) { - self.directories - .entry(directory.virtual_path.clone()) - .or_default(); - if let Some(parent) = directory.virtual_parent { - self.directories - .entry(parent.clone()) - .or_default() - .insert(collection::File::Directory(directory.virtual_path)); - } - } - - pub fn add_song(&mut self, song: collection::Song) { - let song_id: SongID = song.song_id(); - self.flattened.push( - song.virtual_path - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(), - ); - self.directories - .entry(song.virtual_parent.clone()) - .or_default() - .insert(collection::File::Song(song.virtual_path.clone())); - self.add_song_to_album(&song); - self.add_album_to_artists(&song); - self.songs.insert(song_id, song); - } - - fn add_album_to_artists(&mut self, song: &collection::Song) { - let album_id: AlbumID = song.album_id(); - - for artist_name in &song.album_artists { - let artist = self.get_or_create_artist(artist_name); - artist.albums.insert(album_id); - } - - for artist_name in &song.artists { - let artist = self.get_or_create_artist(artist_name); - if song.album_artists.is_empty() { - artist.albums.insert(album_id); - } else if !song.album_artists.contains(artist_name) { - artist.album_appearances.insert(album_id); - } - } - } - - fn get_or_create_artist(&mut self, name: &String) -> &mut Artist { - let artist_key = ArtistKey { - name: Some(name.clone()), - }; - let artist_id: ArtistID = (&artist_key).into(); - self.artists - .entry(artist_id) - .or_insert_with(|| Artist { - name: Some(name.clone()), - albums: HashSet::new(), - album_appearances: HashSet::new(), - }) - .borrow_mut() - } - - fn add_song_to_album(&mut self, song: &collection::Song) { - let song_id: SongID = song.song_id(); - let album_id: AlbumID = song.album_id(); - - let album = self.albums.entry(album_id).or_default().borrow_mut(); - - if album.name.is_none() { - album.name = song.album.clone(); - } - - if album.artwork.is_none() { - album.artwork = song.artwork.clone(); - } - - if album.year.is_none() { - album.year = song.year.clone(); - } - - album.date_added = album.date_added.min(song.date_added); - - if !song.album_artists.is_empty() { - album.artists = song.album_artists.clone(); - } else if !song.artists.is_empty() { - album.artists = song.artists.clone(); - } - - album.songs.insert(song_id); - } - - pub fn build(self) -> Index { - let mut recent_albums = self.albums.keys().cloned().collect::>(); - recent_albums.sort_by_key(|a| { - self.albums - .get(a) - .map(|a| -a.date_added) - .unwrap_or_default() - }); - - Index { - directories: self.directories, - flattened: self.flattened.build(), - songs: self.songs, - artists: self.artists, - albums: self.albums, - recent_albums, - } - } -} - -#[derive(Serialize, Deserialize)] -pub(super) struct Index { - directories: HashMap>, - flattened: Trie, - songs: HashMap, - artists: HashMap, - albums: HashMap, - recent_albums: Vec, -} - -impl Index { - pub fn new() -> Self { - Self { - directories: HashMap::new(), - flattened: TrieBuilder::new().build(), - songs: HashMap::new(), - artists: HashMap::new(), - albums: HashMap::new(), - recent_albums: Vec::new(), - } - } - - pub(self) fn browse>( - &self, - virtual_path: P, - ) -> Result, collection::Error> { - let Some(files) = self.directories.get(virtual_path.as_ref()) else { - return Err(collection::Error::DirectoryNotFound( - virtual_path.as_ref().to_owned(), - )); - }; - Ok(files.iter().cloned().collect()) - } - - pub(self) fn flatten>( - &self, - virtual_path: P, - ) -> Result, collection::Error> { - let path_components = virtual_path - .as_ref() - .components() - .map(|c| c.as_os_str().to_string_lossy().to_string()) - .collect::>(); - - if !self.flattened.is_prefix(&path_components) { - return Err(collection::Error::DirectoryNotFound( - virtual_path.as_ref().to_owned(), - )); - } - - Ok(self - .flattened - .predictive_search(path_components) - .map(|c: Vec| -> PathBuf { c.join(std::path::MAIN_SEPARATOR_STR).into() }) - .map(|s| SongKey { virtual_path: s }) - .collect::>()) - } - - pub(self) fn get_artist(&self, artist_id: ArtistID) -> Option { - self.artists.get(&artist_id).map(|a| { - let albums = { - let mut albums = a - .albums - .iter() - .filter_map(|album_id| self.get_album(*album_id)) - .collect::>(); - albums.sort_by(|a, b| (a.year, &a.name).partial_cmp(&(b.year, &b.name)).unwrap()); - albums - }; - - let album_appearances = { - let mut album_appearances = a - .album_appearances - .iter() - .filter_map(|album_id| self.get_album(*album_id)) - .collect::>(); - album_appearances.sort_by(|a, b| { - (&a.artists, a.year, &a.name) - .partial_cmp(&(&b.artists, b.year, &b.name)) - .unwrap() - }); - album_appearances - }; - - collection::Artist { - name: a.name.clone(), - albums, - album_appearances, - } - }) - } - - pub(self) fn get_album(&self, album_id: AlbumID) -> Option { - self.albums.get(&album_id).map(|a| { - let mut songs = a - .songs - .iter() - .filter_map(|s| self.songs.get(s)) - .cloned() - .collect::>(); - - songs.sort_by_key(|s| (s.disc_number.unwrap_or(-1), s.track_number.unwrap_or(-1))); - - collection::Album { - name: a.name.clone(), - artwork: a.artwork.clone(), - artists: a.artists.clone(), - year: a.year, - date_added: a.date_added, - songs, - } - }) - } -} - -#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] -struct SongID(u64); - -#[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] -pub struct SongKey { - pub virtual_path: PathBuf, -} - -impl From<&collection::Song> for SongKey { - fn from(song: &collection::Song) -> Self { - SongKey { - virtual_path: song.virtual_path.clone(), - } - } -} - -impl From<&SongKey> for SongID { - fn from(key: &SongKey) -> Self { - SongID(key.id()) - } -} - -impl collection::Song { - pub(self) fn song_id(&self) -> SongID { - let key: SongKey = self.into(); - (&key).into() - } -} - -#[derive(Serialize, Deserialize)] -struct Artist { - pub name: Option, - pub albums: HashSet, - pub album_appearances: HashSet, -} - -#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] -struct ArtistID(u64); - -#[derive(Clone, Eq, Hash, PartialEq)] -pub struct ArtistKey { - pub name: Option, -} - -impl From<&ArtistKey> for ArtistID { - fn from(key: &ArtistKey) -> Self { - ArtistID(key.id()) - } -} - -#[derive(Clone, Default, Serialize, Deserialize)] -struct Album { - pub name: Option, - pub artwork: Option, - pub artists: Vec, - pub year: Option, - pub date_added: i64, - pub songs: HashSet, -} - -#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] -struct AlbumID(u64); - -#[derive(Clone, Eq, Hash, PartialEq)] -pub struct AlbumKey { - pub artists: Vec, - pub name: Option, -} - -impl From<&collection::Song> for AlbumKey { - fn from(song: &collection::Song) -> Self { - let album_artists = match song.album_artists.is_empty() { - true => &song.artists, - false => &song.album_artists, - }; - - AlbumKey { - artists: album_artists.iter().cloned().collect(), - name: song.album.clone(), - } - } -} - -impl From<&AlbumKey> for AlbumID { - fn from(key: &AlbumKey) -> Self { - AlbumID(key.id()) - } -} - -impl collection::Song { - pub(self) fn album_id(&self) -> AlbumID { - let key: AlbumKey = self.into(); - (&key).into() - } -} - -trait ID { - fn id(&self) -> u64; -} - -impl ID for T { - fn id(&self) -> u64 { - let mut hasher = DefaultHasher::default(); - self.hash(&mut hasher); - hasher.finish() - } -} - -#[cfg(test)] -mod test { - - use crate::app::test; - use crate::test_name; - - const TEST_MOUNT_NAME: &str = "root"; - - #[tokio::test] - async fn can_get_random_albums() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let albums = ctx.index_manager.get_random_albums(1).await.unwrap(); - assert_eq!(albums.len(), 1); - } - - #[tokio::test] - async fn can_get_recent_albums() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let albums = ctx.index_manager.get_recent_albums(2).await.unwrap(); - assert_eq!(albums.len(), 2); - } -} diff --git a/src/app/collection/scanner.rs b/src/app/collection/scanner.rs deleted file mode 100644 index 7fe7ad4..0000000 --- a/src/app/collection/scanner.rs +++ /dev/null @@ -1,180 +0,0 @@ -use log::{error, info}; -use rayon::{Scope, ThreadPoolBuilder}; -use regex::Regex; -use std::cmp::min; -use std::fs; -use std::path::Path; -use std::str::FromStr; -use tokio::sync::mpsc::UnboundedSender; - -use crate::app::vfs; -use crate::app::{ - collection::{self}, - formats, -}; - -pub struct Scanner { - directories_output: UnboundedSender, - songs_output: UnboundedSender, - vfs_manager: vfs::Manager, - artwork_regex: Option, -} - -impl Scanner { - pub fn new( - directories_output: UnboundedSender, - songs_output: UnboundedSender, - vfs_manager: vfs::Manager, - artwork_regex: Option, - ) -> Self { - Self { - directories_output, - songs_output, - vfs_manager, - artwork_regex, - } - } - - pub async fn scan(self) -> Result<(), collection::Error> { - let vfs = self.vfs_manager.get_vfs().await?; - let roots = vfs.mounts().clone(); - - let key = "POLARIS_NUM_TRAVERSER_THREADS"; - let num_threads = std::env::var_os(key) - .map(|v| v.to_string_lossy().to_string()) - .and_then(|v| usize::from_str(&v).ok()) - .unwrap_or_else(|| min(num_cpus::get(), 8)); - info!("Browsing collection using {} threads", num_threads); - - let directories_output = self.directories_output.clone(); - let songs_output = self.songs_output.clone(); - let artwork_regex = self.artwork_regex.clone(); - - let thread_pool = ThreadPoolBuilder::new().num_threads(num_threads).build()?; - thread_pool.scope({ - |scope| { - for root in roots { - scope.spawn(|scope| { - process_directory( - scope, - root.source, - root.name, - directories_output.clone(), - songs_output.clone(), - artwork_regex.clone(), - ); - }); - } - } - }); - - Ok(()) - } -} - -fn process_directory, Q: AsRef>( - scope: &Scope, - real_path: P, - virtual_path: Q, - directories_output: UnboundedSender, - songs_output: UnboundedSender, - artwork_regex: Option, -) { - let read_dir = match fs::read_dir(&real_path) { - Ok(read_dir) => read_dir, - Err(e) => { - error!( - "Directory read error for `{}`: {}", - real_path.as_ref().display(), - e - ); - return; - } - }; - - let mut songs = vec![]; - let mut artwork_file = None; - - for entry in read_dir { - let name = match entry { - Ok(ref f) => f.file_name(), - Err(e) => { - error!( - "File read error within `{}`: {}", - real_path.as_ref().display(), - e - ); - break; - } - }; - - let entry_real_path = real_path.as_ref().join(&name); - let entry_virtual_path = virtual_path.as_ref().join(&name); - - if entry_real_path.is_dir() { - scope.spawn({ - let directories_output = directories_output.clone(); - let songs_output = songs_output.clone(); - let artwork_regex = artwork_regex.clone(); - |scope| { - process_directory( - scope, - entry_real_path, - entry_virtual_path, - directories_output, - songs_output, - artwork_regex, - ); - } - }); - } else if let Some(metadata) = formats::read_metadata(&entry_real_path) { - songs.push(collection::Song { - path: entry_real_path.clone(), - virtual_path: entry_virtual_path.clone(), - virtual_parent: entry_virtual_path.parent().unwrap().to_owned(), - track_number: metadata.track_number.map(|n| n as i64), - disc_number: metadata.disc_number.map(|n| n as i64), - title: metadata.title, - artists: metadata.artists, - album_artists: metadata.album_artists, - year: metadata.year.map(|n| n as i64), - album: metadata.album, - artwork: metadata.has_artwork.then(|| entry_virtual_path.clone()), - duration: metadata.duration.map(|n| n as i64), - lyricists: metadata.lyricists, - composers: metadata.composers, - genres: metadata.genres, - labels: metadata.labels, - date_added: get_date_created(&entry_real_path).unwrap_or_default(), - }); - } else if artwork_file.is_none() - && artwork_regex - .as_ref() - .is_some_and(|r| r.is_match(name.to_str().unwrap_or_default())) - { - artwork_file = Some(entry_virtual_path); - } - } - - for mut song in songs { - song.artwork = song.artwork.or_else(|| artwork_file.clone()); - songs_output.send(song).ok(); - } - - directories_output - .send(collection::Directory { - virtual_path: virtual_path.as_ref().to_owned(), - virtual_parent: virtual_path.as_ref().parent().map(Path::to_owned), - }) - .ok(); -} - -fn get_date_created>(path: P) -> Option { - if let Ok(t) = fs::metadata(path).and_then(|m| m.created().or_else(|_| m.modified())) { - t.duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .ok() - } else { - None - } -} diff --git a/src/app/collection/types.rs b/src/app/collection/types.rs deleted file mode 100644 index b0180c6..0000000 --- a/src/app/collection/types.rs +++ /dev/null @@ -1,82 +0,0 @@ -use std::path::PathBuf; - -use serde::{Deserialize, Serialize}; - -use crate::{ - app::vfs::{self}, - db, -}; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("Directory not found: {0}")] - DirectoryNotFound(PathBuf), - #[error("Artist not found")] - ArtistNotFound, - #[error("Album not found")] - AlbumNotFound, - #[error(transparent)] - Database(#[from] sqlx::Error), - #[error(transparent)] - DatabaseConnection(#[from] db::Error), - #[error(transparent)] - Vfs(#[from] vfs::Error), - #[error("Could not deserialize collection")] - IndexDeserializationError, - #[error("Could not serialize collection")] - IndexSerializationError, - #[error(transparent)] - ThreadPoolBuilder(#[from] rayon::ThreadPoolBuildError), - #[error(transparent)] - ThreadJoining(#[from] tokio::task::JoinError), -} - -#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] -pub enum File { - Directory(PathBuf), - Song(PathBuf), -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Song { - pub path: PathBuf, - pub virtual_path: PathBuf, - pub virtual_parent: PathBuf, - pub track_number: Option, - pub disc_number: Option, - pub title: Option, - pub artists: Vec, - pub album_artists: Vec, - pub year: Option, - pub album: Option, - pub artwork: Option, - pub duration: Option, - pub lyricists: Vec, - pub composers: Vec, - pub genres: Vec, - pub labels: Vec, - pub date_added: i64, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct Directory { - pub virtual_path: PathBuf, - pub virtual_parent: Option, -} - -#[derive(Debug, Default, PartialEq, Eq)] -pub struct Artist { - pub name: Option, - pub albums: Vec, - pub album_appearances: Vec, -} - -#[derive(Debug, Default, PartialEq, Eq)] -pub struct Album { - pub name: Option, - pub artwork: Option, - pub artists: Vec, - pub year: Option, - pub date_added: i64, - pub songs: Vec, -} diff --git a/src/app/collection/updater.rs b/src/app/collection/updater.rs deleted file mode 100644 index 4900d73..0000000 --- a/src/app/collection/updater.rs +++ /dev/null @@ -1,214 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use log::{error, info}; -use tokio::{ - sync::{mpsc::unbounded_channel, Notify}, - time::Instant, -}; - -use crate::app::{collection::*, settings, vfs}; - -#[derive(Clone)] -pub struct Updater { - index_manager: IndexManager, - settings_manager: settings::Manager, - vfs_manager: vfs::Manager, - pending_scan: Arc, -} - -impl Updater { - pub async fn new( - index_manager: IndexManager, - settings_manager: settings::Manager, - vfs_manager: vfs::Manager, - ) -> Result { - let updater = Self { - index_manager, - vfs_manager, - settings_manager, - pending_scan: Arc::new(Notify::new()), - }; - - tokio::spawn({ - let mut updater = updater.clone(); - async move { - loop { - updater.pending_scan.notified().await; - if let Err(e) = updater.update().await { - error!("Error while updating index: {}", e); - } - } - } - }); - - Ok(updater) - } - - pub fn trigger_scan(&self) { - self.pending_scan.notify_one(); - } - - pub fn begin_periodic_scans(&self) { - tokio::spawn({ - let index = self.clone(); - async move { - loop { - index.trigger_scan(); - let sleep_duration = index - .settings_manager - .get_index_sleep_duration() - .await - .unwrap_or_else(|e| { - error!("Could not retrieve index sleep duration: {}", e); - Duration::from_secs(1800) - }); - tokio::time::sleep(sleep_duration).await; - } - } - }); - } - - pub async fn update(&mut self) -> Result<(), Error> { - let start = Instant::now(); - info!("Beginning collection scan"); - - let album_art_pattern = self - .settings_manager - .get_index_album_art_pattern() - .await - .ok(); - - let (scanner_directories_output, mut collection_directories_input) = unbounded_channel(); - let (scanner_songs_output, mut collection_songs_input) = unbounded_channel(); - - let scanner = Scanner::new( - scanner_directories_output, - scanner_songs_output, - self.vfs_manager.clone(), - album_art_pattern, - ); - - let index_task = tokio::spawn(async move { - let capacity = 500; - let mut index_builder = IndexBuilder::default(); - let mut song_buffer: Vec = Vec::with_capacity(capacity); - let mut directory_buffer: Vec = Vec::with_capacity(capacity); - - loop { - let exhausted_songs = match collection_songs_input - .recv_many(&mut song_buffer, capacity) - .await - { - 0 => true, - _ => { - for song in song_buffer.drain(0..) { - index_builder.add_song(song); - } - false - } - }; - - let exhausted_directories = match collection_directories_input - .recv_many(&mut directory_buffer, capacity) - .await - { - 0 => true, - _ => { - for directory in directory_buffer.drain(0..) { - index_builder.add_directory(directory); - } - false - } - }; - - if exhausted_directories && exhausted_songs { - break; - } - } - - index_builder.build() - }); - - let index = tokio::join!(scanner.scan(), index_task).1?; - self.index_manager.persist_index(&index).await?; - self.index_manager.replace_index(index).await; - - info!( - "Collection scan took {} seconds", - start.elapsed().as_millis() as f32 / 1000.0 - ); - - Ok(()) - } -} - -#[cfg(test)] -mod test { - use std::path::PathBuf; - - use crate::{ - app::{settings, test}, - test_name, - }; - - const TEST_MOUNT_NAME: &str = "root"; - - #[tokio::test] - async fn scan_adds_new_content() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - - ctx.updater.update().await.unwrap(); - ctx.updater.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts - - todo!(); - - // assert_eq!(all_directories.len(), 6); - // assert_eq!(all_songs.len(), 13); - } - - #[tokio::test] - async fn finds_embedded_artwork() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - - ctx.updater.update().await.unwrap(); - - let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); - let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3"); - - let song = ctx.browser.get_song(&song_virtual_path).await.unwrap(); - assert_eq!(song.artwork, Some(song_virtual_path)); - } - - #[tokio::test] - async fn album_art_pattern_is_case_insensitive() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - - let patterns = vec!["folder", "FOLDER"]; - - for pattern in patterns.into_iter() { - ctx.settings_manager - .amend(&settings::NewSettings { - album_art_pattern: Some(pattern.to_owned()), - ..Default::default() - }) - .await - .unwrap(); - ctx.updater.update().await.unwrap(); - - let hunted_virtual_dir: PathBuf = - [TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect(); - let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg"); - let song = &ctx.browser.flatten(&hunted_virtual_dir).await.unwrap()[0]; - assert_eq!(song.artwork, Some(artwork_virtual_path)); - } - } -} diff --git a/src/app/index.rs b/src/app/index.rs new file mode 100644 index 0000000..bf2bb0d --- /dev/null +++ b/src/app/index.rs @@ -0,0 +1,246 @@ +use std::{ + path::PathBuf, + sync::{Arc, RwLock}, +}; + +use log::{error, info}; +use serde::{Deserialize, Serialize}; +use tokio::task::spawn_blocking; + +use crate::app::scanner; +use crate::app::vfs; +use crate::db::{self, DB}; + +mod browser; +mod collection; +mod search; + +pub use browser::File; +pub use collection::{Album, AlbumKey, Artist, ArtistKey, Song, SongKey}; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("Directory not found: {0}")] + DirectoryNotFound(PathBuf), + #[error("Artist not found")] + ArtistNotFound, + #[error("Album not found")] + AlbumNotFound, + #[error("Song not found")] + SongNotFound, + #[error(transparent)] + Database(#[from] sqlx::Error), + #[error(transparent)] + DatabaseConnection(#[from] db::Error), + #[error(transparent)] + Vfs(#[from] vfs::Error), + #[error("Could not deserialize collection")] + IndexDeserializationError, + #[error("Could not serialize collection")] + IndexSerializationError, + #[error(transparent)] + ThreadPoolBuilder(#[from] rayon::ThreadPoolBuildError), + #[error(transparent)] + ThreadJoining(#[from] tokio::task::JoinError), +} + +#[derive(Clone)] +pub struct Manager { + db: DB, + index: Arc>, // Not a tokio RwLock as we want to do CPU-bound work with Index +} + +impl Manager { + pub async fn new(db: DB) -> Self { + let mut index_manager = Self { + db, + index: Arc::default(), + }; + if let Err(e) = index_manager.try_restore_index().await { + error!("Failed to restore index: {}", e); + } + index_manager + } + + pub async fn replace_index(&mut self, new_index: Index) { + spawn_blocking({ + let index_manager = self.clone(); + move || { + let mut lock = index_manager.index.write().unwrap(); + *lock = new_index; + } + }) + .await + .unwrap() + } + + pub async fn persist_index(&mut self, index: &Index) -> Result<(), Error> { + let serialized = match bitcode::serialize(index) { + Ok(s) => s, + Err(_) => return Err(Error::IndexSerializationError), + }; + sqlx::query!("UPDATE collection_index SET content = $1", serialized) + .execute(self.db.connect().await?.as_mut()) + .await?; + Ok(()) + } + + async fn try_restore_index(&mut self) -> Result { + let serialized = sqlx::query_scalar!("SELECT content FROM collection_index") + .fetch_one(self.db.connect().await?.as_mut()) + .await?; + + let Some(serialized) = serialized else { + info!("Database did not contain a collection to restore"); + return Ok(false); + }; + + let index = match bitcode::deserialize(&serialized[..]) { + Ok(i) => i, + Err(_) => return Err(Error::IndexDeserializationError), + }; + + self.replace_index(index).await; + + Ok(true) + } + + pub async fn browse(&self, virtual_path: PathBuf) -> Result, Error> { + spawn_blocking({ + let index_manager = self.clone(); + move || { + let index = index_manager.index.read().unwrap(); + index.browser.browse(virtual_path) + } + }) + .await + .unwrap() + } + + pub async fn flatten(&self, virtual_path: PathBuf) -> Result, Error> { + spawn_blocking({ + let index_manager = self.clone(); + move || { + let index = index_manager.index.read().unwrap(); + index.browser.flatten(virtual_path) + } + }) + .await + .unwrap() + } + + pub async fn get_artist(&self, artist_key: &ArtistKey) -> Result { + spawn_blocking({ + let index_manager = self.clone(); + let artist_id = artist_key.into(); + move || { + let index = index_manager.index.read().unwrap(); + index + .collection + .get_artist(artist_id) + .ok_or_else(|| Error::ArtistNotFound) + } + }) + .await + .unwrap() + } + + pub async fn get_album(&self, album_key: &AlbumKey) -> Result { + spawn_blocking({ + let index_manager = self.clone(); + let album_id = album_key.into(); + move || { + let index = index_manager.index.read().unwrap(); + index + .collection + .get_album(album_id) + .ok_or_else(|| Error::AlbumNotFound) + } + }) + .await + .unwrap() + } + + pub async fn get_random_albums(&self, count: usize) -> Result, Error> { + spawn_blocking({ + let index_manager = self.clone(); + move || { + let index = index_manager.index.read().unwrap(); + Ok(index.collection.get_random_albums(count)) + } + }) + .await + .unwrap() + } + + pub async fn get_recent_albums(&self, count: usize) -> Result, Error> { + spawn_blocking({ + let index_manager = self.clone(); + move || { + let index = index_manager.index.read().unwrap(); + Ok(index.collection.get_recent_albums(count)) + } + }) + .await + .unwrap() + } + + pub async fn get_song(&self, song_key: &SongKey) -> Result { + spawn_blocking({ + let index_manager = self.clone(); + let song_id = song_key.into(); + move || { + let index = index_manager.index.read().unwrap(); + index + .collection + .get_song(song_id) + .ok_or_else(|| Error::SongNotFound) + } + }) + .await + .unwrap() + } + + pub async fn search(&self, _query: &str) -> Result, Error> { + todo!(); + } +} + +#[derive(Serialize, Deserialize)] +pub struct Index { + pub browser: browser::Browser, + pub collection: collection::Collection, +} + +impl Default for Index { + fn default() -> Self { + Self { + browser: browser::Browser::new(), + collection: Default::default(), + } + } +} + +#[derive(Default)] +pub struct Builder { + browser_builder: browser::Builder, + collection_builder: collection::Builder, +} + +impl Builder { + pub fn add_directory(&mut self, directory: scanner::Directory) { + self.browser_builder.add_directory(directory); + } + + pub fn add_song(&mut self, song: scanner::Song) { + self.browser_builder.add_song(&song); + self.collection_builder.add_song(song); + } + + pub fn build(self) -> Index { + Index { + browser: self.browser_builder.build(), + collection: self.collection_builder.build(), + } + } +} diff --git a/src/app/index/browser.rs b/src/app/index/browser.rs new file mode 100644 index 0000000..cbc332d --- /dev/null +++ b/src/app/index/browser.rs @@ -0,0 +1,209 @@ +use std::{ + collections::{HashMap, HashSet}, + hash::Hash, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +use trie_rs::{Trie, TrieBuilder}; + +use crate::app::{index::Error, scanner}; + +#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum File { + Directory(PathBuf), + Song(PathBuf), +} + +#[derive(Serialize, Deserialize)] +pub struct Browser { + directories: HashMap>, + flattened: Trie, +} + +impl Browser { + pub fn new() -> Self { + Self { + directories: HashMap::new(), + flattened: TrieBuilder::new().build(), + } + } + + pub fn browse>(&self, virtual_path: P) -> Result, Error> { + let Some(files) = self.directories.get(virtual_path.as_ref()) else { + return Err(Error::DirectoryNotFound(virtual_path.as_ref().to_owned())); + }; + Ok(files.iter().cloned().collect()) + } + + pub fn flatten>(&self, virtual_path: P) -> Result, Error> { + let path_components = virtual_path + .as_ref() + .components() + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect::>(); + + if !self.flattened.is_prefix(&path_components) { + return Err(Error::DirectoryNotFound(virtual_path.as_ref().to_owned())); + } + + Ok(self + .flattened + .predictive_search(path_components) + .map(|c: Vec| -> PathBuf { c.join(std::path::MAIN_SEPARATOR_STR).into() }) + .collect::>()) + } +} + +pub struct Builder { + directories: HashMap>, + flattened: TrieBuilder, +} + +impl Default for Builder { + fn default() -> Self { + Self { + directories: Default::default(), + flattened: Default::default(), + } + } +} + +impl Builder { + pub fn add_directory(&mut self, directory: scanner::Directory) { + self.directories + .entry(directory.virtual_path.clone()) + .or_default(); + + if let Some(parent) = directory.virtual_parent { + self.directories + .entry(parent.clone()) + .or_default() + .insert(File::Directory(directory.virtual_path)); + } + } + + pub fn add_song(&mut self, song: &scanner::Song) { + self.directories + .entry(song.virtual_parent.clone()) + .or_default() + .insert(File::Song(song.virtual_path.clone())); + + self.flattened.push( + song.virtual_path + .components() + .map(|c| c.as_os_str().to_string_lossy().to_string()) + .collect::>(), + ); + } + + pub fn build(self) -> Browser { + Browser { + directories: self.directories, + flattened: self.flattened.build(), + } + } +} + +#[cfg(test)] +mod test { + use std::path::{Path, PathBuf}; + + use super::*; + use crate::app::test; + use crate::test_name; + + const TEST_MOUNT_NAME: &str = "root"; + + #[tokio::test] + async fn can_browse_top_level() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + + let root_path = Path::new(TEST_MOUNT_NAME); + let files = ctx.index_manager.browse(PathBuf::new()).await.unwrap(); + assert_eq!(files.len(), 1); + match files[0] { + File::Directory(ref d) => { + assert_eq!(d, &root_path) + } + _ => panic!("Expected directory"), + } + } + + #[tokio::test] + async fn can_browse_directory() { + let khemmis_path: PathBuf = [TEST_MOUNT_NAME, "Khemmis"].iter().collect(); + let tobokegao_path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect(); + + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + + let files = ctx + .index_manager + .browse(PathBuf::from(TEST_MOUNT_NAME)) + .await + .unwrap(); + + assert_eq!(files.len(), 2); + match files[0] { + File::Directory(ref d) => { + assert_eq!(d, &khemmis_path) + } + _ => panic!("Expected directory"), + } + + match files[1] { + File::Directory(ref d) => { + assert_eq!(d, &tobokegao_path) + } + _ => panic!("Expected directory"), + } + } + + #[tokio::test] + async fn can_flatten_root() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + let songs = ctx + .index_manager + .flatten(PathBuf::from(TEST_MOUNT_NAME)) + .await + .unwrap(); + assert_eq!(songs.len(), 13); + assert_eq!(songs[0], Path::new("FIX ME")); + } + + #[tokio::test] + async fn can_flatten_directory() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect(); + let songs = ctx.index_manager.flatten(path).await.unwrap(); + assert_eq!(songs.len(), 8); + } + + #[tokio::test] + async fn can_flatten_directory_with_shared_prefix() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); // Prefix of '(Picnic Remixes)' + let songs = ctx.index_manager.flatten(path).await.unwrap(); + assert_eq!(songs.len(), 7); + } +} diff --git a/src/app/index/collection.rs b/src/app/index/collection.rs new file mode 100644 index 0000000..4ee1a32 --- /dev/null +++ b/src/app/index/collection.rs @@ -0,0 +1,422 @@ +use std::{ + borrow::BorrowMut, + collections::{HashMap, HashSet}, + hash::{DefaultHasher, Hash, Hasher}, + path::PathBuf, +}; + +use rand::{rngs::ThreadRng, seq::IteratorRandom}; +use serde::{Deserialize, Serialize}; + +use crate::app::scanner; + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Artist { + pub name: Option, + pub albums: Vec, + pub album_appearances: Vec, +} + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Album { + pub name: Option, + pub artwork: Option, + pub artists: Vec, + pub year: Option, + pub date_added: i64, + pub songs: Vec, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Song { + pub path: PathBuf, + pub virtual_path: PathBuf, + pub virtual_parent: PathBuf, + pub track_number: Option, + pub disc_number: Option, + pub title: Option, + pub artists: Vec, + pub album_artists: Vec, + pub year: Option, + pub album: Option, + pub artwork: Option, + pub duration: Option, + pub lyricists: Vec, + pub composers: Vec, + pub genres: Vec, + pub labels: Vec, + pub date_added: i64, +} + +#[derive(Default, Serialize, Deserialize)] +pub struct Collection { + artists: HashMap, + albums: HashMap, + songs: HashMap, + recent_albums: Vec, +} + +impl Collection { + pub fn get_artist(&self, artist_id: ArtistID) -> Option { + self.artists.get(&artist_id).map(|a| { + let albums = { + let mut albums = a + .albums + .iter() + .filter_map(|album_id| self.get_album(*album_id)) + .collect::>(); + albums.sort_by(|a, b| (a.year, &a.name).partial_cmp(&(b.year, &b.name)).unwrap()); + albums + }; + + let album_appearances = { + let mut album_appearances = a + .album_appearances + .iter() + .filter_map(|album_id| self.get_album(*album_id)) + .collect::>(); + album_appearances.sort_by(|a, b| { + (&a.artists, a.year, &a.name) + .partial_cmp(&(&b.artists, b.year, &b.name)) + .unwrap() + }); + album_appearances + }; + + Artist { + name: a.name.clone(), + albums, + album_appearances, + } + }) + } + + pub fn get_album(&self, album_id: AlbumID) -> Option { + self.albums.get(&album_id).map(|a| { + let mut songs = a + .songs + .iter() + .filter_map(|s| self.songs.get(s)) + .cloned() + .collect::>(); + + songs.sort_by_key(|s| (s.disc_number.unwrap_or(-1), s.track_number.unwrap_or(-1))); + + Album { + name: a.name.clone(), + artwork: a.artwork.clone(), + artists: a.artists.clone(), + year: a.year, + date_added: a.date_added, + songs, + } + }) + } + + pub fn get_random_albums(&self, count: usize) -> Vec { + self.albums + .keys() + .choose_multiple(&mut ThreadRng::default(), count) + .into_iter() + .filter_map(|k| self.get_album(*k)) + .collect() + } + + pub fn get_recent_albums(&self, count: usize) -> Vec { + self.recent_albums + .iter() + .take(count) + .filter_map(|k| self.get_album(*k)) + .collect() + } + + pub fn get_song(&self, song_id: SongID) -> Option { + self.songs.get(&song_id).cloned() + } +} + +#[derive(Default)] +pub struct Builder { + artists: HashMap, + albums: HashMap, + songs: HashMap, +} + +impl Builder { + pub fn add_song(&mut self, song: scanner::Song) { + let song = Song { + path: song.path, + virtual_path: song.virtual_path, + virtual_parent: song.virtual_parent, + track_number: song.track_number, + disc_number: song.disc_number, + title: song.title, + artists: song.artists, + album_artists: song.album_artists, + year: song.year, + album: song.album, + artwork: song.artwork, + duration: song.duration, + lyricists: song.lyricists, + composers: song.composers, + genres: song.genres, + labels: song.labels, + date_added: song.date_added, + }; + + let song_id: SongID = song.song_id(); + self.add_song_to_album(&song); + self.add_album_to_artists(&song); + self.songs.insert(song_id, song); + } + + pub fn build(self) -> Collection { + let mut recent_albums = self.albums.keys().cloned().collect::>(); + recent_albums.sort_by_key(|a| { + self.albums + .get(a) + .map(|a| -a.date_added) + .unwrap_or_default() + }); + + Collection { + artists: self.artists, + albums: self.albums, + songs: self.songs, + recent_albums, + } + } + + fn add_album_to_artists(&mut self, song: &Song) { + let album_id: AlbumID = song.album_id(); + + for artist_name in &song.album_artists { + let artist = self.get_or_create_artist(artist_name); + artist.albums.insert(album_id); + } + + for artist_name in &song.artists { + let artist = self.get_or_create_artist(artist_name); + if song.album_artists.is_empty() { + artist.albums.insert(album_id); + } else if !song.album_artists.contains(artist_name) { + artist.album_appearances.insert(album_id); + } + } + } + + fn get_or_create_artist(&mut self, name: &String) -> &mut storage::Artist { + let artist_key = ArtistKey { + name: Some(name.clone()), + }; + let artist_id: ArtistID = (&artist_key).into(); + self.artists + .entry(artist_id) + .or_insert_with(|| storage::Artist { + name: Some(name.clone()), + albums: HashSet::new(), + album_appearances: HashSet::new(), + }) + .borrow_mut() + } + + fn add_song_to_album(&mut self, song: &Song) { + let song_id: SongID = song.song_id(); + let album_id: AlbumID = song.album_id(); + + let album = self.albums.entry(album_id).or_default().borrow_mut(); + + if album.name.is_none() { + album.name = song.album.clone(); + } + + if album.artwork.is_none() { + album.artwork = song.artwork.clone(); + } + + if album.year.is_none() { + album.year = song.year.clone(); + } + + album.date_added = album.date_added.min(song.date_added); + + if !song.album_artists.is_empty() { + album.artists = song.album_artists.clone(); + } else if !song.artists.is_empty() { + album.artists = song.artists.clone(); + } + + album.songs.insert(song_id); + } +} + +mod storage { + use super::*; + + #[derive(Serialize, Deserialize)] + pub struct Artist { + pub name: Option, + pub albums: HashSet, + pub album_appearances: HashSet, + } + + #[derive(Clone, Default, Serialize, Deserialize)] + pub struct Album { + pub name: Option, + pub artwork: Option, + pub artists: Vec, + pub year: Option, + pub date_added: i64, + pub songs: HashSet, + } +} + +#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub struct ArtistID(u64); + +#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub struct AlbumID(u64); + +#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub struct SongID(u64); + +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct ArtistKey { + pub name: Option, +} + +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct AlbumKey { + pub artists: Vec, + pub name: Option, +} + +#[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub struct SongKey { + pub virtual_path: PathBuf, +} + +impl Song { + pub fn album_key(&self) -> AlbumKey { + let album_artists = match self.album_artists.is_empty() { + true => &self.artists, + false => &self.album_artists, + }; + + AlbumKey { + artists: album_artists.iter().cloned().collect(), + name: self.album.clone(), + } + } + pub fn album_id(&self) -> AlbumID { + // TODO we .song_key is cloning names just so we can hash them! Slow! + let key: AlbumKey = self.album_key(); + (&key).into() + } + + pub fn song_key(&self) -> SongKey { + SongKey { + virtual_path: self.virtual_path.clone(), + } + } + + pub fn song_id(&self) -> SongID { + // TODO we .song_key is cloning path just so we can hash it! Slow! + let key: SongKey = self.song_key(); + (&key).into() + } +} + +impl From<&ArtistKey> for ArtistID { + fn from(key: &ArtistKey) -> Self { + ArtistID(key.id()) + } +} + +impl From<&AlbumKey> for AlbumID { + fn from(key: &AlbumKey) -> Self { + AlbumID(key.id()) + } +} + +impl From<&SongKey> for SongID { + fn from(key: &SongKey) -> Self { + SongID(key.id()) + } +} + +trait ID { + fn id(&self) -> u64; +} + +impl ID for T { + fn id(&self) -> u64 { + let mut hasher = DefaultHasher::default(); + self.hash(&mut hasher); + hasher.finish() + } +} + +#[cfg(test)] +mod test { + + use super::*; + + use crate::app::test; + use crate::test_name; + + const TEST_MOUNT_NAME: &str = "root"; + + #[tokio::test] + async fn can_get_random_albums() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + let albums = ctx.index_manager.get_random_albums(1).await.unwrap(); + assert_eq!(albums.len(), 1); + } + + #[tokio::test] + async fn can_get_recent_albums() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.scanner.update().await.unwrap(); + let albums = ctx.index_manager.get_recent_albums(2).await.unwrap(); + assert_eq!(albums.len(), 2); + } + + #[tokio::test] + async fn can_get_a_song() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + + ctx.scanner.update().await.unwrap(); + + let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); + let song_virtual_path = picnic_virtual_dir.join("05 - シャーベット (Sherbet).mp3"); + let artwork_virtual_path = picnic_virtual_dir.join("Folder.png"); + + let song = ctx + .index_manager + .get_song(&SongKey { + virtual_path: song_virtual_path.clone(), + }) + .await + .unwrap(); + assert_eq!(song.virtual_path, song_virtual_path); + assert_eq!(song.track_number, Some(5)); + assert_eq!(song.disc_number, None); + assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned())); + assert_eq!(song.artists, vec!["Tobokegao".to_owned()]); + assert_eq!(song.album_artists, Vec::::new()); + assert_eq!(song.album, Some("Picnic".to_owned())); + assert_eq!(song.year, Some(2016)); + assert_eq!(song.artwork, Some(artwork_virtual_path)); + } +} diff --git a/src/app/index/search.rs b/src/app/index/search.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/app/index/search.rs @@ -0,0 +1 @@ + diff --git a/src/app/lastfm.rs b/src/app/lastfm.rs index eec1bb2..a602877 100644 --- a/src/app/lastfm.rs +++ b/src/app/lastfm.rs @@ -2,7 +2,7 @@ use rustfm_scrobble::{Scrobble, Scrobbler}; use std::path::Path; use user::AuthToken; -use crate::app::{collection, user}; +use crate::app::{index, user}; const LASTFM_API_KEY: &str = "02b96c939a2b451c31dfd67add1f696e"; const LASTFM_API_SECRET: &str = "0f25a80ceef4b470b5cb97d99d4b3420"; @@ -16,21 +16,21 @@ pub enum Error { #[error("Failed to emit last.fm now playing update")] NowPlaying(rustfm_scrobble::ScrobblerError), #[error(transparent)] - Query(#[from] collection::Error), + Query(#[from] index::Error), #[error(transparent)] User(#[from] user::Error), } #[derive(Clone)] pub struct Manager { - browser: collection::Browser, + index_manager: index::Manager, user_manager: user::Manager, } impl Manager { - pub fn new(browser: collection::Browser, user_manager: user::Manager) -> Self { + pub fn new(index_manager: index::Manager, user_manager: user::Manager) -> Self { Self { - browser, + index_manager, user_manager, } } @@ -81,7 +81,10 @@ impl Manager { } async fn scrobble_from_path(&self, track: &Path) -> Result { - let song = self.browser.get_song(track).await?; + let song_key = index::SongKey { + virtual_path: track.to_owned(), + }; + let song = self.index_manager.get_song(&song_key).await?; Ok(Scrobble::new( song.artists.first().map(|s| s.as_str()).unwrap_or(""), song.title.as_deref().unwrap_or(""), diff --git a/src/app/playlist.rs b/src/app/playlist.rs index ca70b92..3bb9d37 100644 --- a/src/app/playlist.rs +++ b/src/app/playlist.rs @@ -2,7 +2,6 @@ use core::clone::Clone; use sqlx::{Acquire, QueryBuilder, Sqlite}; use std::path::PathBuf; -use crate::app::collection::SongKey; use crate::app::vfs; use crate::db::{self, DB}; @@ -126,7 +125,7 @@ impl Manager { &self, playlist_name: &str, owner: &str, - ) -> Result, Error> { + ) -> Result, Error> { let songs = { let mut connection = self.db.connect().await?; @@ -231,15 +230,14 @@ mod test { .build() .await; - ctx.updater.update().await.unwrap(); + ctx.scanner.update().await.unwrap(); let playlist_content = ctx - .browser - .flatten(Path::new(TEST_MOUNT_NAME)) + .index_manager + .flatten(PathBuf::from(TEST_MOUNT_NAME)) .await .unwrap() .into_iter() - .map(|s| s.virtual_path) .collect::>(); assert_eq!(playlist_content.len(), 13); @@ -296,15 +294,14 @@ mod test { .build() .await; - ctx.updater.update().await.unwrap(); + ctx.scanner.update().await.unwrap(); let playlist_content = ctx - .browser - .flatten(Path::new(TEST_MOUNT_NAME)) + .index_manager + .flatten(PathBuf::from(TEST_MOUNT_NAME)) .await .unwrap() .into_iter() - .map(|s| s.virtual_path) .collect::>(); assert_eq!(playlist_content.len(), 13); @@ -329,6 +326,6 @@ mod test { ] .iter() .collect(); - assert_eq!(songs[0].virtual_path, first_song_path); + assert_eq!(songs[0], first_song_path); } } diff --git a/src/app/scanner.rs b/src/app/scanner.rs new file mode 100644 index 0000000..17fd071 --- /dev/null +++ b/src/app/scanner.rs @@ -0,0 +1,410 @@ +use log::{error, info}; +use rayon::{Scope, ThreadPoolBuilder}; +use regex::Regex; +use std::fs; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::Arc; +use std::{cmp::min, time::Duration}; +use tokio::sync::mpsc::error::TryRecvError; +use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; +use tokio::sync::Notify; +use tokio::time::Instant; + +use crate::app::{formats, index, settings, vfs}; + +#[derive(Debug, PartialEq, Eq)] +pub struct Directory { + pub virtual_path: PathBuf, + pub virtual_parent: Option, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Song { + pub path: PathBuf, + pub virtual_path: PathBuf, + pub virtual_parent: PathBuf, + pub track_number: Option, + pub disc_number: Option, + pub title: Option, + pub artists: Vec, + pub album_artists: Vec, + pub year: Option, + pub album: Option, + pub artwork: Option, + pub duration: Option, + pub lyricists: Vec, + pub composers: Vec, + pub genres: Vec, + pub labels: Vec, + pub date_added: i64, +} + +#[derive(Clone)] +pub struct Scanner { + index_manager: index::Manager, + settings_manager: settings::Manager, + vfs_manager: vfs::Manager, + pending_scan: Arc, +} + +impl Scanner { + pub async fn new( + index_manager: index::Manager, + settings_manager: settings::Manager, + vfs_manager: vfs::Manager, + ) -> Result { + let scanner = Self { + index_manager, + vfs_manager, + settings_manager, + pending_scan: Arc::new(Notify::new()), + }; + + tokio::spawn({ + let mut scanner = scanner.clone(); + async move { + loop { + scanner.pending_scan.notified().await; + if let Err(e) = scanner.update().await { + error!("Error while updating index: {}", e); + } + } + } + }); + + Ok(scanner) + } + + pub fn trigger_scan(&self) { + self.pending_scan.notify_one(); + } + + pub fn begin_periodic_scans(&self) { + tokio::spawn({ + let index = self.clone(); + async move { + loop { + index.trigger_scan(); + let sleep_duration = index + .settings_manager + .get_index_sleep_duration() + .await + .unwrap_or_else(|e| { + error!("Could not retrieve index sleep duration: {}", e); + Duration::from_secs(1800) + }); + tokio::time::sleep(sleep_duration).await; + } + } + }); + } + + pub async fn update(&mut self) -> Result<(), index::Error> { + let start = Instant::now(); + info!("Beginning collection scan"); + + let album_art_pattern = self + .settings_manager + .get_index_album_art_pattern() + .await + .ok(); + + let (scan_directories_output, mut collection_directories_input) = unbounded_channel(); + let (scan_songs_output, mut collection_songs_input) = unbounded_channel(); + + let scan = Scan::new( + scan_directories_output, + scan_songs_output, + self.vfs_manager.clone(), + album_art_pattern, + ); + + let index_task = tokio::task::spawn_blocking(move || { + let mut index_builder = index::Builder::default(); + + loop { + let exhausted_songs = match collection_songs_input.try_recv() { + Ok(song) => { + index_builder.add_song(song); + false + } + Err(TryRecvError::Empty) => false, + Err(TryRecvError::Disconnected) => true, + }; + + let exhausted_directories = match collection_directories_input.try_recv() { + Ok(directory) => { + index_builder.add_directory(directory); + false + } + Err(TryRecvError::Empty) => false, + Err(TryRecvError::Disconnected) => true, + }; + + if exhausted_directories && exhausted_songs { + break; + } + } + + index_builder.build() + }); + + let index = tokio::join!(scan.start(), index_task).1?; + self.index_manager.persist_index(&index).await?; + self.index_manager.replace_index(index).await; + + info!( + "Collection scan took {} seconds", + start.elapsed().as_millis() as f32 / 1000.0 + ); + + Ok(()) + } +} + +struct Scan { + directories_output: UnboundedSender, + songs_output: UnboundedSender, + vfs_manager: vfs::Manager, + artwork_regex: Option, +} + +impl Scan { + pub fn new( + directories_output: UnboundedSender, + songs_output: UnboundedSender, + vfs_manager: vfs::Manager, + artwork_regex: Option, + ) -> Self { + Self { + directories_output, + songs_output, + vfs_manager, + artwork_regex, + } + } + + pub async fn start(self) -> Result<(), index::Error> { + let vfs = self.vfs_manager.get_vfs().await?; + let roots = vfs.mounts().clone(); + + let key = "POLARIS_NUM_TRAVERSER_THREADS"; + let num_threads = std::env::var_os(key) + .map(|v| v.to_string_lossy().to_string()) + .and_then(|v| usize::from_str(&v).ok()) + .unwrap_or_else(|| min(num_cpus::get(), 8)); + info!("Browsing collection using {} threads", num_threads); + + let directories_output = self.directories_output.clone(); + let songs_output = self.songs_output.clone(); + let artwork_regex = self.artwork_regex.clone(); + + let thread_pool = ThreadPoolBuilder::new().num_threads(num_threads).build()?; + thread_pool.scope({ + |scope| { + for root in roots { + scope.spawn(|scope| { + process_directory( + scope, + root.source, + root.name, + directories_output.clone(), + songs_output.clone(), + artwork_regex.clone(), + ); + }); + } + } + }); + + Ok(()) + } +} + +fn process_directory, Q: AsRef>( + scope: &Scope, + real_path: P, + virtual_path: Q, + directories_output: UnboundedSender, + songs_output: UnboundedSender, + artwork_regex: Option, +) { + let read_dir = match fs::read_dir(&real_path) { + Ok(read_dir) => read_dir, + Err(e) => { + error!( + "Directory read error for `{}`: {}", + real_path.as_ref().display(), + e + ); + return; + } + }; + + let mut songs = vec![]; + let mut artwork_file = None; + + for entry in read_dir { + let name = match entry { + Ok(ref f) => f.file_name(), + Err(e) => { + error!( + "File read error within `{}`: {}", + real_path.as_ref().display(), + e + ); + break; + } + }; + + let entry_real_path = real_path.as_ref().join(&name); + let entry_virtual_path = virtual_path.as_ref().join(&name); + + if entry_real_path.is_dir() { + scope.spawn({ + let directories_output = directories_output.clone(); + let songs_output = songs_output.clone(); + let artwork_regex = artwork_regex.clone(); + |scope| { + process_directory( + scope, + entry_real_path, + entry_virtual_path, + directories_output, + songs_output, + artwork_regex, + ); + } + }); + } else if let Some(metadata) = formats::read_metadata(&entry_real_path) { + songs.push(Song { + path: entry_real_path.clone(), + virtual_path: entry_virtual_path.clone(), + virtual_parent: entry_virtual_path.parent().unwrap().to_owned(), + track_number: metadata.track_number.map(|n| n as i64), + disc_number: metadata.disc_number.map(|n| n as i64), + title: metadata.title, + artists: metadata.artists, + album_artists: metadata.album_artists, + year: metadata.year.map(|n| n as i64), + album: metadata.album, + artwork: metadata.has_artwork.then(|| entry_virtual_path.clone()), + duration: metadata.duration.map(|n| n as i64), + lyricists: metadata.lyricists, + composers: metadata.composers, + genres: metadata.genres, + labels: metadata.labels, + date_added: get_date_created(&entry_real_path).unwrap_or_default(), + }); + } else if artwork_file.is_none() + && artwork_regex + .as_ref() + .is_some_and(|r| r.is_match(name.to_str().unwrap_or_default())) + { + artwork_file = Some(entry_virtual_path); + } + } + + for mut song in songs { + song.artwork = song.artwork.or_else(|| artwork_file.clone()); + songs_output.send(song).ok(); + } + + directories_output + .send(Directory { + virtual_path: virtual_path.as_ref().to_owned(), + virtual_parent: virtual_path.as_ref().parent().map(Path::to_owned), + }) + .ok(); +} + +fn get_date_created>(path: P) -> Option { + if let Ok(t) = fs::metadata(path).and_then(|m| m.created().or_else(|_| m.modified())) { + t.duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .ok() + } else { + None + } +} + +#[cfg(test)] +mod test { + use super::*; + + use std::path::PathBuf; + + use crate::{ + app::{settings, test}, + test_name, + }; + + const TEST_MOUNT_NAME: &str = "root"; + + #[tokio::test] + async fn scan_adds_new_content() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + + ctx.scanner.update().await.unwrap(); + ctx.scanner.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts + + todo!(); + + // assert_eq!(all_directories.len(), 6); + // assert_eq!(all_songs.len(), 13); + } + + #[tokio::test] + async fn finds_embedded_artwork() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + + ctx.scanner.update().await.unwrap(); + + let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); + let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3"); + + let song = ctx + .index_manager + .get_song(&index::SongKey { + virtual_path: song_virtual_path.clone(), + }) + .await + .unwrap(); + assert_eq!(song.artwork, Some(song_virtual_path)); + } + + #[tokio::test] + async fn album_art_pattern_is_case_insensitive() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + + let patterns = vec!["folder", "FOLDER"]; + + for pattern in patterns.into_iter() { + ctx.settings_manager + .amend(&settings::NewSettings { + album_art_pattern: Some(pattern.to_owned()), + ..Default::default() + }) + .await + .unwrap(); + ctx.scanner.update().await.unwrap(); + + let hunted_virtual_dir: PathBuf = + [TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect(); + let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg"); + let song = &ctx.index_manager.flatten(hunted_virtual_dir).await.unwrap()[0]; + todo!(); + // assert_eq!(song.artwork, Some(artwork_virtual_path)); + } + } +} diff --git a/src/app/test.rs b/src/app/test.rs index dd4b8ca..ba4fbbb 100644 --- a/src/app/test.rs +++ b/src/app/test.rs @@ -1,13 +1,12 @@ use std::path::PathBuf; -use crate::app::{collection, config, ddns, playlist, settings, user, vfs}; +use crate::app::{config, ddns, index, playlist, scanner, settings, user, vfs}; use crate::db::DB; use crate::test::*; pub struct Context { - pub browser: collection::Browser, - pub index_manager: collection::IndexManager, - pub updater: collection::Updater, + pub index_manager: index::Manager, + pub scanner: scanner::Scanner, pub config_manager: config::Manager, pub ddns_manager: ddns::Manager, pub playlist_manager: playlist::Manager, @@ -66,9 +65,8 @@ impl ContextBuilder { vfs_manager.clone(), ddns_manager.clone(), ); - let browser = collection::Browser::new(db.clone(), vfs_manager.clone()); - let index_manager = collection::IndexManager::new(db.clone()).await; - let updater = collection::Updater::new( + let index_manager = index::Manager::new(db.clone()).await; + let scanner = scanner::Scanner::new( index_manager.clone(), settings_manager.clone(), vfs_manager.clone(), @@ -80,9 +78,8 @@ impl ContextBuilder { config_manager.apply(&self.config).await.unwrap(); Context { - browser, index_manager, - updater, + scanner, config_manager, ddns_manager, playlist_manager, diff --git a/src/main.rs b/src/main.rs index ba90457..ab02616 100644 --- a/src/main.rs +++ b/src/main.rs @@ -144,7 +144,7 @@ fn main() -> Result<(), Error> { async fn async_main(cli_options: CLIOptions, paths: paths::Paths) -> Result<(), Error> { // Create and run app let app = app::App::new(cli_options.port.unwrap_or(5050), paths).await?; - app.updater.begin_periodic_scans(); + app.scanner.begin_periodic_scans(); app.ddns_manager.begin_periodic_updates(); // Start server diff --git a/src/server/axum.rs b/src/server/axum.rs index 58a88d8..c6fc111 100644 --- a/src/server/axum.rs +++ b/src/server/axum.rs @@ -27,21 +27,15 @@ pub async fn launch(app: App) -> Result<(), std::io::Error> { Ok(()) } -impl FromRef for app::collection::Browser { - fn from_ref(app: &App) -> Self { - app.browser.clone() - } -} - -impl FromRef for app::collection::IndexManager { +impl FromRef for app::index::Manager { fn from_ref(app: &App) -> Self { app.index_manager.clone() } } -impl FromRef for app::collection::Updater { +impl FromRef for app::scanner::Scanner { fn from_ref(app: &App) -> Self { - app.updater.clone() + app.scanner.clone() } } diff --git a/src/server/axum/api.rs b/src/server/axum/api.rs index 0b766d1..5c76074 100644 --- a/src/server/axum/api.rs +++ b/src/server/axum/api.rs @@ -13,7 +13,7 @@ use base64::{prelude::BASE64_STANDARD_NO_PAD, Engine}; use percent_encoding::percent_decode_str; use crate::{ - app::{collection, config, ddns, lastfm, playlist, settings, thumbnail, user, vfs, App}, + app::{config, ddns, index, lastfm, playlist, scanner, settings, thumbnail, user, vfs, App}, server::{ dto, error::APIError, APIMajorVersion, API_ARRAY_SEPARATOR, API_MAJOR_VERSION, API_MINOR_VERSION, @@ -254,16 +254,13 @@ async fn put_preferences( async fn post_trigger_index( _admin_rights: AdminRights, - State(updater): State, + State(scanner): State, ) -> Result<(), APIError> { - updater.trigger_scan(); + scanner.trigger_scan(); Ok(()) } -fn collection_files_to_response( - files: Vec, - api_version: APIMajorVersion, -) -> Response { +fn index_files_to_response(files: Vec, api_version: APIMajorVersion) -> Response { match api_version { APIMajorVersion::V7 => Json( files @@ -282,23 +279,23 @@ fn collection_files_to_response( } } -fn songs_to_response(files: Vec, api_version: APIMajorVersion) -> Response { +fn songs_to_response(files: Vec, api_version: APIMajorVersion) -> Response { match api_version { APIMajorVersion::V7 => Json( files .into_iter() - .map(|f| f.into()) + .map(|p| index::SongKey { virtual_path: p }.into()) .collect::>(), ) .into_response(), APIMajorVersion::V8 => Json(dto::SongList { - paths: files.into_iter().map(|s| s.virtual_path).collect(), + paths: files.into_iter().collect(), }) .into_response(), } } -fn albums_to_response(albums: Vec, api_version: APIMajorVersion) -> Response { +fn albums_to_response(albums: Vec, api_version: APIMajorVersion) -> Response { match api_version { APIMajorVersion::V7 => Json( albums @@ -320,32 +317,32 @@ fn albums_to_response(albums: Vec, api_version: APIMajorVersi async fn get_browse_root( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, ) -> Response { let result = match index_manager.browse(PathBuf::new()).await { Ok(r) => r, Err(e) => return APIError::from(e).into_response(), }; - collection_files_to_response(result, api_version) + index_files_to_response(result, api_version) } async fn get_browse( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, Path(path): Path, ) -> Response { let result = match index_manager.browse(path).await { Ok(r) => r, Err(e) => return APIError::from(e).into_response(), }; - collection_files_to_response(result, api_version) + index_files_to_response(result, api_version) } async fn get_flatten_root( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, ) -> Response { let songs = match index_manager.flatten(PathBuf::new()).await { Ok(s) => s, @@ -357,7 +354,7 @@ async fn get_flatten_root( async fn get_flatten( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, Path(path): Path, ) -> Response { let songs = match index_manager.flatten(path).await { @@ -369,10 +366,10 @@ async fn get_flatten( async fn get_artist( _auth: Auth, - State(index_manager): State, + State(index_manager): State, Path(artist): Path, ) -> Result, APIError> { - let artist_key = collection::ArtistKey { + let artist_key = index::ArtistKey { name: (!artist.is_empty()).then_some(artist), }; Ok(Json(index_manager.get_artist(&artist_key).await?.into())) @@ -380,10 +377,10 @@ async fn get_artist( async fn get_album( _auth: Auth, - State(index_manager): State, + State(index_manager): State, Path((artists, name)): Path<(String, String)>, ) -> Result, APIError> { - let album_key = collection::AlbumKey { + let album_key = index::AlbumKey { artists: artists .split(API_ARRAY_SEPARATOR) .map(str::to_owned) @@ -396,7 +393,7 @@ async fn get_album( async fn get_random( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, ) -> Response { let albums = match index_manager.get_random_albums(20).await { Ok(d) => d, @@ -408,7 +405,7 @@ async fn get_random( async fn get_recent( _auth: Auth, api_version: APIMajorVersion, - State(index_manager): State, + State(index_manager): State, ) -> Response { let albums = match index_manager.get_recent_albums(20).await { Ok(d) => d, @@ -420,26 +417,26 @@ async fn get_recent( async fn get_search_root( _auth: Auth, api_version: APIMajorVersion, - State(browser): State, + State(index_manager): State, ) -> Response { - let files = match browser.search("").await { + let files = match index_manager.search("").await { Ok(f) => f, Err(e) => return APIError::from(e).into_response(), }; - collection_files_to_response(files, api_version) + songs_to_response(files, api_version) } async fn get_search( _auth: Auth, api_version: APIMajorVersion, - State(browser): State, + State(index_manager): State, Path(query): Path, ) -> Response { - let files = match browser.search(&query).await { + let files = match index_manager.search(&query).await { Ok(f) => f, Err(e) => return APIError::from(e).into_response(), }; - collection_files_to_response(files, api_version) + songs_to_response(files, api_version) } async fn get_playlists( diff --git a/src/server/axum/error.rs b/src/server/axum/error.rs index 0b98fcd..630bfac 100644 --- a/src/server/axum/error.rs +++ b/src/server/axum/error.rs @@ -23,6 +23,7 @@ impl IntoResponse for APIError { APIError::DirectoryNotFound(_) => StatusCode::NOT_FOUND, APIError::ArtistNotFound => StatusCode::NOT_FOUND, APIError::AlbumNotFound => StatusCode::NOT_FOUND, + APIError::SongNotFound => StatusCode::NOT_FOUND, APIError::EmbeddedArtworkNotFound => StatusCode::NOT_FOUND, APIError::EmptyPassword => StatusCode::BAD_REQUEST, APIError::EmptyUsername => StatusCode::BAD_REQUEST, diff --git a/src/server/dto/v7.rs b/src/server/dto/v7.rs index 5eaa0c8..c9ec27d 100644 --- a/src/server/dto/v7.rs +++ b/src/server/dto/v7.rs @@ -1,9 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::app::{ - collection::{self}, - config, ddns, settings, thumbnail, user, vfs, -}; +use crate::app::{config, ddns, index, settings, thumbnail, user, vfs}; use std::{convert::From, path::PathBuf}; #[derive(PartialEq, Eq, Debug, Serialize, Deserialize)] @@ -237,17 +234,17 @@ pub enum CollectionFile { Song(Song), } -impl From for CollectionFile { - fn from(f: collection::File) -> Self { +impl From for CollectionFile { + fn from(f: index::File) -> Self { match f { - collection::File::Directory(d) => Self::Directory(Directory { + index::File::Directory(d) => Self::Directory(Directory { path: d, artist: None, year: None, album: None, artwork: None, }), - collection::File::Song(s) => Self::Song(Song { + index::File::Song(s) => Self::Song(Song { path: s, track_number: None, disc_number: None, @@ -299,8 +296,8 @@ pub struct Song { pub label: Option, } -impl From for Song { - fn from(song_key: collection::SongKey) -> Self { +impl From for Song { + fn from(song_key: index::SongKey) -> Self { Self { path: song_key.virtual_path, track_number: None, @@ -320,8 +317,8 @@ impl From for Song { } } -impl From for Song { - fn from(s: collection::Song) -> Self { +impl From for Song { + fn from(s: index::Song) -> Self { Self { path: s.virtual_path, track_number: s.track_number, @@ -350,8 +347,8 @@ pub struct Directory { pub artwork: Option, } -impl From for Directory { - fn from(album: collection::Album) -> Self { +impl From for Directory { + fn from(album: index::Album) -> Self { let path = album .songs .first() diff --git a/src/server/dto/v8.rs b/src/server/dto/v8.rs index c158746..3ec727e 100644 --- a/src/server/dto/v8.rs +++ b/src/server/dto/v8.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -use crate::app::{collection, config, ddns, settings, thumbnail, user, vfs}; +use crate::app::{config, ddns, index, settings, thumbnail, user, vfs}; use std::{convert::From, path::PathBuf}; #[derive(PartialEq, Eq, Debug, Serialize, Deserialize)] @@ -259,8 +259,8 @@ pub struct Song { pub labels: Vec, } -impl From for Song { - fn from(s: collection::Song) -> Self { +impl From for Song { + fn from(s: index::Song) -> Self { Self { path: s.virtual_path, track_number: s.track_number, @@ -291,14 +291,14 @@ pub struct BrowserEntry { pub is_directory: bool, } -impl From for BrowserEntry { - fn from(file: collection::File) -> Self { +impl From for BrowserEntry { + fn from(file: index::File) -> Self { match file { - collection::File::Directory(d) => Self { + index::File::Directory(d) => Self { is_directory: true, path: d, }, - collection::File::Song(s) => Self { + index::File::Song(s) => Self { is_directory: false, path: s, }, @@ -313,8 +313,8 @@ pub struct Artist { pub album_appearances: Vec, } -impl From for Artist { - fn from(a: collection::Artist) -> Self { +impl From for Artist { + fn from(a: index::Artist) -> Self { Self { name: a.name, albums: a.albums.into_iter().map(|a| a.into()).collect(), @@ -335,8 +335,8 @@ pub struct AlbumHeader { pub year: Option, } -impl From for AlbumHeader { - fn from(a: collection::Album) -> Self { +impl From for AlbumHeader { + fn from(a: index::Album) -> Self { Self { name: a.name, artwork: a.artwork.map(|a| a.to_string_lossy().to_string()), @@ -353,8 +353,8 @@ pub struct Album { pub songs: Vec, } -impl From for Album { - fn from(mut a: collection::Album) -> Self { +impl From for Album { + fn from(mut a: index::Album) -> Self { let songs = a.songs.drain(..).map(|s| s.into()).collect(); Self { header: a.into(), diff --git a/src/server/error.rs b/src/server/error.rs index 8c42963..44faf6c 100644 --- a/src/server/error.rs +++ b/src/server/error.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use thiserror::Error; -use crate::app::{collection, config, ddns, lastfm, playlist, settings, thumbnail, user, vfs}; +use crate::app::{config, ddns, index, lastfm, playlist, settings, thumbnail, user, vfs}; use crate::db; #[derive(Error, Debug)] @@ -30,6 +30,8 @@ pub enum APIError { ArtistNotFound, #[error("Album not found")] AlbumNotFound, + #[error("Song not found")] + SongNotFound, #[error("DDNS update query failed with HTTP status {0}")] DdnsUpdateQueryFailed(u16), #[error("Cannot delete your own account")] @@ -86,19 +88,20 @@ pub enum APIError { VFSPathNotFound, } -impl From for APIError { - fn from(error: collection::Error) -> APIError { +impl From for APIError { + fn from(error: index::Error) -> APIError { match error { - collection::Error::DirectoryNotFound(d) => APIError::DirectoryNotFound(d), - collection::Error::ArtistNotFound => APIError::ArtistNotFound, - collection::Error::AlbumNotFound => APIError::AlbumNotFound, - collection::Error::Database(e) => APIError::Database(e), - collection::Error::DatabaseConnection(e) => e.into(), - collection::Error::Vfs(e) => e.into(), - collection::Error::IndexDeserializationError => APIError::Internal, - collection::Error::IndexSerializationError => APIError::Internal, - collection::Error::ThreadPoolBuilder(_) => APIError::Internal, - collection::Error::ThreadJoining(_) => APIError::Internal, + index::Error::DirectoryNotFound(d) => APIError::DirectoryNotFound(d), + index::Error::ArtistNotFound => APIError::ArtistNotFound, + index::Error::AlbumNotFound => APIError::AlbumNotFound, + index::Error::SongNotFound => APIError::SongNotFound, + index::Error::Database(e) => APIError::Database(e), + index::Error::DatabaseConnection(e) => e.into(), + index::Error::Vfs(e) => e.into(), + index::Error::IndexDeserializationError => APIError::Internal, + index::Error::IndexSerializationError => APIError::Internal, + index::Error::ThreadPoolBuilder(_) => APIError::Internal, + index::Error::ThreadJoining(_) => APIError::Internal, } } }