From 2012258a725237cb38b298716747f698363a2acf Mon Sep 17 00:00:00 2001 From: Antoine Gersant Date: Mon, 29 Jul 2024 02:07:28 -0700 Subject: [PATCH] Indexing WIP --- src/app/collection/browser.rs | 38 ---------- src/app/collection/index.rs | 126 ++++++++++++++++++++++++++++++--- src/app/collection/inserter.rs | 11 +++ src/app/collection/types.rs | 15 +++- src/app/collection/updater.rs | 2 + src/app/test.rs | 2 + src/server/axum.rs | 6 ++ src/server/axum/api.rs | 25 +++---- src/server/dto/v7.rs | 17 ++++- src/server/dto/v8.rs | 58 +++++++++------ src/server/test.rs | 2 +- src/server/test/admin.rs | 4 +- src/server/test/collection.rs | 66 +++++++++++++---- 13 files changed, 270 insertions(+), 102 deletions(-) diff --git a/src/app/collection/browser.rs b/src/app/collection/browser.rs index 51184a9..64f84cc 100644 --- a/src/app/collection/browser.rs +++ b/src/app/collection/browser.rs @@ -108,22 +108,6 @@ impl Browser { Ok(songs) } - pub async fn get_random_albums( - &self, - count: i64, - ) -> Result, collection::Error> { - // TODO move to Index - Ok(vec![]) - } - - pub async fn get_recent_albums( - &self, - count: i64, - ) -> Result, collection::Error> { - // TODO move to Index - Ok(vec![]) - } - pub async fn search(&self, query: &str) -> Result, collection::Error> { let mut connection = self.db.connect().await?; let like_test = format!("%{}%", query); @@ -285,28 +269,6 @@ mod test { assert_eq!(songs.len(), 7); } - #[tokio::test] - async fn can_get_random_albums() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let albums = ctx.browser.get_random_albums(1).await.unwrap(); - assert_eq!(albums.len(), 1); - } - - #[tokio::test] - async fn can_get_recent_albums() { - let mut ctx = test::ContextBuilder::new(test_name!()) - .mount(TEST_MOUNT_NAME, "test-data/small-collection") - .build() - .await; - ctx.updater.update().await.unwrap(); - let albums = ctx.browser.get_recent_albums(2).await.unwrap(); - assert_eq!(albums.len(), 2); - } - #[tokio::test] async fn can_get_a_song() { let mut ctx = test::ContextBuilder::new(test_name!()) diff --git a/src/app/collection/index.rs b/src/app/collection/index.rs index 123ee44..22e9613 100644 --- a/src/app/collection/index.rs +++ b/src/app/collection/index.rs @@ -1,32 +1,142 @@ -use std::{collections::HashMap, sync::Arc}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; +use rand::{rngs::ThreadRng, seq::IteratorRandom}; use tokio::sync::RwLock; use crate::app::collection; -#[derive(Clone, Default)] +#[derive(Clone)] pub struct Index { lookups: Arc>, } impl Index { pub fn new() -> Self { - Self::default() + Self { + lookups: Arc::default(), + } } - pub async fn replace_lookup_tables(&mut self, new_lookups: Lookups) { + pub(super) async fn replace_lookup_tables(&mut self, new_lookups: Lookups) { let mut lock = self.lookups.write().await; *lock = new_lookups; } + + pub async fn get_random_albums( + &self, + count: usize, + ) -> Result, collection::Error> { + let lookups = self.lookups.read().await; + Ok(lookups + .songs_by_albums + .keys() + .choose_multiple(&mut ThreadRng::default(), count) + .iter() + .filter_map(|k| lookups.get_album(k)) + .collect()) + } + + pub async fn get_recent_albums( + &self, + count: i64, + ) -> Result, collection::Error> { + // TODO implement + Ok(vec![]) + } +} + +// TODO how can clients refer to an album? +#[derive(Clone, PartialEq, Eq, Hash)] +struct AlbumKey { + pub artists: Vec, + pub name: Option, } #[derive(Default)] -pub struct Lookups { - data: HashMap, +pub(super) struct Lookups { + all_songs: HashMap, + songs_by_albums: HashMap>, // TODO should this store collection::Album structs instead? } impl Lookups { - pub fn add_song(&mut self, _song: &collection::Song) { - // todo!() + pub fn add_song(&mut self, song: &collection::Song) { + self.all_songs + .insert(song.virtual_path.clone(), song.clone()); + + let album_artists = match song.album_artists.0.is_empty() { + true => &song.artists.0, + false => &song.album_artists.0, + }; + + let album_key = AlbumKey { + artists: album_artists.iter().cloned().collect(), + name: song.album.clone(), + }; + + let song_list = match self.songs_by_albums.get_mut(&album_key) { + Some(l) => l, + None => { + self.songs_by_albums + .insert(album_key.clone(), HashSet::new()); + self.songs_by_albums.get_mut(&album_key).unwrap() + } + }; + + song_list.insert(song.virtual_path.clone()); + } + + pub fn get_album(&self, key: &AlbumKey) -> Option { + let Some(songs) = self.songs_by_albums.get(key) else { + return None; + }; + + let songs: Vec<&collection::Song> = + songs.iter().filter_map(|s| self.all_songs.get(s)).collect(); + + Some(collection::Album { + name: key.name.clone(), + artwork: songs.iter().find_map(|s| s.artwork.clone()), + artists: key.artists.iter().cloned().collect(), + year: songs.iter().find_map(|s| s.year), + date_added: songs + .iter() + .min_by_key(|s| s.date_added) + .map(|s| s.date_added) + .unwrap_or_default(), + }) + } +} + +#[cfg(test)] +mod test { + + use crate::app::test; + use crate::test_name; + + const TEST_MOUNT_NAME: &str = "root"; + + #[tokio::test] + async fn can_get_random_albums() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.updater.update().await.unwrap(); + let albums = ctx.index.get_random_albums(1).await.unwrap(); + assert_eq!(albums.len(), 1); + } + + #[tokio::test] + async fn can_get_recent_albums() { + let mut ctx = test::ContextBuilder::new(test_name!()) + .mount(TEST_MOUNT_NAME, "test-data/small-collection") + .build() + .await; + ctx.updater.update().await.unwrap(); + let albums = ctx.index.get_recent_albums(2).await.unwrap(); + assert_eq!(albums.len(), 2); } } diff --git a/src/app/collection/inserter.rs b/src/app/collection/inserter.rs index 5d86139..9f682b8 100644 --- a/src/app/collection/inserter.rs +++ b/src/app/collection/inserter.rs @@ -23,6 +23,17 @@ impl<'q> sqlx::Encode<'q, Sqlite> for MultiString { } } +impl<'q> sqlx::Decode<'q, Sqlite> for MultiString { + fn decode( + value: >::ValueRef, + ) -> Result { + let s: &str = sqlx::Decode::::decode(value)?; + Ok(MultiString( + s.split(MultiString::SEPARATOR).map(str::to_owned).collect(), + )) + } +} + impl sqlx::Type for MultiString { fn type_info() -> SqliteTypeInfo { <&str as sqlx::Type>::type_info() diff --git a/src/app/collection/types.rs b/src/app/collection/types.rs index 1286ae3..26b083b 100644 --- a/src/app/collection/types.rs +++ b/src/app/collection/types.rs @@ -1,11 +1,13 @@ use std::path::PathBuf; +use sqlx::prelude::FromRow; + use crate::{ app::vfs::{self}, db, }; -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, FromRow, PartialEq, Eq)] pub struct MultiString(pub Vec); impl MultiString { @@ -43,7 +45,7 @@ pub enum File { Song(Song), } -#[derive(Debug, PartialEq, Eq)] +#[derive(Clone, Debug, PartialEq, Eq)] pub struct Song { pub id: i64, pub path: String, @@ -72,3 +74,12 @@ pub struct Directory { pub virtual_path: String, pub virtual_parent: Option, } + +#[derive(Debug, Default, PartialEq, Eq)] +pub struct Album { + pub name: Option, + pub artwork: Option, + pub artists: Vec, + pub year: Option, + pub date_added: i64, +} diff --git a/src/app/collection/updater.rs b/src/app/collection/updater.rs index ff72646..d33f828 100644 --- a/src/app/collection/updater.rs +++ b/src/app/collection/updater.rs @@ -47,6 +47,8 @@ impl Updater { } }); + // TODO populate index w/ whatever is already in DB + updater } diff --git a/src/app/test.rs b/src/app/test.rs index b90922e..a135bc5 100644 --- a/src/app/test.rs +++ b/src/app/test.rs @@ -7,6 +7,7 @@ use crate::test::*; pub struct Context { pub db: DB, pub browser: collection::Browser, + pub index: collection::Index, pub updater: collection::Updater, pub config_manager: config::Manager, pub ddns_manager: ddns::Manager, @@ -81,6 +82,7 @@ impl ContextBuilder { Context { db, browser, + index, updater, config_manager, ddns_manager, diff --git a/src/server/axum.rs b/src/server/axum.rs index 580006d..7a9ac71 100644 --- a/src/server/axum.rs +++ b/src/server/axum.rs @@ -33,6 +33,12 @@ impl FromRef for app::collection::Browser { } } +impl FromRef for app::collection::Index { + fn from_ref(app: &App) -> Self { + app.index.clone() + } +} + impl FromRef for app::collection::Updater { fn from_ref(app: &App) -> Self { app.updater.clone() diff --git a/src/server/axum/api.rs b/src/server/axum/api.rs index ae50dc8..4462534 100644 --- a/src/server/axum/api.rs +++ b/src/server/axum/api.rs @@ -269,7 +269,7 @@ fn collection_files_to_response( files .into_iter() .map(|f| f.into()) - .collect::>(), + .collect::>(), ) .into_response(), } @@ -294,23 +294,20 @@ fn songs_to_response(files: Vec, api_version: APIMajorVersion) } } -fn directories_to_response( - files: Vec, - api_version: APIMajorVersion, -) -> Response { +fn albums_to_response(albums: Vec, api_version: APIMajorVersion) -> Response { match api_version { APIMajorVersion::V7 => Json( - files + albums .into_iter() .map(|f| f.into()) .collect::>(), ) .into_response(), APIMajorVersion::V8 => Json( - files + albums .into_iter() .map(|f| f.into()) - .collect::>(), + .collect::>(), ) .into_response(), } @@ -371,25 +368,25 @@ async fn get_flatten( async fn get_random( _auth: Auth, api_version: APIMajorVersion, - State(browser): State, + State(index): State, ) -> Response { - let directories = match browser.get_random_albums(20).await { + let albums = match index.get_random_albums(20).await { Ok(d) => d, Err(e) => return APIError::from(e).into_response(), }; - directories_to_response(directories, api_version) + albums_to_response(albums, api_version) } async fn get_recent( _auth: Auth, api_version: APIMajorVersion, - State(browser): State, + State(index): State, ) -> Response { - let directories = match browser.get_recent_albums(20).await { + let albums = match index.get_recent_albums(20).await { Ok(d) => d, Err(e) => return APIError::from(e).into_response(), }; - directories_to_response(directories, api_version) + albums_to_response(albums, api_version) } async fn get_search_root( diff --git a/src/server/dto/v7.rs b/src/server/dto/v7.rs index 2f6290f..be6538d 100644 --- a/src/server/dto/v7.rs +++ b/src/server/dto/v7.rs @@ -302,7 +302,6 @@ pub struct Directory { pub year: Option, pub album: Option, pub artwork: Option, - pub date_added: i64, } impl From for Directory { @@ -313,7 +312,21 @@ impl From for Directory { year: None, album: None, artwork: None, - date_added: 0, + } + } +} + +impl From for Directory { + fn from(a: collection::Album) -> Self { + Self { + path: todo!(), // TODO implement + artist: match a.artists.is_empty() { + true => None, + false => Some(a.artists.join("")), + }, + year: a.year, + album: a.name, + artwork: a.artwork, } } } diff --git a/src/server/dto/v8.rs b/src/server/dto/v8.rs index 1484b81..a093045 100644 --- a/src/server/dto/v8.rs +++ b/src/server/dto/v8.rs @@ -228,21 +228,6 @@ impl From for Settings { } } -#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum CollectionFile { - Directory(Directory), - Song(Song), -} - -impl From for CollectionFile { - fn from(f: collection::File) -> Self { - match f { - collection::File::Directory(d) => Self::Directory(d.into()), - collection::File::Song(s) => Self::Song(s.into()), - } - } -} - #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct Song { pub path: String, @@ -296,16 +281,47 @@ impl From for Song { } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Directory { +pub struct BrowserEntry { pub path: String, + pub is_directory: bool, } -impl From for Directory { - fn from(d: collection::Directory) -> Self { - Self { - path: d.virtual_path, +impl From for BrowserEntry { + fn from(file: collection::File) -> Self { + match file { + collection::File::Directory(d) => Self { + is_directory: true, + path: d.virtual_path, + }, + collection::File::Song(s) => Self { + is_directory: false, + path: s.virtual_path, + }, } } } -// TODO: Preferencesshould have dto types +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct Album { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub artwork: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub artists: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub year: Option, +} + +impl From for Album { + fn from(a: collection::Album) -> Self { + Self { + name: a.name, + artwork: a.artwork, + artists: a.artists, + year: a.year, + } + } +} + +// TODO: Preferences should have dto types diff --git a/src/server/test.rs b/src/server/test.rs index ea0857f..a80eb6b 100644 --- a/src/server/test.rs +++ b/src/server/test.rs @@ -119,7 +119,7 @@ pub trait TestService { loop { let browse_request = protocol::browse::(Path::new("")); let response = self - .fetch_json::<(), Vec>(&browse_request) + .fetch_json::<(), Vec>(&browse_request) .await; let entries = response.body(); if !entries.is_empty() { diff --git a/src/server/test/admin.rs b/src/server/test/admin.rs index 4497c24..ba5d633 100644 --- a/src/server/test/admin.rs +++ b/src/server/test/admin.rs @@ -50,13 +50,13 @@ async fn trigger_index_golden_path() { let request = protocol::random::(); - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; let entries = response.body(); assert_eq!(entries.len(), 0); service.index().await; - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; let entries = response.body(); assert_eq!(entries.len(), 3); } diff --git a/src/server/test/collection.rs b/src/server/test/collection.rs index 41ca32b..6261aa0 100644 --- a/src/server/test/collection.rs +++ b/src/server/test/collection.rs @@ -24,7 +24,7 @@ async fn browse_root() { let request = protocol::browse::(&PathBuf::new()); let response = service - .fetch_json::<_, Vec>(&request) + .fetch_json::<_, Vec>(&request) .await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); @@ -42,7 +42,7 @@ async fn browse_directory() { let path: PathBuf = [TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect(); let request = protocol::browse::(&path); let response = service - .fetch_json::<_, Vec>(&request) + .fetch_json::<_, Vec>(&request) .await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); @@ -168,7 +168,7 @@ async fn random_golden_path() { service.login().await; let request = protocol::random::(); - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); assert_eq!(entries.len(), 3); @@ -184,7 +184,24 @@ async fn random_with_trailing_slash() { let mut request = protocol::random::(); add_trailing_slash(&mut request); - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; + assert_eq!(response.status(), StatusCode::OK); + let entries = response.body(); + assert_eq!(entries.len(), 3); +} + +#[tokio::test] +async fn random_golden_path_api_v7() { + let mut service = ServiceType::new(&test_name!()).await; + service.complete_initial_setup().await; + service.login_admin().await; + service.index().await; + service.login().await; + + let request = protocol::random::(); + let response = service + .fetch_json::<_, Vec>(&request) + .await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); assert_eq!(entries.len(), 3); @@ -207,7 +224,7 @@ async fn recent_golden_path() { service.login().await; let request = protocol::recent::(); - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); assert_eq!(entries.len(), 3); @@ -223,7 +240,24 @@ async fn recent_with_trailing_slash() { let mut request = protocol::recent::(); add_trailing_slash(&mut request); - let response = service.fetch_json::<_, Vec>(&request).await; + let response = service.fetch_json::<_, Vec>(&request).await; + assert_eq!(response.status(), StatusCode::OK); + let entries = response.body(); + assert_eq!(entries.len(), 3); +} + +#[tokio::test] +async fn recent_golden_path_api_v7() { + let mut service = ServiceType::new(&test_name!()).await; + service.complete_initial_setup().await; + service.login_admin().await; + service.index().await; + service.login().await; + + let request = protocol::recent::(); + let response = service + .fetch_json::<_, Vec>(&request) + .await; assert_eq!(response.status(), StatusCode::OK); let entries = response.body(); assert_eq!(entries.len(), 3); @@ -245,7 +279,7 @@ async fn search_without_query() { let request = protocol::search::(""); let response = service - .fetch_json::<_, Vec>(&request) + .fetch_json::<_, Vec>(&request) .await; assert_eq!(response.status(), StatusCode::OK); } @@ -260,14 +294,18 @@ async fn search_with_query() { let request = protocol::search::("door"); let response = service - .fetch_json::<_, Vec>(&request) + .fetch_json::<_, Vec>(&request) .await; let results = response.body(); assert_eq!(results.len(), 1); - match results[0] { - dto::CollectionFile::Song(ref s) => { - assert_eq!(s.title, Some("Beyond The Door".into())) - } - _ => panic!(), - } + + let path: PathBuf = [ + TEST_MOUNT_NAME, + "Khemmis", + "Hunted", + "04 - Beyond The Door.mp3", + ] + .iter() + .collect(); + assert_eq!(results[0].path, path.to_string_lossy()); }