vikarti.anatra/polaris-mirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor index

Browse source
This commit is contained in:
Antoine Gersant 2024-07-31 23:38:38 -07:00
parent a0624f7968
commit 8f2566f574
22 changed files with 1391 additions and 1311 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/app.rs
View file

@ -4,12 +4,13 @@ use std::path::PathBuf;
use crate::db::{self, DB}; use crate::db::{self, DB};
use crate::paths::Paths; use crate::paths::Paths;
pub mod collection;
pub mod config; pub mod config;
pub mod ddns; pub mod ddns;
pub mod formats; pub mod formats;
pub mod index;
pub mod lastfm; pub mod lastfm;
pub mod playlist; pub mod playlist;
pub mod scanner;
pub mod settings; pub mod settings;
pub mod thumbnail; pub mod thumbnail;
pub mod user; pub mod user;
@ -21,7 +22,7 @@ pub mod test;
#[derive(thiserror::Error, Debug)] #[derive(thiserror::Error, Debug)]
pub enum Error { pub enum Error {
#[error(transparent)] #[error(transparent)]
Collection(#[from] collection::Error), Collection(#[from] index::Error),
#[error(transparent)] #[error(transparent)]
Config(#[from] config::Error), Config(#[from] config::Error),
#[error(transparent)] #[error(transparent)]
@ -37,9 +38,8 @@ pub struct App {
pub port: u16, pub port: u16,
pub web_dir_path: PathBuf, pub web_dir_path: PathBuf,
pub swagger_dir_path: PathBuf, pub swagger_dir_path: PathBuf,
pub updater: collection::Updater, pub scanner: scanner::Scanner,
pub browser: collection::Browser, pub index_manager: index::Manager,
pub index_manager: collection::IndexManager,
pub config_manager: config::Manager, pub config_manager: config::Manager,
pub ddns_manager: ddns::Manager, pub ddns_manager: ddns::Manager,
pub lastfm_manager: lastfm::Manager, pub lastfm_manager: lastfm::Manager,
@ -67,9 +67,8 @@ impl App {
let auth_secret = settings_manager.get_auth_secret().await?; let auth_secret = settings_manager.get_auth_secret().await?;
let ddns_manager = ddns::Manager::new(db.clone()); let ddns_manager = ddns::Manager::new(db.clone());
let user_manager = user::Manager::new(db.clone(), auth_secret); let user_manager = user::Manager::new(db.clone(), auth_secret);
let index_manager = collection::IndexManager::new(db.clone()).await; let index_manager = index::Manager::new(db.clone()).await;
let browser = collection::Browser::new(db.clone(), vfs_manager.clone()); let scanner = scanner::Scanner::new(
let updater = collection::Updater::new(
index_manager.clone(), index_manager.clone(),
settings_manager.clone(), settings_manager.clone(),
vfs_manager.clone(), vfs_manager.clone(),
@ -83,7 +82,7 @@ impl App {
); );
let playlist_manager = playlist::Manager::new(db.clone(), vfs_manager.clone()); let playlist_manager = playlist::Manager::new(db.clone(), vfs_manager.clone());
let thumbnail_manager = thumbnail::Manager::new(thumbnails_dir_path); let thumbnail_manager = thumbnail::Manager::new(thumbnails_dir_path);
let lastfm_manager = lastfm::Manager::new(browser.clone(), user_manager.clone()); let lastfm_manager = lastfm::Manager::new(index_manager.clone(), user_manager.clone());
if let Some(config_path) = paths.config_file_path { if let Some(config_path) = paths.config_file_path {
let config = config::Config::from_path(&config_path)?; let config = config::Config::from_path(&config_path)?;
@ -94,8 +93,7 @@ impl App {
port, port,
web_dir_path: paths.web_dir_path, web_dir_path: paths.web_dir_path,
swagger_dir_path: paths.swagger_dir_path, swagger_dir_path: paths.swagger_dir_path,
updater, scanner,
browser,
index_manager, index_manager,
config_manager, config_manager,
ddns_manager, ddns_manager,

11
src/app/collection.rs
View file

@ -1,11 +0,0 @@
mod browser;
mod index;
mod scanner;
mod types;
mod updater;
pub use browser::*;
pub use index::*;
pub use scanner::*;
pub use types::*;
pub use updater::*;

152
src/app/collection/browser.rs
View file

@ -1,152 +0,0 @@
use std::path::Path;
use crate::app::{collection, vfs};
use crate::db::DB;
#[derive(Clone)]
pub struct Browser {
db: DB,
vfs_manager: vfs::Manager,
}
impl Browser {
pub fn new(db: DB, vfs_manager: vfs::Manager) -> Self {
Self { db, vfs_manager }
}
pub async fn search(&self, query: &str) -> Result<Vec<collection::File>, collection::Error> {
todo!();
}
pub async fn get_song(&self, path: &Path) -> Result<collection::Song, collection::Error> {
todo!();
}
}
#[cfg(test)]
mod test {
use std::path::{Path, PathBuf};
use super::*;
use crate::app::test;
use crate::test_name;
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn can_browse_top_level() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let root_path = Path::new(TEST_MOUNT_NAME);
let files = ctx.browser.browse(Path::new("")).await.unwrap();
assert_eq!(files.len(), 1);
match files[0] {
collection::File::Directory(ref d) => {
assert_eq!(d, &root_path)
}
_ => panic!("Expected directory"),
}
}
#[tokio::test]
async fn can_browse_directory() {
let khemmis_path: PathBuf = [TEST_MOUNT_NAME, "Khemmis"].iter().collect();
let tobokegao_path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect();
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let files = ctx
.browser
.browse(Path::new(TEST_MOUNT_NAME))
.await
.unwrap();
assert_eq!(files.len(), 2);
match files[0] {
collection::File::Directory(ref d) => {
assert_eq!(d, &khemmis_path)
}
_ => panic!("Expected directory"),
}
match files[1] {
collection::File::Directory(ref d) => {
assert_eq!(d, &tobokegao_path)
}
_ => panic!("Expected directory"),
}
}
#[tokio::test]
async fn can_flatten_root() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let songs = ctx
.browser
.flatten(Path::new(TEST_MOUNT_NAME))
.await
.unwrap();
assert_eq!(songs.len(), 13);
assert_eq!(songs[0].title, Some("Above The Water".to_owned()));
}
#[tokio::test]
async fn can_flatten_directory() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect();
let songs = ctx.browser.flatten(path).await.unwrap();
assert_eq!(songs.len(), 8);
}
#[tokio::test]
async fn can_flatten_directory_with_shared_prefix() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); // Prefix of '(Picnic Remixes)'
let songs = ctx.browser.flatten(path).await.unwrap();
assert_eq!(songs.len(), 7);
}
#[tokio::test]
async fn can_get_a_song() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
let song_virtual_path = picnic_virtual_dir.join("05 - シャーベット (Sherbet).mp3");
let artwork_virtual_path = picnic_virtual_dir.join("Folder.png");
let song = ctx.browser.get_song(&song_virtual_path).await.unwrap();
assert_eq!(song.virtual_path, song_virtual_path);
assert_eq!(song.track_number, Some(5));
assert_eq!(song.disc_number, None);
assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned()));
assert_eq!(song.artists, vec!["Tobokegao".to_owned()]);
assert_eq!(song.album_artists, Vec::<String>::new());
assert_eq!(song.album, Some("Picnic".to_owned()));
assert_eq!(song.year, Some(2016));
assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}

556
src/app/collection/index.rs
View file

@ -1,556 +0,0 @@
use std::{
borrow::BorrowMut,
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::{Path, PathBuf},
sync::{Arc, RwLock},
};
use log::{error, info};
use rand::{rngs::ThreadRng, seq::IteratorRandom};
use serde::{Deserialize, Serialize};
use tokio::task::spawn_blocking;
use trie_rs::{Trie, TrieBuilder};
use crate::{app::collection, db::DB};
#[derive(Clone)]
pub struct IndexManager {
db: DB,
index: Arc<RwLock<Index>>, // Not a tokio RwLock as we want to do CPU-bound work with Index
}
impl IndexManager {
pub async fn new(db: DB) -> Self {
let mut index_manager = Self {
db,
index: Arc::new(RwLock::new(Index::new())),
};
if let Err(e) = index_manager.try_restore_index().await {
error!("Failed to restore index: {}", e);
}
index_manager
}
pub(super) async fn replace_index(&mut self, new_index: Index) {
spawn_blocking({
let index_manager = self.clone();
move || {
let mut lock = index_manager.index.write().unwrap();
*lock = new_index;
}
})
.await
.unwrap()
}
pub(super) async fn persist_index(&mut self, index: &Index) -> Result<(), collection::Error> {
let serialized = match bitcode::serialize(index) {
Ok(s) => s,
Err(_) => return Err(collection::Error::IndexSerializationError),
};
sqlx::query!("UPDATE collection_index SET content = $1", serialized)
.execute(self.db.connect().await?.as_mut())
.await?;
Ok(())
}
async fn try_restore_index(&mut self) -> Result<bool, collection::Error> {
let serialized = sqlx::query_scalar!("SELECT content FROM collection_index")
.fetch_one(self.db.connect().await?.as_mut())
.await?;
let Some(serialized) = serialized else {
info!("Database did not contain a collection to restore");
return Ok(false);
};
let index = match bitcode::deserialize(&serialized[..]) {
Ok(i) => i,
Err(_) => return Err(collection::Error::IndexDeserializationError),
};
self.replace_index(index).await;
Ok(true)
}
pub async fn browse(
&self,
virtual_path: PathBuf,
) -> Result<Vec<collection::File>, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
index.browse(virtual_path)
}
})
.await
.unwrap()
}
pub async fn flatten(&self, virtual_path: PathBuf) -> Result<Vec<SongKey>, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
index.flatten(virtual_path)
}
})
.await
.unwrap()
}
pub async fn get_artist(
&self,
artist_key: &ArtistKey,
) -> Result<collection::Artist, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
let artist_id = artist_key.into();
move || {
let index = index_manager.index.read().unwrap();
index
.get_artist(artist_id)
.ok_or_else(|| collection::Error::ArtistNotFound)
}
})
.await
.unwrap()
}
pub async fn get_album(
&self,
album_key: &AlbumKey,
) -> Result<collection::Album, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
let album_id = album_key.into();
move || {
let index = index_manager.index.read().unwrap();
index
.get_album(album_id)
.ok_or_else(|| collection::Error::AlbumNotFound)
}
})
.await
.unwrap()
}
pub async fn get_random_albums(
&self,
count: usize,
) -> Result<Vec<collection::Album>, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
Ok(index
.albums
.keys()
.choose_multiple(&mut ThreadRng::default(), count)
.into_iter()
.filter_map(|k| index.get_album(*k))
.collect())
}
})
.await
.unwrap()
}
pub async fn get_recent_albums(
&self,
count: usize,
) -> Result<Vec<collection::Album>, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
Ok(index
.recent_albums
.iter()
.take(count)
.filter_map(|k| index.get_album(*k))
.collect())
}
})
.await
.unwrap()
}
}
#[derive(Default)]
pub(super) struct IndexBuilder {
directories: HashMap<PathBuf, HashSet<collection::File>>,
flattened: TrieBuilder<String>,
songs: HashMap<SongID, collection::Song>,
artists: HashMap<ArtistID, Artist>,
albums: HashMap<AlbumID, Album>,
}
impl IndexBuilder {
pub fn add_directory(&mut self, directory: collection::Directory) {
self.directories
.entry(directory.virtual_path.clone())
.or_default();
if let Some(parent) = directory.virtual_parent {
self.directories
.entry(parent.clone())
.or_default()
.insert(collection::File::Directory(directory.virtual_path));
}
}
pub fn add_song(&mut self, song: collection::Song) {
let song_id: SongID = song.song_id();
self.flattened.push(
song.virtual_path
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
);
self.directories
.entry(song.virtual_parent.clone())
.or_default()
.insert(collection::File::Song(song.virtual_path.clone()));
self.add_song_to_album(&song);
self.add_album_to_artists(&song);
self.songs.insert(song_id, song);
}
fn add_album_to_artists(&mut self, song: &collection::Song) {
let album_id: AlbumID = song.album_id();
for artist_name in &song.album_artists {
let artist = self.get_or_create_artist(artist_name);
artist.albums.insert(album_id);
}
for artist_name in &song.artists {
let artist = self.get_or_create_artist(artist_name);
if song.album_artists.is_empty() {
artist.albums.insert(album_id);
} else if !song.album_artists.contains(artist_name) {
artist.album_appearances.insert(album_id);
}
}
}
fn get_or_create_artist(&mut self, name: &String) -> &mut Artist {
let artist_key = ArtistKey {
name: Some(name.clone()),
};
let artist_id: ArtistID = (&artist_key).into();
self.artists
.entry(artist_id)
.or_insert_with(|| Artist {
name: Some(name.clone()),
albums: HashSet::new(),
album_appearances: HashSet::new(),
})
.borrow_mut()
}
fn add_song_to_album(&mut self, song: &collection::Song) {
let song_id: SongID = song.song_id();
let album_id: AlbumID = song.album_id();
let album = self.albums.entry(album_id).or_default().borrow_mut();
if album.name.is_none() {
album.name = song.album.clone();
}
if album.artwork.is_none() {
album.artwork = song.artwork.clone();
}
if album.year.is_none() {
album.year = song.year.clone();
}
album.date_added = album.date_added.min(song.date_added);
if !song.album_artists.is_empty() {
album.artists = song.album_artists.clone();
} else if !song.artists.is_empty() {
album.artists = song.artists.clone();
}
album.songs.insert(song_id);
}
pub fn build(self) -> Index {
let mut recent_albums = self.albums.keys().cloned().collect::<Vec<_>>();
recent_albums.sort_by_key(|a| {
self.albums
.get(a)
.map(|a| -a.date_added)
.unwrap_or_default()
});
Index {
directories: self.directories,
flattened: self.flattened.build(),
songs: self.songs,
artists: self.artists,
albums: self.albums,
recent_albums,
}
}
}
#[derive(Serialize, Deserialize)]
pub(super) struct Index {
directories: HashMap<PathBuf, HashSet<collection::File>>,
flattened: Trie<String>,
songs: HashMap<SongID, collection::Song>,
artists: HashMap<ArtistID, Artist>,
albums: HashMap<AlbumID, Album>,
recent_albums: Vec<AlbumID>,
}
impl Index {
pub fn new() -> Self {
Self {
directories: HashMap::new(),
flattened: TrieBuilder::new().build(),
songs: HashMap::new(),
artists: HashMap::new(),
albums: HashMap::new(),
recent_albums: Vec::new(),
}
}
pub(self) fn browse<P: AsRef<Path>>(
&self,
virtual_path: P,
) -> Result<Vec<collection::File>, collection::Error> {
let Some(files) = self.directories.get(virtual_path.as_ref()) else {
return Err(collection::Error::DirectoryNotFound(
virtual_path.as_ref().to_owned(),
));
};
Ok(files.iter().cloned().collect())
}
pub(self) fn flatten<P: AsRef<Path>>(
&self,
virtual_path: P,
) -> Result<Vec<SongKey>, collection::Error> {
let path_components = virtual_path
.as_ref()
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<String>>();
if !self.flattened.is_prefix(&path_components) {
return Err(collection::Error::DirectoryNotFound(
virtual_path.as_ref().to_owned(),
));
}
Ok(self
.flattened
.predictive_search(path_components)
.map(|c: Vec<String>| -> PathBuf { c.join(std::path::MAIN_SEPARATOR_STR).into() })
.map(|s| SongKey { virtual_path: s })
.collect::<Vec<_>>())
}
pub(self) fn get_artist(&self, artist_id: ArtistID) -> Option<collection::Artist> {
self.artists.get(&artist_id).map(|a| {
let albums = {
let mut albums = a
.albums
.iter()
.filter_map(|album_id| self.get_album(*album_id))
.collect::<Vec<_>>();
albums.sort_by(|a, b| (a.year, &a.name).partial_cmp(&(b.year, &b.name)).unwrap());
albums
};
let album_appearances = {
let mut album_appearances = a
.album_appearances
.iter()
.filter_map(|album_id| self.get_album(*album_id))
.collect::<Vec<_>>();
album_appearances.sort_by(|a, b| {
(&a.artists, a.year, &a.name)
.partial_cmp(&(&b.artists, b.year, &b.name))
.unwrap()
});
album_appearances
};
collection::Artist {
name: a.name.clone(),
albums,
album_appearances,
}
})
}
pub(self) fn get_album(&self, album_id: AlbumID) -> Option<collection::Album> {
self.albums.get(&album_id).map(|a| {
let mut songs = a
.songs
.iter()
.filter_map(|s| self.songs.get(s))
.cloned()
.collect::<Vec<_>>();
songs.sort_by_key(|s| (s.disc_number.unwrap_or(-1), s.track_number.unwrap_or(-1)));
collection::Album {
name: a.name.clone(),
artwork: a.artwork.clone(),
artists: a.artists.clone(),
year: a.year,
date_added: a.date_added,
songs,
}
})
}
}
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
struct SongID(u64);
#[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct SongKey {
pub virtual_path: PathBuf,
}
impl From<&collection::Song> for SongKey {
fn from(song: &collection::Song) -> Self {
SongKey {
virtual_path: song.virtual_path.clone(),
}
}
}
impl From<&SongKey> for SongID {
fn from(key: &SongKey) -> Self {
SongID(key.id())
}
}
impl collection::Song {
pub(self) fn song_id(&self) -> SongID {
let key: SongKey = self.into();
(&key).into()
}
}
#[derive(Serialize, Deserialize)]
struct Artist {
pub name: Option<String>,
pub albums: HashSet<AlbumID>,
pub album_appearances: HashSet<AlbumID>,
}
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
struct ArtistID(u64);
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct ArtistKey {
pub name: Option<String>,
}
impl From<&ArtistKey> for ArtistID {
fn from(key: &ArtistKey) -> Self {
ArtistID(key.id())
}
}
#[derive(Clone, Default, Serialize, Deserialize)]
struct Album {
pub name: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,
pub songs: HashSet<SongID>,
}
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
struct AlbumID(u64);
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct AlbumKey {
pub artists: Vec<String>,
pub name: Option<String>,
}
impl From<&collection::Song> for AlbumKey {
fn from(song: &collection::Song) -> Self {
let album_artists = match song.album_artists.is_empty() {
true => &song.artists,
false => &song.album_artists,
};
AlbumKey {
artists: album_artists.iter().cloned().collect(),
name: song.album.clone(),
}
}
}
impl From<&AlbumKey> for AlbumID {
fn from(key: &AlbumKey) -> Self {
AlbumID(key.id())
}
}
impl collection::Song {
pub(self) fn album_id(&self) -> AlbumID {
let key: AlbumKey = self.into();
(&key).into()
}
}
trait ID {
fn id(&self) -> u64;
}
impl<T: Hash> ID for T {
fn id(&self) -> u64 {
let mut hasher = DefaultHasher::default();
self.hash(&mut hasher);
hasher.finish()
}
}
#[cfg(test)]
mod test {
use crate::app::test;
use crate::test_name;
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn can_get_random_albums() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let albums = ctx.index_manager.get_random_albums(1).await.unwrap();
assert_eq!(albums.len(), 1);
}
#[tokio::test]
async fn can_get_recent_albums() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let albums = ctx.index_manager.get_recent_albums(2).await.unwrap();
assert_eq!(albums.len(), 2);
}
}

180
src/app/collection/scanner.rs
View file

@ -1,180 +0,0 @@
use log::{error, info};
use rayon::{Scope, ThreadPoolBuilder};
use regex::Regex;
use std::cmp::min;
use std::fs;
use std::path::Path;
use std::str::FromStr;
use tokio::sync::mpsc::UnboundedSender;
use crate::app::vfs;
use crate::app::{
collection::{self},
formats,
};
pub struct Scanner {
directories_output: UnboundedSender<collection::Directory>,
songs_output: UnboundedSender<collection::Song>,
vfs_manager: vfs::Manager,
artwork_regex: Option<Regex>,
}
impl Scanner {
pub fn new(
directories_output: UnboundedSender<collection::Directory>,
songs_output: UnboundedSender<collection::Song>,
vfs_manager: vfs::Manager,
artwork_regex: Option<Regex>,
) -> Self {
Self {
directories_output,
songs_output,
vfs_manager,
artwork_regex,
}
}
pub async fn scan(self) -> Result<(), collection::Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let roots = vfs.mounts().clone();
let key = "POLARIS_NUM_TRAVERSER_THREADS";
let num_threads = std::env::var_os(key)
.map(|v| v.to_string_lossy().to_string())
.and_then(|v| usize::from_str(&v).ok())
.unwrap_or_else(|| min(num_cpus::get(), 8));
info!("Browsing collection using {} threads", num_threads);
let directories_output = self.directories_output.clone();
let songs_output = self.songs_output.clone();
let artwork_regex = self.artwork_regex.clone();
let thread_pool = ThreadPoolBuilder::new().num_threads(num_threads).build()?;
thread_pool.scope({
|scope| {
for root in roots {
scope.spawn(|scope| {
process_directory(
scope,
root.source,
root.name,
directories_output.clone(),
songs_output.clone(),
artwork_regex.clone(),
);
});
}
}
});
Ok(())
}
}
fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
scope: &Scope,
real_path: P,
virtual_path: Q,
directories_output: UnboundedSender<collection::Directory>,
songs_output: UnboundedSender<collection::Song>,
artwork_regex: Option<Regex>,
) {
let read_dir = match fs::read_dir(&real_path) {
Ok(read_dir) => read_dir,
Err(e) => {
error!(
"Directory read error for `{}`: {}",
real_path.as_ref().display(),
e
);
return;
}
};
let mut songs = vec![];
let mut artwork_file = None;
for entry in read_dir {
let name = match entry {
Ok(ref f) => f.file_name(),
Err(e) => {
error!(
"File read error within `{}`: {}",
real_path.as_ref().display(),
e
);
break;
}
};
let entry_real_path = real_path.as_ref().join(&name);
let entry_virtual_path = virtual_path.as_ref().join(&name);
if entry_real_path.is_dir() {
scope.spawn({
let directories_output = directories_output.clone();
let songs_output = songs_output.clone();
let artwork_regex = artwork_regex.clone();
|scope| {
process_directory(
scope,
entry_real_path,
entry_virtual_path,
directories_output,
songs_output,
artwork_regex,
);
}
});
} else if let Some(metadata) = formats::read_metadata(&entry_real_path) {
songs.push(collection::Song {
path: entry_real_path.clone(),
virtual_path: entry_virtual_path.clone(),
virtual_parent: entry_virtual_path.parent().unwrap().to_owned(),
track_number: metadata.track_number.map(|n| n as i64),
disc_number: metadata.disc_number.map(|n| n as i64),
title: metadata.title,
artists: metadata.artists,
album_artists: metadata.album_artists,
year: metadata.year.map(|n| n as i64),
album: metadata.album,
artwork: metadata.has_artwork.then(|| entry_virtual_path.clone()),
duration: metadata.duration.map(|n| n as i64),
lyricists: metadata.lyricists,
composers: metadata.composers,
genres: metadata.genres,
labels: metadata.labels,
date_added: get_date_created(&entry_real_path).unwrap_or_default(),
});
} else if artwork_file.is_none()
&& artwork_regex
.as_ref()
.is_some_and(|r| r.is_match(name.to_str().unwrap_or_default()))
{
artwork_file = Some(entry_virtual_path);
}
}
for mut song in songs {
song.artwork = song.artwork.or_else(|| artwork_file.clone());
songs_output.send(song).ok();
}
directories_output
.send(collection::Directory {
virtual_path: virtual_path.as_ref().to_owned(),
virtual_parent: virtual_path.as_ref().parent().map(Path::to_owned),
})
.ok();
}
fn get_date_created<P: AsRef<Path>>(path: P) -> Option<i64> {
if let Ok(t) = fs::metadata(path).and_then(|m| m.created().or_else(|_| m.modified())) {
t.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.ok()
} else {
None
}
}

82
src/app/collection/types.rs
View file

@ -1,82 +0,0 @@
use std::path::PathBuf;
use serde::{Deserialize, Serialize};
use crate::{
app::vfs::{self},
db,
};
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Directory not found: {0}")]
DirectoryNotFound(PathBuf),
#[error("Artist not found")]
ArtistNotFound,
#[error("Album not found")]
AlbumNotFound,
#[error(transparent)]
Database(#[from] sqlx::Error),
#[error(transparent)]
DatabaseConnection(#[from] db::Error),
#[error(transparent)]
Vfs(#[from] vfs::Error),
#[error("Could not deserialize collection")]
IndexDeserializationError,
#[error("Could not serialize collection")]
IndexSerializationError,
#[error(transparent)]
ThreadPoolBuilder(#[from] rayon::ThreadPoolBuildError),
#[error(transparent)]
ThreadJoining(#[from] tokio::task::JoinError),
}
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum File {
Directory(PathBuf),
Song(PathBuf),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Song {
pub path: PathBuf,
pub virtual_path: PathBuf,
pub virtual_parent: PathBuf,
pub track_number: Option<i64>,
pub disc_number: Option<i64>,
pub title: Option<String>,
pub artists: Vec<String>,
pub album_artists: Vec<String>,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<PathBuf>,
pub duration: Option<i64>,
pub lyricists: Vec<String>,
pub composers: Vec<String>,
pub genres: Vec<String>,
pub labels: Vec<String>,
pub date_added: i64,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Directory {
pub virtual_path: PathBuf,
pub virtual_parent: Option<PathBuf>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Artist {
pub name: Option<String>,
pub albums: Vec<Album>,
pub album_appearances: Vec<Album>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Album {
pub name: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,
pub songs: Vec<Song>,
}

214
src/app/collection/updater.rs
View file

@ -1,214 +0,0 @@
use std::{sync::Arc, time::Duration};
use log::{error, info};
use tokio::{
sync::{mpsc::unbounded_channel, Notify},
time::Instant,
};
use crate::app::{collection::*, settings, vfs};
#[derive(Clone)]
pub struct Updater {
index_manager: IndexManager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
pending_scan: Arc<Notify>,
}
impl Updater {
pub async fn new(
index_manager: IndexManager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
) -> Result<Self, Error> {
let updater = Self {
index_manager,
vfs_manager,
settings_manager,
pending_scan: Arc::new(Notify::new()),
};
tokio::spawn({
let mut updater = updater.clone();
async move {
loop {
updater.pending_scan.notified().await;
if let Err(e) = updater.update().await {
error!("Error while updating index: {}", e);
}
}
}
});
Ok(updater)
}
pub fn trigger_scan(&self) {
self.pending_scan.notify_one();
}
pub fn begin_periodic_scans(&self) {
tokio::spawn({
let index = self.clone();
async move {
loop {
index.trigger_scan();
let sleep_duration = index
.settings_manager
.get_index_sleep_duration()
.await
.unwrap_or_else(|e| {
error!("Could not retrieve index sleep duration: {}", e);
Duration::from_secs(1800)
});
tokio::time::sleep(sleep_duration).await;
}
}
});
}
pub async fn update(&mut self) -> Result<(), Error> {
let start = Instant::now();
info!("Beginning collection scan");
let album_art_pattern = self
.settings_manager
.get_index_album_art_pattern()
.await
.ok();
let (scanner_directories_output, mut collection_directories_input) = unbounded_channel();
let (scanner_songs_output, mut collection_songs_input) = unbounded_channel();
let scanner = Scanner::new(
scanner_directories_output,
scanner_songs_output,
self.vfs_manager.clone(),
album_art_pattern,
);
let index_task = tokio::spawn(async move {
let capacity = 500;
let mut index_builder = IndexBuilder::default();
let mut song_buffer: Vec<Song> = Vec::with_capacity(capacity);
let mut directory_buffer: Vec<Directory> = Vec::with_capacity(capacity);
loop {
let exhausted_songs = match collection_songs_input
.recv_many(&mut song_buffer, capacity)
.await
{
0 => true,
_ => {
for song in song_buffer.drain(0..) {
index_builder.add_song(song);
}
false
}
};
let exhausted_directories = match collection_directories_input
.recv_many(&mut directory_buffer, capacity)
.await
{
0 => true,
_ => {
for directory in directory_buffer.drain(0..) {
index_builder.add_directory(directory);
}
false
}
};
if exhausted_directories && exhausted_songs {
break;
}
}
index_builder.build()
});
let index = tokio::join!(scanner.scan(), index_task).1?;
self.index_manager.persist_index(&index).await?;
self.index_manager.replace_index(index).await;
info!(
"Collection scan took {} seconds",
start.elapsed().as_millis() as f32 / 1000.0
);
Ok(())
}
}
#[cfg(test)]
mod test {
use std::path::PathBuf;
use crate::{
app::{settings, test},
test_name,
};
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn scan_adds_new_content() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
ctx.updater.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts
todo!();
// assert_eq!(all_directories.len(), 6);
// assert_eq!(all_songs.len(), 13);
}
#[tokio::test]
async fn finds_embedded_artwork() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.updater.update().await.unwrap();
let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3");
let song = ctx.browser.get_song(&song_virtual_path).await.unwrap();
assert_eq!(song.artwork, Some(song_virtual_path));
}
#[tokio::test]
async fn album_art_pattern_is_case_insensitive() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
let patterns = vec!["folder", "FOLDER"];
for pattern in patterns.into_iter() {
ctx.settings_manager
.amend(&settings::NewSettings {
album_art_pattern: Some(pattern.to_owned()),
..Default::default()
})
.await
.unwrap();
ctx.updater.update().await.unwrap();
let hunted_virtual_dir: PathBuf =
[TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect();
let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg");
let song = &ctx.browser.flatten(&hunted_virtual_dir).await.unwrap()[0];
assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}
}

246
src/app/index.rs Normal file
View file

@ -0,0 +1,246 @@
use std::{
path::PathBuf,
sync::{Arc, RwLock},
};
use log::{error, info};
use serde::{Deserialize, Serialize};
use tokio::task::spawn_blocking;
use crate::app::scanner;
use crate::app::vfs;
use crate::db::{self, DB};
mod browser;
mod collection;
mod search;
pub use browser::File;
pub use collection::{Album, AlbumKey, Artist, ArtistKey, Song, SongKey};
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Directory not found: {0}")]
DirectoryNotFound(PathBuf),
#[error("Artist not found")]
ArtistNotFound,
#[error("Album not found")]
AlbumNotFound,
#[error("Song not found")]
SongNotFound,
#[error(transparent)]
Database(#[from] sqlx::Error),
#[error(transparent)]
DatabaseConnection(#[from] db::Error),
#[error(transparent)]
Vfs(#[from] vfs::Error),
#[error("Could not deserialize collection")]
IndexDeserializationError,
#[error("Could not serialize collection")]
IndexSerializationError,
#[error(transparent)]
ThreadPoolBuilder(#[from] rayon::ThreadPoolBuildError),
#[error(transparent)]
ThreadJoining(#[from] tokio::task::JoinError),
}
#[derive(Clone)]
pub struct Manager {
db: DB,
index: Arc<RwLock<Index>>, // Not a tokio RwLock as we want to do CPU-bound work with Index
}
impl Manager {
pub async fn new(db: DB) -> Self {
let mut index_manager = Self {
db,
index: Arc::default(),
};
if let Err(e) = index_manager.try_restore_index().await {
error!("Failed to restore index: {}", e);
}
index_manager
}
pub async fn replace_index(&mut self, new_index: Index) {
spawn_blocking({
let index_manager = self.clone();
move || {
let mut lock = index_manager.index.write().unwrap();
*lock = new_index;
}
})
.await
.unwrap()
}
pub async fn persist_index(&mut self, index: &Index) -> Result<(), Error> {
let serialized = match bitcode::serialize(index) {
Ok(s) => s,
Err(_) => return Err(Error::IndexSerializationError),
};
sqlx::query!("UPDATE collection_index SET content = $1", serialized)
.execute(self.db.connect().await?.as_mut())
.await?;
Ok(())
}
async fn try_restore_index(&mut self) -> Result<bool, Error> {
let serialized = sqlx::query_scalar!("SELECT content FROM collection_index")
.fetch_one(self.db.connect().await?.as_mut())
.await?;
let Some(serialized) = serialized else {
info!("Database did not contain a collection to restore");
return Ok(false);
};
let index = match bitcode::deserialize(&serialized[..]) {
Ok(i) => i,
Err(_) => return Err(Error::IndexDeserializationError),
};
self.replace_index(index).await;
Ok(true)
}
pub async fn browse(&self, virtual_path: PathBuf) -> Result<Vec<browser::File>, Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
index.browser.browse(virtual_path)
}
})
.await
.unwrap()
}
pub async fn flatten(&self, virtual_path: PathBuf) -> Result<Vec<PathBuf>, Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
index.browser.flatten(virtual_path)
}
})
.await
.unwrap()
}
pub async fn get_artist(&self, artist_key: &ArtistKey) -> Result<Artist, Error> {
spawn_blocking({
let index_manager = self.clone();
let artist_id = artist_key.into();
move || {
let index = index_manager.index.read().unwrap();
index
.collection
.get_artist(artist_id)
.ok_or_else(|| Error::ArtistNotFound)
}
})
.await
.unwrap()
}
pub async fn get_album(&self, album_key: &AlbumKey) -> Result<Album, Error> {
spawn_blocking({
let index_manager = self.clone();
let album_id = album_key.into();
move || {
let index = index_manager.index.read().unwrap();
index
.collection
.get_album(album_id)
.ok_or_else(|| Error::AlbumNotFound)
}
})
.await
.unwrap()
}
pub async fn get_random_albums(&self, count: usize) -> Result<Vec<Album>, Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
Ok(index.collection.get_random_albums(count))
}
})
.await
.unwrap()
}
pub async fn get_recent_albums(&self, count: usize) -> Result<Vec<Album>, Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
Ok(index.collection.get_recent_albums(count))
}
})
.await
.unwrap()
}
pub async fn get_song(&self, song_key: &SongKey) -> Result<Song, Error> {
spawn_blocking({
let index_manager = self.clone();
let song_id = song_key.into();
move || {
let index = index_manager.index.read().unwrap();
index
.collection
.get_song(song_id)
.ok_or_else(|| Error::SongNotFound)
}
})
.await
.unwrap()
}
pub async fn search(&self, _query: &str) -> Result<Vec<PathBuf>, Error> {
todo!();
}
}
#[derive(Serialize, Deserialize)]
pub struct Index {
pub browser: browser::Browser,
pub collection: collection::Collection,
}
impl Default for Index {
fn default() -> Self {
Self {
browser: browser::Browser::new(),
collection: Default::default(),
}
}
}
#[derive(Default)]
pub struct Builder {
browser_builder: browser::Builder,
collection_builder: collection::Builder,
}
impl Builder {
pub fn add_directory(&mut self, directory: scanner::Directory) {
self.browser_builder.add_directory(directory);
}
pub fn add_song(&mut self, song: scanner::Song) {
self.browser_builder.add_song(&song);
self.collection_builder.add_song(song);
}
pub fn build(self) -> Index {
Index {
browser: self.browser_builder.build(),
collection: self.collection_builder.build(),
}
}
}

209
src/app/index/browser.rs Normal file
View file

@ -0,0 +1,209 @@
use std::{
collections::{HashMap, HashSet},
hash::Hash,
path::{Path, PathBuf},
};
use serde::{Deserialize, Serialize};
use trie_rs::{Trie, TrieBuilder};
use crate::app::{index::Error, scanner};
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum File {
Directory(PathBuf),
Song(PathBuf),
}
#[derive(Serialize, Deserialize)]
pub struct Browser {
directories: HashMap<PathBuf, HashSet<File>>,
flattened: Trie<String>,
}
impl Browser {
pub fn new() -> Self {
Self {
directories: HashMap::new(),
flattened: TrieBuilder::new().build(),
}
}
pub fn browse<P: AsRef<Path>>(&self, virtual_path: P) -> Result<Vec<File>, Error> {
let Some(files) = self.directories.get(virtual_path.as_ref()) else {
return Err(Error::DirectoryNotFound(virtual_path.as_ref().to_owned()));
};
Ok(files.iter().cloned().collect())
}
pub fn flatten<P: AsRef<Path>>(&self, virtual_path: P) -> Result<Vec<PathBuf>, Error> {
let path_components = virtual_path
.as_ref()
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<String>>();
if !self.flattened.is_prefix(&path_components) {
return Err(Error::DirectoryNotFound(virtual_path.as_ref().to_owned()));
}
Ok(self
.flattened
.predictive_search(path_components)
.map(|c: Vec<String>| -> PathBuf { c.join(std::path::MAIN_SEPARATOR_STR).into() })
.collect::<Vec<_>>())
}
}
pub struct Builder {
directories: HashMap<PathBuf, HashSet<File>>,
flattened: TrieBuilder<String>,
}
impl Default for Builder {
fn default() -> Self {
Self {
directories: Default::default(),
flattened: Default::default(),
}
}
}
impl Builder {
pub fn add_directory(&mut self, directory: scanner::Directory) {
self.directories
.entry(directory.virtual_path.clone())
.or_default();
if let Some(parent) = directory.virtual_parent {
self.directories
.entry(parent.clone())
.or_default()
.insert(File::Directory(directory.virtual_path));
}
}
pub fn add_song(&mut self, song: &scanner::Song) {
self.directories
.entry(song.virtual_parent.clone())
.or_default()
.insert(File::Song(song.virtual_path.clone()));
self.flattened.push(
song.virtual_path
.components()
.map(|c| c.as_os_str().to_string_lossy().to_string())
.collect::<Vec<_>>(),
);
}
pub fn build(self) -> Browser {
Browser {
directories: self.directories,
flattened: self.flattened.build(),
}
}
}
#[cfg(test)]
mod test {
use std::path::{Path, PathBuf};
use super::*;
use crate::app::test;
use crate::test_name;
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn can_browse_top_level() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let root_path = Path::new(TEST_MOUNT_NAME);
let files = ctx.index_manager.browse(PathBuf::new()).await.unwrap();
assert_eq!(files.len(), 1);
match files[0] {
File::Directory(ref d) => {
assert_eq!(d, &root_path)
}
_ => panic!("Expected directory"),
}
}
#[tokio::test]
async fn can_browse_directory() {
let khemmis_path: PathBuf = [TEST_MOUNT_NAME, "Khemmis"].iter().collect();
let tobokegao_path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect();
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let files = ctx
.index_manager
.browse(PathBuf::from(TEST_MOUNT_NAME))
.await
.unwrap();
assert_eq!(files.len(), 2);
match files[0] {
File::Directory(ref d) => {
assert_eq!(d, &khemmis_path)
}
_ => panic!("Expected directory"),
}
match files[1] {
File::Directory(ref d) => {
assert_eq!(d, &tobokegao_path)
}
_ => panic!("Expected directory"),
}
}
#[tokio::test]
async fn can_flatten_root() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let songs = ctx
.index_manager
.flatten(PathBuf::from(TEST_MOUNT_NAME))
.await
.unwrap();
assert_eq!(songs.len(), 13);
assert_eq!(songs[0], Path::new("FIX ME"));
}
#[tokio::test]
async fn can_flatten_directory() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect();
let songs = ctx.index_manager.flatten(path).await.unwrap();
assert_eq!(songs.len(), 8);
}
#[tokio::test]
async fn can_flatten_directory_with_shared_prefix() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); // Prefix of '(Picnic Remixes)'
let songs = ctx.index_manager.flatten(path).await.unwrap();
assert_eq!(songs.len(), 7);
}
}

422
src/app/index/collection.rs Normal file
View file

@ -0,0 +1,422 @@
use std::{
borrow::BorrowMut,
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::PathBuf,
};
use rand::{rngs::ThreadRng, seq::IteratorRandom};
use serde::{Deserialize, Serialize};
use crate::app::scanner;
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Artist {
pub name: Option<String>,
pub albums: Vec<Album>,
pub album_appearances: Vec<Album>,
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Album {
pub name: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,
pub songs: Vec<Song>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Song {
pub path: PathBuf,
pub virtual_path: PathBuf,
pub virtual_parent: PathBuf,
pub track_number: Option<i64>,
pub disc_number: Option<i64>,
pub title: Option<String>,
pub artists: Vec<String>,
pub album_artists: Vec<String>,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<PathBuf>,
pub duration: Option<i64>,
pub lyricists: Vec<String>,
pub composers: Vec<String>,
pub genres: Vec<String>,
pub labels: Vec<String>,
pub date_added: i64,
}
#[derive(Default, Serialize, Deserialize)]
pub struct Collection {
artists: HashMap<ArtistID, storage::Artist>,
albums: HashMap<AlbumID, storage::Album>,
songs: HashMap<SongID, Song>,
recent_albums: Vec<AlbumID>,
}
impl Collection {
pub fn get_artist(&self, artist_id: ArtistID) -> Option<Artist> {
self.artists.get(&artist_id).map(|a| {
let albums = {
let mut albums = a
.albums
.iter()
.filter_map(|album_id| self.get_album(*album_id))
.collect::<Vec<_>>();
albums.sort_by(|a, b| (a.year, &a.name).partial_cmp(&(b.year, &b.name)).unwrap());
albums
};
let album_appearances = {
let mut album_appearances = a
.album_appearances
.iter()
.filter_map(|album_id| self.get_album(*album_id))
.collect::<Vec<_>>();
album_appearances.sort_by(|a, b| {
(&a.artists, a.year, &a.name)
.partial_cmp(&(&b.artists, b.year, &b.name))
.unwrap()
});
album_appearances
};
Artist {
name: a.name.clone(),
albums,
album_appearances,
}
})
}
pub fn get_album(&self, album_id: AlbumID) -> Option<Album> {
self.albums.get(&album_id).map(|a| {
let mut songs = a
.songs
.iter()
.filter_map(|s| self.songs.get(s))
.cloned()
.collect::<Vec<_>>();
songs.sort_by_key(|s| (s.disc_number.unwrap_or(-1), s.track_number.unwrap_or(-1)));
Album {
name: a.name.clone(),
artwork: a.artwork.clone(),
artists: a.artists.clone(),
year: a.year,
date_added: a.date_added,
songs,
}
})
}
pub fn get_random_albums(&self, count: usize) -> Vec<Album> {
self.albums
.keys()
.choose_multiple(&mut ThreadRng::default(), count)
.into_iter()
.filter_map(|k| self.get_album(*k))
.collect()
}
pub fn get_recent_albums(&self, count: usize) -> Vec<Album> {
self.recent_albums
.iter()
.take(count)
.filter_map(|k| self.get_album(*k))
.collect()
}
pub fn get_song(&self, song_id: SongID) -> Option<Song> {
self.songs.get(&song_id).cloned()
}
}
#[derive(Default)]
pub struct Builder {
artists: HashMap<ArtistID, storage::Artist>,
albums: HashMap<AlbumID, storage::Album>,
songs: HashMap<SongID, Song>,
}
impl Builder {
pub fn add_song(&mut self, song: scanner::Song) {
let song = Song {
path: song.path,
virtual_path: song.virtual_path,
virtual_parent: song.virtual_parent,
track_number: song.track_number,
disc_number: song.disc_number,
title: song.title,
artists: song.artists,
album_artists: song.album_artists,
year: song.year,
album: song.album,
artwork: song.artwork,
duration: song.duration,
lyricists: song.lyricists,
composers: song.composers,
genres: song.genres,
labels: song.labels,
date_added: song.date_added,
};
let song_id: SongID = song.song_id();
self.add_song_to_album(&song);
self.add_album_to_artists(&song);
self.songs.insert(song_id, song);
}
pub fn build(self) -> Collection {
let mut recent_albums = self.albums.keys().cloned().collect::<Vec<_>>();
recent_albums.sort_by_key(|a| {
self.albums
.get(a)
.map(|a| -a.date_added)
.unwrap_or_default()
});
Collection {
artists: self.artists,
albums: self.albums,
songs: self.songs,
recent_albums,
}
}
fn add_album_to_artists(&mut self, song: &Song) {
let album_id: AlbumID = song.album_id();
for artist_name in &song.album_artists {
let artist = self.get_or_create_artist(artist_name);
artist.albums.insert(album_id);
}
for artist_name in &song.artists {
let artist = self.get_or_create_artist(artist_name);
if song.album_artists.is_empty() {
artist.albums.insert(album_id);
} else if !song.album_artists.contains(artist_name) {
artist.album_appearances.insert(album_id);
}
}
}
fn get_or_create_artist(&mut self, name: &String) -> &mut storage::Artist {
let artist_key = ArtistKey {
name: Some(name.clone()),
};
let artist_id: ArtistID = (&artist_key).into();
self.artists
.entry(artist_id)
.or_insert_with(|| storage::Artist {
name: Some(name.clone()),
albums: HashSet::new(),
album_appearances: HashSet::new(),
})
.borrow_mut()
}
fn add_song_to_album(&mut self, song: &Song) {
let song_id: SongID = song.song_id();
let album_id: AlbumID = song.album_id();
let album = self.albums.entry(album_id).or_default().borrow_mut();
if album.name.is_none() {
album.name = song.album.clone();
}
if album.artwork.is_none() {
album.artwork = song.artwork.clone();
}
if album.year.is_none() {
album.year = song.year.clone();
}
album.date_added = album.date_added.min(song.date_added);
if !song.album_artists.is_empty() {
album.artists = song.album_artists.clone();
} else if !song.artists.is_empty() {
album.artists = song.artists.clone();
}
album.songs.insert(song_id);
}
}
mod storage {
use super::*;
#[derive(Serialize, Deserialize)]
pub struct Artist {
pub name: Option<String>,
pub albums: HashSet<AlbumID>,
pub album_appearances: HashSet<AlbumID>,
}
#[derive(Clone, Default, Serialize, Deserialize)]
pub struct Album {
pub name: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,
pub songs: HashSet<SongID>,
}
}
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct ArtistID(u64);
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct AlbumID(u64);
#[derive(Clone, Copy, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct SongID(u64);
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct ArtistKey {
pub name: Option<String>,
}
#[derive(Clone, Eq, Hash, PartialEq)]
pub struct AlbumKey {
pub artists: Vec<String>,
pub name: Option<String>,
}
#[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct SongKey {
pub virtual_path: PathBuf,
}
impl Song {
pub fn album_key(&self) -> AlbumKey {
let album_artists = match self.album_artists.is_empty() {
true => &self.artists,
false => &self.album_artists,
};
AlbumKey {
artists: album_artists.iter().cloned().collect(),
name: self.album.clone(),
}
}
pub fn album_id(&self) -> AlbumID {
// TODO we .song_key is cloning names just so we can hash them! Slow!
let key: AlbumKey = self.album_key();
(&key).into()
}
pub fn song_key(&self) -> SongKey {
SongKey {
virtual_path: self.virtual_path.clone(),
}
}
pub fn song_id(&self) -> SongID {
// TODO we .song_key is cloning path just so we can hash it! Slow!
let key: SongKey = self.song_key();
(&key).into()
}
}
impl From<&ArtistKey> for ArtistID {
fn from(key: &ArtistKey) -> Self {
ArtistID(key.id())
}
}
impl From<&AlbumKey> for AlbumID {
fn from(key: &AlbumKey) -> Self {
AlbumID(key.id())
}
}
impl From<&SongKey> for SongID {
fn from(key: &SongKey) -> Self {
SongID(key.id())
}
}
trait ID {
fn id(&self) -> u64;
}
impl<T: Hash> ID for T {
fn id(&self) -> u64 {
let mut hasher = DefaultHasher::default();
self.hash(&mut hasher);
hasher.finish()
}
}
#[cfg(test)]
mod test {
use super::*;
use crate::app::test;
use crate::test_name;
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn can_get_random_albums() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let albums = ctx.index_manager.get_random_albums(1).await.unwrap();
assert_eq!(albums.len(), 1);
}
#[tokio::test]
async fn can_get_recent_albums() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let albums = ctx.index_manager.get_recent_albums(2).await.unwrap();
assert_eq!(albums.len(), 2);
}
#[tokio::test]
async fn can_get_a_song() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
let song_virtual_path = picnic_virtual_dir.join("05 - シャーベット (Sherbet).mp3");
let artwork_virtual_path = picnic_virtual_dir.join("Folder.png");
let song = ctx
.index_manager
.get_song(&SongKey {
virtual_path: song_virtual_path.clone(),
})
.await
.unwrap();
assert_eq!(song.virtual_path, song_virtual_path);
assert_eq!(song.track_number, Some(5));
assert_eq!(song.disc_number, None);
assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned()));
assert_eq!(song.artists, vec!["Tobokegao".to_owned()]);
assert_eq!(song.album_artists, Vec::<String>::new());
assert_eq!(song.album, Some("Picnic".to_owned()));
assert_eq!(song.year, Some(2016));
assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}

1
src/app/index/search.rs Normal file
View file

@ -0,0 +1 @@

15
src/app/lastfm.rs
View file

@ -2,7 +2,7 @@ use rustfm_scrobble::{Scrobble, Scrobbler};
use std::path::Path; use std::path::Path;
use user::AuthToken; use user::AuthToken;
use crate::app::{collection, user}; use crate::app::{index, user};
const LASTFM_API_KEY: &str = "02b96c939a2b451c31dfd67add1f696e"; const LASTFM_API_KEY: &str = "02b96c939a2b451c31dfd67add1f696e";
const LASTFM_API_SECRET: &str = "0f25a80ceef4b470b5cb97d99d4b3420"; const LASTFM_API_SECRET: &str = "0f25a80ceef4b470b5cb97d99d4b3420";
@ -16,21 +16,21 @@ pub enum Error {
#[error("Failed to emit last.fm now playing update")] #[error("Failed to emit last.fm now playing update")]
NowPlaying(rustfm_scrobble::ScrobblerError), NowPlaying(rustfm_scrobble::ScrobblerError),
#[error(transparent)] #[error(transparent)]
Query(#[from] collection::Error), Query(#[from] index::Error),
#[error(transparent)] #[error(transparent)]
User(#[from] user::Error), User(#[from] user::Error),
} }
#[derive(Clone)] #[derive(Clone)]
pub struct Manager { pub struct Manager {
browser: collection::Browser, index_manager: index::Manager,
user_manager: user::Manager, user_manager: user::Manager,
} }
impl Manager { impl Manager {
pub fn new(browser: collection::Browser, user_manager: user::Manager) -> Self { pub fn new(index_manager: index::Manager, user_manager: user::Manager) -> Self {
Self { Self {
browser, index_manager,
user_manager, user_manager,
} }
} }
@ -81,7 +81,10 @@ impl Manager {
} }
async fn scrobble_from_path(&self, track: &Path) -> Result<Scrobble, Error> { async fn scrobble_from_path(&self, track: &Path) -> Result<Scrobble, Error> {
let song = self.browser.get_song(track).await?; let song_key = index::SongKey {
virtual_path: track.to_owned(),
};
let song = self.index_manager.get_song(&song_key).await?;
Ok(Scrobble::new( Ok(Scrobble::new(
song.artists.first().map(|s| s.as_str()).unwrap_or(""), song.artists.first().map(|s| s.as_str()).unwrap_or(""),
song.title.as_deref().unwrap_or(""), song.title.as_deref().unwrap_or(""),

19
src/app/playlist.rs
View file

@ -2,7 +2,6 @@ use core::clone::Clone;
use sqlx::{Acquire, QueryBuilder, Sqlite}; use sqlx::{Acquire, QueryBuilder, Sqlite};
use std::path::PathBuf; use std::path::PathBuf;
use crate::app::collection::SongKey;
use crate::app::vfs; use crate::app::vfs;
use crate::db::{self, DB}; use crate::db::{self, DB};
@ -126,7 +125,7 @@ impl Manager {
&self, &self,
playlist_name: &str, playlist_name: &str,
owner: &str, owner: &str,
) -> Result<Vec<SongKey>, Error> { ) -> Result<Vec<PathBuf>, Error> {
let songs = { let songs = {
let mut connection = self.db.connect().await?; let mut connection = self.db.connect().await?;
@ -231,15 +230,14 @@ mod test {
.build() .build()
.await; .await;
ctx.updater.update().await.unwrap(); ctx.scanner.update().await.unwrap();
let playlist_content = ctx let playlist_content = ctx
.browser .index_manager
.flatten(Path::new(TEST_MOUNT_NAME)) .flatten(PathBuf::from(TEST_MOUNT_NAME))
.await .await
.unwrap() .unwrap()
.into_iter() .into_iter()
.map(|s| s.virtual_path)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(playlist_content.len(), 13); assert_eq!(playlist_content.len(), 13);
@ -296,15 +294,14 @@ mod test {
.build() .build()
.await; .await;
ctx.updater.update().await.unwrap(); ctx.scanner.update().await.unwrap();
let playlist_content = ctx let playlist_content = ctx
.browser .index_manager
.flatten(Path::new(TEST_MOUNT_NAME)) .flatten(PathBuf::from(TEST_MOUNT_NAME))
.await .await
.unwrap() .unwrap()
.into_iter() .into_iter()
.map(|s| s.virtual_path)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
assert_eq!(playlist_content.len(), 13); assert_eq!(playlist_content.len(), 13);
@ -329,6 +326,6 @@ mod test {
] ]
.iter() .iter()
.collect(); .collect();
assert_eq!(songs[0].virtual_path, first_song_path); assert_eq!(songs[0], first_song_path);
} }
} }

410
src/app/scanner.rs Normal file
View file

@ -0,0 +1,410 @@
use log::{error, info};
use rayon::{Scope, ThreadPoolBuilder};
use regex::Regex;
use std::fs;
use std::path::{Path, PathBuf};
use std::str::FromStr;
use std::sync::Arc;
use std::{cmp::min, time::Duration};
use tokio::sync::mpsc::error::TryRecvError;
use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
use tokio::sync::Notify;
use tokio::time::Instant;
use crate::app::{formats, index, settings, vfs};
#[derive(Debug, PartialEq, Eq)]
pub struct Directory {
pub virtual_path: PathBuf,
pub virtual_parent: Option<PathBuf>,
}
#[derive(Debug, PartialEq, Eq)]
pub struct Song {
pub path: PathBuf,
pub virtual_path: PathBuf,
pub virtual_parent: PathBuf,
pub track_number: Option<i64>,
pub disc_number: Option<i64>,
pub title: Option<String>,
pub artists: Vec<String>,
pub album_artists: Vec<String>,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<PathBuf>,
pub duration: Option<i64>,
pub lyricists: Vec<String>,
pub composers: Vec<String>,
pub genres: Vec<String>,
pub labels: Vec<String>,
pub date_added: i64,
}
#[derive(Clone)]
pub struct Scanner {
index_manager: index::Manager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
pending_scan: Arc<Notify>,
}
impl Scanner {
pub async fn new(
index_manager: index::Manager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
) -> Result<Self, index::Error> {
let scanner = Self {
index_manager,
vfs_manager,
settings_manager,
pending_scan: Arc::new(Notify::new()),
};
tokio::spawn({
let mut scanner = scanner.clone();
async move {
loop {
scanner.pending_scan.notified().await;
if let Err(e) = scanner.update().await {
error!("Error while updating index: {}", e);
}
}
}
});
Ok(scanner)
}
pub fn trigger_scan(&self) {
self.pending_scan.notify_one();
}
pub fn begin_periodic_scans(&self) {
tokio::spawn({
let index = self.clone();
async move {
loop {
index.trigger_scan();
let sleep_duration = index
.settings_manager
.get_index_sleep_duration()
.await
.unwrap_or_else(|e| {
error!("Could not retrieve index sleep duration: {}", e);
Duration::from_secs(1800)
});
tokio::time::sleep(sleep_duration).await;
}
}
});
}
pub async fn update(&mut self) -> Result<(), index::Error> {
let start = Instant::now();
info!("Beginning collection scan");
let album_art_pattern = self
.settings_manager
.get_index_album_art_pattern()
.await
.ok();
let (scan_directories_output, mut collection_directories_input) = unbounded_channel();
let (scan_songs_output, mut collection_songs_input) = unbounded_channel();
let scan = Scan::new(
scan_directories_output,
scan_songs_output,
self.vfs_manager.clone(),
album_art_pattern,
);
let index_task = tokio::task::spawn_blocking(move || {
let mut index_builder = index::Builder::default();
loop {
let exhausted_songs = match collection_songs_input.try_recv() {
Ok(song) => {
index_builder.add_song(song);
false
}
Err(TryRecvError::Empty) => false,
Err(TryRecvError::Disconnected) => true,
};
let exhausted_directories = match collection_directories_input.try_recv() {
Ok(directory) => {
index_builder.add_directory(directory);
false
}
Err(TryRecvError::Empty) => false,
Err(TryRecvError::Disconnected) => true,
};
if exhausted_directories && exhausted_songs {
break;
}
}
index_builder.build()
});
let index = tokio::join!(scan.start(), index_task).1?;
self.index_manager.persist_index(&index).await?;
self.index_manager.replace_index(index).await;
info!(
"Collection scan took {} seconds",
start.elapsed().as_millis() as f32 / 1000.0
);
Ok(())
}
}
struct Scan {
directories_output: UnboundedSender<Directory>,
songs_output: UnboundedSender<Song>,
vfs_manager: vfs::Manager,
artwork_regex: Option<Regex>,
}
impl Scan {
pub fn new(
directories_output: UnboundedSender<Directory>,
songs_output: UnboundedSender<Song>,
vfs_manager: vfs::Manager,
artwork_regex: Option<Regex>,
) -> Self {
Self {
directories_output,
songs_output,
vfs_manager,
artwork_regex,
}
}
pub async fn start(self) -> Result<(), index::Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let roots = vfs.mounts().clone();
let key = "POLARIS_NUM_TRAVERSER_THREADS";
let num_threads = std::env::var_os(key)
.map(|v| v.to_string_lossy().to_string())
.and_then(|v| usize::from_str(&v).ok())
.unwrap_or_else(|| min(num_cpus::get(), 8));
info!("Browsing collection using {} threads", num_threads);
let directories_output = self.directories_output.clone();
let songs_output = self.songs_output.clone();
let artwork_regex = self.artwork_regex.clone();
let thread_pool = ThreadPoolBuilder::new().num_threads(num_threads).build()?;
thread_pool.scope({
|scope| {
for root in roots {
scope.spawn(|scope| {
process_directory(
scope,
root.source,
root.name,
directories_output.clone(),
songs_output.clone(),
artwork_regex.clone(),
);
});
}
}
});
Ok(())
}
}
fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
scope: &Scope,
real_path: P,
virtual_path: Q,
directories_output: UnboundedSender<Directory>,
songs_output: UnboundedSender<Song>,
artwork_regex: Option<Regex>,
) {
let read_dir = match fs::read_dir(&real_path) {
Ok(read_dir) => read_dir,
Err(e) => {
error!(
"Directory read error for `{}`: {}",
real_path.as_ref().display(),
e
);
return;
}
};
let mut songs = vec![];
let mut artwork_file = None;
for entry in read_dir {
let name = match entry {
Ok(ref f) => f.file_name(),
Err(e) => {
error!(
"File read error within `{}`: {}",
real_path.as_ref().display(),
e
);
break;
}
};
let entry_real_path = real_path.as_ref().join(&name);
let entry_virtual_path = virtual_path.as_ref().join(&name);
if entry_real_path.is_dir() {
scope.spawn({
let directories_output = directories_output.clone();
let songs_output = songs_output.clone();
let artwork_regex = artwork_regex.clone();
|scope| {
process_directory(
scope,
entry_real_path,
entry_virtual_path,
directories_output,
songs_output,
artwork_regex,
);
}
});
} else if let Some(metadata) = formats::read_metadata(&entry_real_path) {
songs.push(Song {
path: entry_real_path.clone(),
virtual_path: entry_virtual_path.clone(),
virtual_parent: entry_virtual_path.parent().unwrap().to_owned(),
track_number: metadata.track_number.map(|n| n as i64),
disc_number: metadata.disc_number.map(|n| n as i64),
title: metadata.title,
artists: metadata.artists,
album_artists: metadata.album_artists,
year: metadata.year.map(|n| n as i64),
album: metadata.album,
artwork: metadata.has_artwork.then(|| entry_virtual_path.clone()),
duration: metadata.duration.map(|n| n as i64),
lyricists: metadata.lyricists,
composers: metadata.composers,
genres: metadata.genres,
labels: metadata.labels,
date_added: get_date_created(&entry_real_path).unwrap_or_default(),
});
} else if artwork_file.is_none()
&& artwork_regex
.as_ref()
.is_some_and(|r| r.is_match(name.to_str().unwrap_or_default()))
{
artwork_file = Some(entry_virtual_path);
}
}
for mut song in songs {
song.artwork = song.artwork.or_else(|| artwork_file.clone());
songs_output.send(song).ok();
}
directories_output
.send(Directory {
virtual_path: virtual_path.as_ref().to_owned(),
virtual_parent: virtual_path.as_ref().parent().map(Path::to_owned),
})
.ok();
}
fn get_date_created<P: AsRef<Path>>(path: P) -> Option<i64> {
if let Ok(t) = fs::metadata(path).and_then(|m| m.created().or_else(|_| m.modified())) {
t.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.ok()
} else {
None
}
}
#[cfg(test)]
mod test {
use super::*;
use std::path::PathBuf;
use crate::{
app::{settings, test},
test_name,
};
const TEST_MOUNT_NAME: &str = "root";
#[tokio::test]
async fn scan_adds_new_content() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
ctx.scanner.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts
todo!();
// assert_eq!(all_directories.len(), 6);
// assert_eq!(all_songs.len(), 13);
}
#[tokio::test]
async fn finds_embedded_artwork() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
ctx.scanner.update().await.unwrap();
let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3");
let song = ctx
.index_manager
.get_song(&index::SongKey {
virtual_path: song_virtual_path.clone(),
})
.await
.unwrap();
assert_eq!(song.artwork, Some(song_virtual_path));
}
#[tokio::test]
async fn album_art_pattern_is_case_insensitive() {
let mut ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
let patterns = vec!["folder", "FOLDER"];
for pattern in patterns.into_iter() {
ctx.settings_manager
.amend(&settings::NewSettings {
album_art_pattern: Some(pattern.to_owned()),
..Default::default()
})
.await
.unwrap();
ctx.scanner.update().await.unwrap();
let hunted_virtual_dir: PathBuf =
[TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect();
let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg");
let song = &ctx.index_manager.flatten(hunted_virtual_dir).await.unwrap()[0];
todo!();
// assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}
}

15
src/app/test.rs
View file

@ -1,13 +1,12 @@
use std::path::PathBuf; use std::path::PathBuf;
use crate::app::{collection, config, ddns, playlist, settings, user, vfs}; use crate::app::{config, ddns, index, playlist, scanner, settings, user, vfs};
use crate::db::DB; use crate::db::DB;
use crate::test::*; use crate::test::*;
pub struct Context { pub struct Context {
pub browser: collection::Browser, pub index_manager: index::Manager,
pub index_manager: collection::IndexManager, pub scanner: scanner::Scanner,
pub updater: collection::Updater,
pub config_manager: config::Manager, pub config_manager: config::Manager,
pub ddns_manager: ddns::Manager, pub ddns_manager: ddns::Manager,
pub playlist_manager: playlist::Manager, pub playlist_manager: playlist::Manager,
@ -66,9 +65,8 @@ impl ContextBuilder {
vfs_manager.clone(), vfs_manager.clone(),
ddns_manager.clone(), ddns_manager.clone(),
); );
let browser = collection::Browser::new(db.clone(), vfs_manager.clone()); let index_manager = index::Manager::new(db.clone()).await;
let index_manager = collection::IndexManager::new(db.clone()).await; let scanner = scanner::Scanner::new(
let updater = collection::Updater::new(
index_manager.clone(), index_manager.clone(),
settings_manager.clone(), settings_manager.clone(),
vfs_manager.clone(), vfs_manager.clone(),
@ -80,9 +78,8 @@ impl ContextBuilder {
config_manager.apply(&self.config).await.unwrap(); config_manager.apply(&self.config).await.unwrap();
Context { Context {
browser,
index_manager, index_manager,
updater, scanner,
config_manager, config_manager,
ddns_manager, ddns_manager,
playlist_manager, playlist_manager,

2
src/main.rs
View file

@ -144,7 +144,7 @@ fn main() -> Result<(), Error> {
async fn async_main(cli_options: CLIOptions, paths: paths::Paths) -> Result<(), Error> { async fn async_main(cli_options: CLIOptions, paths: paths::Paths) -> Result<(), Error> {
// Create and run app // Create and run app
let app = app::App::new(cli_options.port.unwrap_or(5050), paths).await?; let app = app::App::new(cli_options.port.unwrap_or(5050), paths).await?;
app.updater.begin_periodic_scans(); app.scanner.begin_periodic_scans();
app.ddns_manager.begin_periodic_updates(); app.ddns_manager.begin_periodic_updates();
// Start server // Start server

12
src/server/axum.rs
View file

@ -27,21 +27,15 @@ pub async fn launch(app: App) -> Result<(), std::io::Error> {
Ok(()) Ok(())
} }
impl FromRef<App> for app::collection::Browser { impl FromRef<App> for app::index::Manager {
fn from_ref(app: &App) -> Self {
app.browser.clone()
}
}
impl FromRef<App> for app::collection::IndexManager {
fn from_ref(app: &App) -> Self { fn from_ref(app: &App) -> Self {
app.index_manager.clone() app.index_manager.clone()
} }
} }
impl FromRef<App> for app::collection::Updater { impl FromRef<App> for app::scanner::Scanner {
fn from_ref(app: &App) -> Self { fn from_ref(app: &App) -> Self {
app.updater.clone() app.scanner.clone()
} }
} }

55
src/server/axum/api.rs
View file

@ -13,7 +13,7 @@ use base64::{prelude::BASE64_STANDARD_NO_PAD, Engine};
use percent_encoding::percent_decode_str; use percent_encoding::percent_decode_str;
use crate::{ use crate::{
app::{collection, config, ddns, lastfm, playlist, settings, thumbnail, user, vfs, App}, app::{config, ddns, index, lastfm, playlist, scanner, settings, thumbnail, user, vfs, App},
server::{ server::{
dto, error::APIError, APIMajorVersion, API_ARRAY_SEPARATOR, API_MAJOR_VERSION, dto, error::APIError, APIMajorVersion, API_ARRAY_SEPARATOR, API_MAJOR_VERSION,
API_MINOR_VERSION, API_MINOR_VERSION,
@ -254,16 +254,13 @@ async fn put_preferences(
async fn post_trigger_index( async fn post_trigger_index(
_admin_rights: AdminRights, _admin_rights: AdminRights,
State(updater): State<collection::Updater>, State(scanner): State<scanner::Scanner>,
) -> Result<(), APIError> { ) -> Result<(), APIError> {
updater.trigger_scan(); scanner.trigger_scan();
Ok(()) Ok(())
} }
fn collection_files_to_response( fn index_files_to_response(files: Vec<index::File>, api_version: APIMajorVersion) -> Response {
files: Vec<collection::File>,
api_version: APIMajorVersion,
) -> Response {
match api_version { match api_version {
APIMajorVersion::V7 => Json( APIMajorVersion::V7 => Json(
files files
@ -282,23 +279,23 @@ fn collection_files_to_response(
} }
} }
fn songs_to_response(files: Vec<collection::SongKey>, api_version: APIMajorVersion) -> Response { fn songs_to_response(files: Vec<PathBuf>, api_version: APIMajorVersion) -> Response {
match api_version { match api_version {
APIMajorVersion::V7 => Json( APIMajorVersion::V7 => Json(
files files
.into_iter() .into_iter()
.map(|f| f.into()) .map(|p| index::SongKey { virtual_path: p }.into())
.collect::<Vec<dto::v7::Song>>(), .collect::<Vec<dto::v7::Song>>(),
) )
.into_response(), .into_response(),
APIMajorVersion::V8 => Json(dto::SongList { APIMajorVersion::V8 => Json(dto::SongList {
paths: files.into_iter().map(|s| s.virtual_path).collect(), paths: files.into_iter().collect(),
}) })
.into_response(), .into_response(),
} }
} }
fn albums_to_response(albums: Vec<collection::Album>, api_version: APIMajorVersion) -> Response { fn albums_to_response(albums: Vec<index::Album>, api_version: APIMajorVersion) -> Response {
match api_version { match api_version {
APIMajorVersion::V7 => Json( APIMajorVersion::V7 => Json(
albums albums
@ -320,32 +317,32 @@ fn albums_to_response(albums: Vec<collection::Album>, api_version: APIMajorVersi
async fn get_browse_root( async fn get_browse_root(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
) -> Response { ) -> Response {
let result = match index_manager.browse(PathBuf::new()).await { let result = match index_manager.browse(PathBuf::new()).await {
Ok(r) => r, Ok(r) => r,
Err(e) => return APIError::from(e).into_response(), Err(e) => return APIError::from(e).into_response(),
}; };
collection_files_to_response(result, api_version) index_files_to_response(result, api_version)
} }
async fn get_browse( async fn get_browse(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
Path(path): Path<PathBuf>, Path(path): Path<PathBuf>,
) -> Response { ) -> Response {
let result = match index_manager.browse(path).await { let result = match index_manager.browse(path).await {
Ok(r) => r, Ok(r) => r,
Err(e) => return APIError::from(e).into_response(), Err(e) => return APIError::from(e).into_response(),
}; };
collection_files_to_response(result, api_version) index_files_to_response(result, api_version)
} }
async fn get_flatten_root( async fn get_flatten_root(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
) -> Response { ) -> Response {
let songs = match index_manager.flatten(PathBuf::new()).await { let songs = match index_manager.flatten(PathBuf::new()).await {
Ok(s) => s, Ok(s) => s,
@ -357,7 +354,7 @@ async fn get_flatten_root(
async fn get_flatten( async fn get_flatten(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
Path(path): Path<PathBuf>, Path(path): Path<PathBuf>,
) -> Response { ) -> Response {
let songs = match index_manager.flatten(path).await { let songs = match index_manager.flatten(path).await {
@ -369,10 +366,10 @@ async fn get_flatten(
async fn get_artist( async fn get_artist(
_auth: Auth, _auth: Auth,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
Path(artist): Path<String>, Path(artist): Path<String>,
) -> Result<Json<dto::Artist>, APIError> { ) -> Result<Json<dto::Artist>, APIError> {
let artist_key = collection::ArtistKey { let artist_key = index::ArtistKey {
name: (!artist.is_empty()).then_some(artist), name: (!artist.is_empty()).then_some(artist),
}; };
Ok(Json(index_manager.get_artist(&artist_key).await?.into())) Ok(Json(index_manager.get_artist(&artist_key).await?.into()))
@ -380,10 +377,10 @@ async fn get_artist(
async fn get_album( async fn get_album(
_auth: Auth, _auth: Auth,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
Path((artists, name)): Path<(String, String)>, Path((artists, name)): Path<(String, String)>,
) -> Result<Json<dto::Album>, APIError> { ) -> Result<Json<dto::Album>, APIError> {
let album_key = collection::AlbumKey { let album_key = index::AlbumKey {
artists: artists artists: artists
.split(API_ARRAY_SEPARATOR) .split(API_ARRAY_SEPARATOR)
.map(str::to_owned) .map(str::to_owned)
@ -396,7 +393,7 @@ async fn get_album(
async fn get_random( async fn get_random(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
) -> Response { ) -> Response {
let albums = match index_manager.get_random_albums(20).await { let albums = match index_manager.get_random_albums(20).await {
Ok(d) => d, Ok(d) => d,
@ -408,7 +405,7 @@ async fn get_random(
async fn get_recent( async fn get_recent(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(index_manager): State<collection::IndexManager>, State(index_manager): State<index::Manager>,
) -> Response { ) -> Response {
let albums = match index_manager.get_recent_albums(20).await { let albums = match index_manager.get_recent_albums(20).await {
Ok(d) => d, Ok(d) => d,
@ -420,26 +417,26 @@ async fn get_recent(
async fn get_search_root( async fn get_search_root(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(browser): State<collection::Browser>, State(index_manager): State<index::Manager>,
) -> Response { ) -> Response {
let files = match browser.search("").await { let files = match index_manager.search("").await {
Ok(f) => f, Ok(f) => f,
Err(e) => return APIError::from(e).into_response(), Err(e) => return APIError::from(e).into_response(),
}; };
collection_files_to_response(files, api_version) songs_to_response(files, api_version)
} }
async fn get_search( async fn get_search(
_auth: Auth, _auth: Auth,
api_version: APIMajorVersion, api_version: APIMajorVersion,
State(browser): State<collection::Browser>, State(index_manager): State<index::Manager>,
Path(query): Path<String>, Path(query): Path<String>,
) -> Response { ) -> Response {
let files = match browser.search(&query).await { let files = match index_manager.search(&query).await {
Ok(f) => f, Ok(f) => f,
Err(e) => return APIError::from(e).into_response(), Err(e) => return APIError::from(e).into_response(),
}; };
collection_files_to_response(files, api_version) songs_to_response(files, api_version)
} }
async fn get_playlists( async fn get_playlists(

1
src/server/axum/error.rs
View file

@ -23,6 +23,7 @@ impl IntoResponse for APIError {
APIError::DirectoryNotFound(_) => StatusCode::NOT_FOUND, APIError::DirectoryNotFound(_) => StatusCode::NOT_FOUND,
APIError::ArtistNotFound => StatusCode::NOT_FOUND, APIError::ArtistNotFound => StatusCode::NOT_FOUND,
APIError::AlbumNotFound => StatusCode::NOT_FOUND, APIError::AlbumNotFound => StatusCode::NOT_FOUND,
APIError::SongNotFound => StatusCode::NOT_FOUND,
APIError::EmbeddedArtworkNotFound => StatusCode::NOT_FOUND, APIError::EmbeddedArtworkNotFound => StatusCode::NOT_FOUND,
APIError::EmptyPassword => StatusCode::BAD_REQUEST, APIError::EmptyPassword => StatusCode::BAD_REQUEST,
APIError::EmptyUsername => StatusCode::BAD_REQUEST, APIError::EmptyUsername => StatusCode::BAD_REQUEST,

25
src/server/dto/v7.rs
View file

@ -1,9 +1,6 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::app::{ use crate::app::{config, ddns, index, settings, thumbnail, user, vfs};
collection::{self},
config, ddns, settings, thumbnail, user, vfs,
};
use std::{convert::From, path::PathBuf}; use std::{convert::From, path::PathBuf};
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)] #[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
@ -237,17 +234,17 @@ pub enum CollectionFile {
Song(Song), Song(Song),
} }
impl From<collection::File> for CollectionFile { impl From<index::File> for CollectionFile {
fn from(f: collection::File) -> Self { fn from(f: index::File) -> Self {
match f { match f {
collection::File::Directory(d) => Self::Directory(Directory { index::File::Directory(d) => Self::Directory(Directory {
path: d, path: d,
artist: None, artist: None,
year: None, year: None,
album: None, album: None,
artwork: None, artwork: None,
}), }),
collection::File::Song(s) => Self::Song(Song { index::File::Song(s) => Self::Song(Song {
path: s, path: s,
track_number: None, track_number: None,
disc_number: None, disc_number: None,
@ -299,8 +296,8 @@ pub struct Song {
pub label: Option<String>, pub label: Option<String>,
} }
impl From<collection::SongKey> for Song { impl From<index::SongKey> for Song {
fn from(song_key: collection::SongKey) -> Self { fn from(song_key: index::SongKey) -> Self {
Self { Self {
path: song_key.virtual_path, path: song_key.virtual_path,
track_number: None, track_number: None,
@ -320,8 +317,8 @@ impl From<collection::SongKey> for Song {
} }
} }
impl From<collection::Song> for Song { impl From<index::Song> for Song {
fn from(s: collection::Song) -> Self { fn from(s: index::Song) -> Self {
Self { Self {
path: s.virtual_path, path: s.virtual_path,
track_number: s.track_number, track_number: s.track_number,
@ -350,8 +347,8 @@ pub struct Directory {
pub artwork: Option<PathBuf>, pub artwork: Option<PathBuf>,
} }
impl From<collection::Album> for Directory { impl From<index::Album> for Directory {
fn from(album: collection::Album) -> Self { fn from(album: index::Album) -> Self {
let path = album let path = album
.songs .songs
.first() .first()

26
src/server/dto/v8.rs
View file

@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::app::{collection, config, ddns, settings, thumbnail, user, vfs}; use crate::app::{config, ddns, index, settings, thumbnail, user, vfs};
use std::{convert::From, path::PathBuf}; use std::{convert::From, path::PathBuf};
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)] #[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
@ -259,8 +259,8 @@ pub struct Song {
pub labels: Vec<String>, pub labels: Vec<String>,
} }
impl From<collection::Song> for Song { impl From<index::Song> for Song {
fn from(s: collection::Song) -> Self { fn from(s: index::Song) -> Self {
Self { Self {
path: s.virtual_path, path: s.virtual_path,
track_number: s.track_number, track_number: s.track_number,
@ -291,14 +291,14 @@ pub struct BrowserEntry {
pub is_directory: bool, pub is_directory: bool,
} }
impl From<collection::File> for BrowserEntry { impl From<index::File> for BrowserEntry {
fn from(file: collection::File) -> Self { fn from(file: index::File) -> Self {
match file { match file {
collection::File::Directory(d) => Self { index::File::Directory(d) => Self {
is_directory: true, is_directory: true,
path: d, path: d,
}, },
collection::File::Song(s) => Self { index::File::Song(s) => Self {
is_directory: false, is_directory: false,
path: s, path: s,
}, },
@ -313,8 +313,8 @@ pub struct Artist {
pub album_appearances: Vec<AlbumHeader>, pub album_appearances: Vec<AlbumHeader>,
} }
impl From<collection::Artist> for Artist { impl From<index::Artist> for Artist {
fn from(a: collection::Artist) -> Self { fn from(a: index::Artist) -> Self {
Self { Self {
name: a.name, name: a.name,
albums: a.albums.into_iter().map(|a| a.into()).collect(), albums: a.albums.into_iter().map(|a| a.into()).collect(),
@ -335,8 +335,8 @@ pub struct AlbumHeader {
pub year: Option<i64>, pub year: Option<i64>,
} }
impl From<collection::Album> for AlbumHeader { impl From<index::Album> for AlbumHeader {
fn from(a: collection::Album) -> Self { fn from(a: index::Album) -> Self {
Self { Self {
name: a.name, name: a.name,
artwork: a.artwork.map(|a| a.to_string_lossy().to_string()), artwork: a.artwork.map(|a| a.to_string_lossy().to_string()),
@ -353,8 +353,8 @@ pub struct Album {
pub songs: Vec<Song>, pub songs: Vec<Song>,
} }
impl From<collection::Album> for Album { impl From<index::Album> for Album {
fn from(mut a: collection::Album) -> Self { fn from(mut a: index::Album) -> Self {
let songs = a.songs.drain(..).map(|s| s.into()).collect(); let songs = a.songs.drain(..).map(|s| s.into()).collect();
Self { Self {
header: a.into(), header: a.into(),

29
src/server/error.rs
View file

@ -1,7 +1,7 @@
use std::path::PathBuf; use std::path::PathBuf;
use thiserror::Error; use thiserror::Error;
use crate::app::{collection, config, ddns, lastfm, playlist, settings, thumbnail, user, vfs}; use crate::app::{config, ddns, index, lastfm, playlist, settings, thumbnail, user, vfs};
use crate::db; use crate::db;
#[derive(Error, Debug)] #[derive(Error, Debug)]
@ -30,6 +30,8 @@ pub enum APIError {
ArtistNotFound, ArtistNotFound,
#[error("Album not found")] #[error("Album not found")]
AlbumNotFound, AlbumNotFound,
#[error("Song not found")]
SongNotFound,
#[error("DDNS update query failed with HTTP status {0}")] #[error("DDNS update query failed with HTTP status {0}")]
DdnsUpdateQueryFailed(u16), DdnsUpdateQueryFailed(u16),
#[error("Cannot delete your own account")] #[error("Cannot delete your own account")]
@ -86,19 +88,20 @@ pub enum APIError {
VFSPathNotFound, VFSPathNotFound,
} }
impl From<collection::Error> for APIError { impl From<index::Error> for APIError {
fn from(error: collection::Error) -> APIError { fn from(error: index::Error) -> APIError {
match error { match error {
collection::Error::DirectoryNotFound(d) => APIError::DirectoryNotFound(d), index::Error::DirectoryNotFound(d) => APIError::DirectoryNotFound(d),
collection::Error::ArtistNotFound => APIError::ArtistNotFound, index::Error::ArtistNotFound => APIError::ArtistNotFound,
collection::Error::AlbumNotFound => APIError::AlbumNotFound, index::Error::AlbumNotFound => APIError::AlbumNotFound,
collection::Error::Database(e) => APIError::Database(e), index::Error::SongNotFound => APIError::SongNotFound,
collection::Error::DatabaseConnection(e) => e.into(), index::Error::Database(e) => APIError::Database(e),
collection::Error::Vfs(e) => e.into(), index::Error::DatabaseConnection(e) => e.into(),
collection::Error::IndexDeserializationError => APIError::Internal, index::Error::Vfs(e) => e.into(),
collection::Error::IndexSerializationError => APIError::Internal, index::Error::IndexDeserializationError => APIError::Internal,
collection::Error::ThreadPoolBuilder(_) => APIError::Internal, index::Error::IndexSerializationError => APIError::Internal,
collection::Error::ThreadJoining(_) => APIError::Internal, index::Error::ThreadPoolBuilder(_) => APIError::Internal,
index::Error::ThreadJoining(_) => APIError::Internal,
} }
} }
} }