vikarti.anatra/polaris-mirror
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Browsing via index (WIP)

Browse source
This commit is contained in:
Antoine Gersant 2024-07-31 03:41:32 -07:00
parent b4b0e1181f
commit e8af339cde
18 changed files with 170 additions and 646 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/app.rs
View file

@ -70,7 +70,6 @@ impl App {
let index_manager = collection::IndexManager::new(db.clone()).await;
let browser = collection::Browser::new(db.clone(), vfs_manager.clone());
let updater = collection::Updater::new(
db.clone(),
index_manager.clone(),
settings_manager.clone(),
vfs_manager.clone(),

4
src/app/collection.rs
View file

@ -1,15 +1,11 @@
mod browser;
mod cleaner;
mod index;
mod inserter;
mod scanner;
mod types;
mod updater;
pub use browser::*;
pub use cleaner::*;
pub use index::*;
pub use inserter::*;
pub use scanner::*;
pub use types::*;
pub use updater::*;

154
src/app/collection/browser.rs
View file

@ -18,152 +18,22 @@ impl Browser {
where
P: AsRef<Path>,
{
let mut output = Vec::new();
let mut connection = self.db.connect().await?;
if path.as_ref().components().count() == 0 {
// Browse top-level
let directories = sqlx::query_as!(
collection::Directory,
"SELECT * FROM directories WHERE virtual_parent IS NULL"
)
.fetch_all(connection.as_mut())
.await?;
output.extend(directories.into_iter().map(collection::File::Directory));
} else {
let vfs = self.vfs_manager.get_vfs().await?;
match vfs.virtual_to_real(&path) {
Ok(p) if p.exists() => {}
_ => {
return Err(collection::Error::DirectoryNotFound(
path.as_ref().to_owned(),
))
}
}
let path = path.as_ref().to_string_lossy();
// Browse sub-directory
let directories = sqlx::query_as!(
collection::Directory,
"SELECT * FROM directories WHERE virtual_parent = $1 ORDER BY virtual_path COLLATE NOCASE ASC",
path
)
.fetch_all(connection.as_mut())
.await?;
output.extend(directories.into_iter().map(collection::File::Directory));
let songs = sqlx::query_as!(
collection::Song,
"SELECT * FROM songs WHERE virtual_parent = $1 ORDER BY virtual_path COLLATE NOCASE ASC",
path
)
.fetch_all(connection.as_mut())
.await?;
output.extend(songs.into_iter().map(collection::File::Song));
}
Ok(output)
todo!();
}
pub async fn flatten<P>(&self, path: P) -> Result<Vec<collection::Song>, collection::Error>
where
P: AsRef<Path>,
{
let mut connection = self.db.connect().await?;
let songs = if path.as_ref().parent().is_some() {
let vfs = self.vfs_manager.get_vfs().await?;
match vfs.virtual_to_real(&path) {
Ok(p) if p.exists() => {}
_ => {
return Err(collection::Error::DirectoryNotFound(
path.as_ref().to_owned(),
))
}
}
let song_path_filter = {
let mut path_buf = path.as_ref().to_owned();
path_buf.push("%");
path_buf.as_path().to_string_lossy().into_owned()
};
sqlx::query_as!(
collection::Song,
"SELECT * FROM songs WHERE virtual_path LIKE $1 ORDER BY virtual_path COLLATE NOCASE ASC",
song_path_filter
)
.fetch_all(connection.as_mut())
.await?
} else {
sqlx::query_as!(
collection::Song,
"SELECT * FROM songs ORDER BY virtual_path COLLATE NOCASE ASC"
)
.fetch_all(connection.as_mut())
.await?
};
Ok(songs)
todo!();
}
pub async fn search(&self, query: &str) -> Result<Vec<collection::File>, collection::Error> {
let mut connection = self.db.connect().await?;
let like_test = format!("%{}%", query);
let mut output = Vec::new();
// Find dirs with matching path and parent not matching
{
let directories = sqlx::query_as!(
collection::Directory,
"SELECT * FROM directories WHERE virtual_path LIKE $1 AND virtual_parent NOT LIKE $1",
like_test
)
.fetch_all(connection.as_mut())
.await?;
output.extend(directories.into_iter().map(collection::File::Directory));
}
// Find songs with matching title/album/artist and non-matching parent
{
let songs = sqlx::query_as!(
collection::Song,
r#"
SELECT * FROM songs
WHERE ( virtual_path LIKE $1
OR title LIKE $1
OR album LIKE $1
OR artists LIKE $1
OR album_artists LIKE $1
)
AND virtual_parent NOT LIKE $1
"#,
like_test
)
.fetch_all(connection.as_mut())
.await?;
output.extend(songs.into_iter().map(collection::File::Song));
}
Ok(output)
todo!();
}
pub async fn get_song(&self, path: &Path) -> Result<collection::Song, collection::Error> {
let mut connection = self.db.connect().await?;
let path = path.to_string_lossy();
let song = sqlx::query_as!(
collection::Song,
"SELECT * FROM songs WHERE virtual_path = $1",
path
)
.fetch_one(connection.as_mut())
.await?;
Ok(song)
todo!();
}
}
@ -190,7 +60,7 @@ mod test {
assert_eq!(files.len(), 1);
match files[0] {
collection::File::Directory(ref d) => {
assert_eq!(d.virtual_path, root_path.to_str().unwrap())
assert_eq!(d, &root_path)
}
_ => panic!("Expected directory"),
}
@ -216,14 +86,14 @@ mod test {
assert_eq!(files.len(), 2);
match files[0] {
collection::File::Directory(ref d) => {
assert_eq!(d.virtual_path, khemmis_path.to_str().unwrap())
assert_eq!(d, &khemmis_path)
}
_ => panic!("Expected directory"),
}
match files[1] {
collection::File::Directory(ref d) => {
assert_eq!(d.virtual_path, tobokegao_path.to_str().unwrap())
assert_eq!(d, &tobokegao_path)
}
_ => panic!("Expected directory"),
}
@ -283,10 +153,7 @@ mod test {
let artwork_virtual_path = picnic_virtual_dir.join("Folder.png");
let song = ctx.browser.get_song(&song_virtual_path).await.unwrap();
assert_eq!(
song.virtual_path,
song_virtual_path.to_string_lossy().as_ref()
);
assert_eq!(song.virtual_path, song_virtual_path);
assert_eq!(song.track_number, Some(5));
assert_eq!(song.disc_number, None);
assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned()));
@ -297,9 +164,6 @@ mod test {
assert_eq!(song.album_artists, collection::MultiString(vec![]));
assert_eq!(song.album, Some("Picnic".to_owned()));
assert_eq!(song.year, Some(2016));
assert_eq!(
song.artwork,
Some(artwork_virtual_path.to_string_lossy().into_owned())
);
assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}

89
src/app/collection/cleaner.rs
View file

@ -1,89 +0,0 @@
use rayon::prelude::*;
use sqlx::{QueryBuilder, Sqlite};
use std::path::Path;
use crate::app::{collection, vfs};
use crate::db::DB;
#[derive(Clone)]
pub struct Cleaner {
db: DB,
vfs_manager: vfs::Manager,
}
impl Cleaner {
const BUFFER_SIZE: usize = 500; // Deletions in each transaction
pub fn new(db: DB, vfs_manager: vfs::Manager) -> Self {
Self { db, vfs_manager }
}
pub async fn clean(&self) -> Result<(), collection::Error> {
tokio::try_join!(self.clean_songs(), self.clean_directories())?;
Ok(())
}
pub async fn clean_directories(&self) -> Result<(), collection::Error> {
let directories = {
let mut connection = self.db.connect().await?;
sqlx::query!("SELECT path, virtual_path FROM directories")
.fetch_all(connection.as_mut())
.await?
};
let vfs = self.vfs_manager.get_vfs().await?;
let missing_directories = tokio::task::spawn_blocking(move || {
directories
.into_par_iter()
.filter(|d| !vfs.exists(&d.virtual_path) || !Path::new(&d.path).exists())
.map(|d| d.virtual_path)
.collect::<Vec<_>>()
})
.await?;
let mut connection = self.db.connect().await?;
for chunk in missing_directories[..].chunks(Self::BUFFER_SIZE) {
QueryBuilder::<Sqlite>::new("DELETE FROM directories WHERE virtual_path IN ")
.push_tuples(chunk, |mut b, virtual_path| {
b.push_bind(virtual_path);
})
.build()
.execute(connection.as_mut())
.await?;
}
Ok(())
}
pub async fn clean_songs(&self) -> Result<(), collection::Error> {
let songs = {
let mut connection = self.db.connect().await?;
sqlx::query!("SELECT path, virtual_path FROM songs")
.fetch_all(connection.as_mut())
.await?
};
let vfs = self.vfs_manager.get_vfs().await?;
let deleted_songs = tokio::task::spawn_blocking(move || {
songs
.into_par_iter()
.filter(|s| !vfs.exists(&s.virtual_path) || !Path::new(&s.path).exists())
.map(|s| s.virtual_path)
.collect::<Vec<_>>()
})
.await?;
for chunk in deleted_songs[..].chunks(Cleaner::BUFFER_SIZE) {
let mut connection = self.db.connect().await?;
QueryBuilder::<Sqlite>::new("DELETE FROM songs WHERE virtual_path IN ")
.push_tuples(chunk, |mut b, virtual_path| {
b.push_bind(virtual_path);
})
.build()
.execute(connection.as_mut())
.await?;
}
Ok(())
}
}

45
src/app/collection/index.rs
View file

@ -2,6 +2,7 @@ use std::{
borrow::BorrowMut,
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::{Path, PathBuf},
sync::{Arc, RwLock},
};
@ -73,18 +74,20 @@ impl IndexManager {
Ok(true)
}
pub(super) async fn get_songs(&self) -> Vec<collection::Song> {
pub async fn browse(
&self,
virtual_path: PathBuf,
) -> Result<Vec<collection::File>, collection::Error> {
spawn_blocking({
let index_manager = self.clone();
move || {
let index = index_manager.index.read().unwrap();
index.songs.values().cloned().collect::<Vec<_>>()
index.browse(virtual_path)
}
})
.await
.unwrap()
}
pub async fn get_artist(
&self,
artist_key: &ArtistKey,
@ -165,14 +168,32 @@ impl IndexManager {
#[derive(Default)]
pub(super) struct IndexBuilder {
directories: HashMap<PathBuf, HashSet<collection::File>>,
// filesystem: Trie<>,
songs: HashMap<SongID, collection::Song>,
artists: HashMap<ArtistID, Artist>,
albums: HashMap<AlbumID, Album>,
}
impl IndexBuilder {
pub fn add_directory(&mut self, directory: collection::Directory) {
self.directories
.entry(directory.virtual_path.clone())
.or_default();
if let Some(parent) = directory.virtual_parent {
self.directories
.entry(parent.clone())
.or_default()
.insert(collection::File::Directory(directory.virtual_path));
}
}
pub fn add_song(&mut self, song: collection::Song) {
let song_id: SongID = song.song_id();
self.directories
.entry(song.virtual_parent.clone())
.or_default()
.insert(collection::File::Song(song.virtual_path.clone()));
self.add_song_to_album(&song);
self.add_album_to_artists(&song);
self.songs.insert(song_id, song);
@ -250,6 +271,7 @@ impl IndexBuilder {
});
Index {
directories: self.directories,
songs: self.songs,
artists: self.artists,
albums: self.albums,
@ -260,6 +282,7 @@ impl IndexBuilder {
#[derive(Default, Serialize, Deserialize)]
pub(super) struct Index {
directories: HashMap<PathBuf, HashSet<collection::File>>,
songs: HashMap<SongID, collection::Song>,
artists: HashMap<ArtistID, Artist>,
albums: HashMap<AlbumID, Album>,
@ -267,6 +290,18 @@ pub(super) struct Index {
}
impl Index {
pub(self) fn browse<P: AsRef<Path>>(
&self,
virtual_path: P,
) -> Result<Vec<collection::File>, collection::Error> {
let Some(files) = self.directories.get(virtual_path.as_ref()) else {
return Err(collection::Error::DirectoryNotFound(
virtual_path.as_ref().to_owned(),
));
};
Ok(files.iter().cloned().collect())
}
pub(self) fn get_artist(&self, artist_id: ArtistID) -> Option<collection::Artist> {
self.artists.get(&artist_id).map(|a| {
let mut albums = a
@ -312,7 +347,7 @@ struct SongID(u64);
#[derive(Clone, Eq, Hash, PartialEq, Serialize, Deserialize)]
pub struct SongKey {
pub virtual_path: String,
pub virtual_path: PathBuf,
}
impl From<&collection::Song> for SongKey {
@ -360,7 +395,7 @@ impl From<&ArtistKey> for ArtistID {
#[derive(Clone, Default, Serialize, Deserialize)]
struct Album {
pub name: Option<String>,
pub artwork: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,

137
src/app/collection/inserter.rs
View file

@ -1,137 +0,0 @@
use std::borrow::Cow;
use log::error;
use sqlx::{
encode::IsNull,
pool::PoolConnection,
sqlite::{SqliteArgumentValue, SqliteTypeInfo},
QueryBuilder, Sqlite,
};
use crate::app::collection::{self, MultiString};
use crate::db::DB;
impl<'q> sqlx::Encode<'q, Sqlite> for MultiString {
fn encode_by_ref(&self, args: &mut Vec<SqliteArgumentValue<'q>>) -> IsNull {
if self.0.is_empty() {
IsNull::Yes
} else {
let joined = self.0.join(MultiString::SEPARATOR);
args.push(SqliteArgumentValue::Text(Cow::Owned(joined)));
IsNull::No
}
}
}
impl<'q> sqlx::Decode<'q, Sqlite> for MultiString {
fn decode(
value: <Sqlite as sqlx::database::HasValueRef<'q>>::ValueRef,
) -> Result<Self, sqlx::error::BoxDynError> {
let s: &str = sqlx::Decode::<Sqlite>::decode(value)?;
Ok(MultiString(
s.split(MultiString::SEPARATOR).map(str::to_owned).collect(),
))
}
}
impl sqlx::Type<Sqlite> for MultiString {
fn type_info() -> SqliteTypeInfo {
<&str as sqlx::Type<Sqlite>>::type_info()
}
}
pub struct Inserter<T> {
new_entries: Vec<T>,
db: DB,
}
impl<T> Inserter<T>
where
T: Insertable,
{
const BUFFER_SIZE: usize = 1000;
pub fn new(db: DB) -> Self {
let new_entries = Vec::with_capacity(Self::BUFFER_SIZE);
Self { new_entries, db }
}
pub async fn insert(&mut self, entry: T) {
self.new_entries.push(entry);
if self.new_entries.len() >= Self::BUFFER_SIZE {
self.flush().await;
}
}
pub async fn flush(&mut self) {
if self.new_entries.is_empty() {
return;
}
let Ok(connection) = self.db.connect().await else {
error!("Could not acquire connection to insert new entries in database");
return;
};
match Insertable::bulk_insert(&self.new_entries, connection).await {
Ok(_) => self.new_entries.clear(),
Err(e) => error!("Could not insert new entries in database: {}", e),
};
}
}
pub trait Insertable
where
Self: Sized,
{
async fn bulk_insert(
entries: &Vec<Self>,
connection: PoolConnection<Sqlite>,
) -> Result<(), sqlx::Error>;
}
impl Insertable for collection::Directory {
async fn bulk_insert(
entries: &Vec<Self>,
mut connection: PoolConnection<Sqlite>,
) -> Result<(), sqlx::Error> {
QueryBuilder::<Sqlite>::new("INSERT INTO directories(path, virtual_path, virtual_parent) ")
.push_values(entries.iter(), |mut b, directory| {
b.push_bind(&directory.path)
.push_bind(&directory.virtual_path)
.push_bind(&directory.virtual_parent);
})
.build()
.execute(connection.as_mut())
.await
.map(|_| ())
}
}
impl Insertable for collection::Song {
async fn bulk_insert(
entries: &Vec<Self>,
mut connection: PoolConnection<Sqlite>,
) -> Result<(), sqlx::Error> {
QueryBuilder::<Sqlite>::new("INSERT INTO songs(path, virtual_path, virtual_parent, track_number, disc_number, title, artists, album_artists, year, album, artwork, duration, lyricists, composers, genres, labels) ")
.push_values(entries.iter(), |mut b, song| {
b.push_bind(&song.path)
.push_bind(&song.virtual_path)
.push_bind(&song.virtual_parent)
.push_bind(song.track_number)
.push_bind(song.disc_number)
.push_bind(&song.title)
.push_bind(&song.artists)
.push_bind(&song.album_artists)
.push_bind(song.year)
.push_bind(&song.album)
.push_bind(&song.artwork)
.push_bind(song.duration)
.push_bind(&song.lyricists)
.push_bind(&song.composers)
.push_bind(&song.genres)
.push_bind(&song.labels);
})
.build()
.execute(connection.as_mut())
.await.map(|_| ())
}
}

30
src/app/collection/scanner.rs
View file

@ -109,10 +109,7 @@ fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
};
let entry_real_path = real_path.as_ref().join(&name);
let entry_real_path_string = entry_real_path.to_string_lossy().to_string();
let entry_virtual_path = virtual_path.as_ref().join(&name);
let entry_virtual_path_string = entry_virtual_path.to_string_lossy().to_string();
if entry_real_path.is_dir() {
scope.spawn({
@ -132,14 +129,9 @@ fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
});
} else if let Some(metadata) = formats::read_metadata(&entry_real_path) {
songs.push(collection::Song {
id: 0,
path: entry_real_path_string.clone(),
virtual_path: entry_virtual_path.to_string_lossy().to_string(),
virtual_parent: entry_virtual_path
.parent()
.unwrap()
.to_string_lossy()
.to_string(),
path: entry_real_path.clone(),
virtual_path: entry_virtual_path.clone(),
virtual_parent: entry_virtual_path.parent().unwrap().to_owned(),
track_number: metadata.track_number.map(|n| n as i64),
disc_number: metadata.disc_number.map(|n| n as i64),
title: metadata.title,
@ -147,9 +139,7 @@ fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
album_artists: MultiString(metadata.album_artists),
year: metadata.year.map(|n| n as i64),
album: metadata.album,
artwork: metadata
.has_artwork
.then(|| entry_virtual_path_string.clone()),
artwork: metadata.has_artwork.then(|| entry_virtual_path.clone()),
duration: metadata.duration.map(|n| n as i64),
lyricists: MultiString(metadata.lyricists),
composers: MultiString(metadata.composers),
@ -162,7 +152,7 @@ fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
.as_ref()
.is_some_and(|r| r.is_match(name.to_str().unwrap_or_default()))
{
artwork_file = Some(entry_virtual_path_string);
artwork_file = Some(entry_virtual_path);
}
}
@ -173,14 +163,8 @@ fn process_directory<P: AsRef<Path>, Q: AsRef<Path>>(
directories_output
.send(collection::Directory {
id: 0,
path: real_path.as_ref().to_string_lossy().to_string(),
virtual_path: virtual_path.as_ref().to_string_lossy().to_string(),
virtual_parent: virtual_path
.as_ref()
.parent()
.map(|p| p.to_string_lossy().to_string())
.filter(|p| !p.is_empty()),
virtual_path: virtual_path.as_ref().to_owned(),
virtual_parent: virtual_path.as_ref().parent().map(Path::to_owned),
})
.ok();
}

24
src/app/collection/types.rs
View file

@ -8,6 +8,7 @@ use crate::{
db,
};
// TODO no longer needed!!
#[derive(Clone, Debug, FromRow, PartialEq, Eq, Serialize, Deserialize)]
pub struct MultiString(pub Vec<String>);
@ -48,18 +49,17 @@ pub enum Error {
ThreadJoining(#[from] tokio::task::JoinError),
}
#[derive(Debug, PartialEq, Eq)]
#[derive(Clone, Debug, Hash, PartialEq, Eq, Serialize, Deserialize)]
pub enum File {
Directory(Directory),
Song(Song),
Directory(PathBuf),
Song(PathBuf),
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Song {
pub id: i64,
pub path: String,
pub virtual_path: String,
pub virtual_parent: String,
pub path: PathBuf,
pub virtual_path: PathBuf,
pub virtual_parent: PathBuf,
pub track_number: Option<i64>,
pub disc_number: Option<i64>,
pub title: Option<String>,
@ -67,7 +67,7 @@ pub struct Song {
pub album_artists: MultiString,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<String>,
pub artwork: Option<PathBuf>,
pub duration: Option<i64>,
pub lyricists: MultiString,
pub composers: MultiString,
@ -78,10 +78,8 @@ pub struct Song {
#[derive(Debug, PartialEq, Eq)]
pub struct Directory {
pub id: i64,
pub path: String,
pub virtual_path: String,
pub virtual_parent: Option<String>,
pub virtual_path: PathBuf,
pub virtual_parent: Option<PathBuf>,
}
#[derive(Debug, Default, PartialEq, Eq)]
@ -93,7 +91,7 @@ pub struct Artist {
#[derive(Debug, Default, PartialEq, Eq)]
pub struct Album {
pub name: Option<String>,
pub artwork: Option<String>,
pub artwork: Option<PathBuf>,
pub artists: Vec<String>,
pub year: Option<i64>,
pub date_added: i64,

169
src/app/collection/updater.rs
View file

@ -6,14 +6,10 @@ use tokio::{
time::Instant,
};
use crate::{
app::{collection::*, settings, vfs},
db::DB,
};
use crate::app::{collection::*, settings, vfs};
#[derive(Clone)]
pub struct Updater {
db: DB,
index_manager: IndexManager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
@ -22,13 +18,11 @@ pub struct Updater {
impl Updater {
pub async fn new(
db: DB,
index_manager: IndexManager,
settings_manager: settings::Manager,
vfs_manager: vfs::Manager,
) -> Result<Self, Error> {
let updater = Self {
db,
index_manager,
vfs_manager,
settings_manager,
@ -94,49 +88,48 @@ impl Updater {
album_art_pattern,
);
let mut directory_inserter = Inserter::<Directory>::new(self.db.clone());
let directory_task = tokio::spawn(async move {
let capacity = 500;
let mut buffer: Vec<Directory> = Vec::with_capacity(capacity);
loop {
match collection_directories_input
.recv_many(&mut buffer, capacity)
.await
{
0 => break,
_ => {
for directory in buffer.drain(0..) {
directory_inserter.insert(directory).await;
}
}
}
}
directory_inserter.flush().await;
});
let song_task = tokio::spawn(async move {
let index_task = tokio::spawn(async move {
let capacity = 500;
let mut index_builder = IndexBuilder::default();
let mut buffer: Vec<Song> = Vec::with_capacity(capacity);
let mut song_buffer: Vec<Song> = Vec::with_capacity(capacity);
let mut directory_buffer: Vec<Directory> = Vec::with_capacity(capacity);
loop {
match collection_songs_input
.recv_many(&mut buffer, capacity)
let exhausted_songs = match collection_songs_input
.recv_many(&mut song_buffer, capacity)
.await
{
0 => break,
0 => true,
_ => {
for song in buffer.drain(0..) {
for song in song_buffer.drain(0..) {
index_builder.add_song(song);
}
false
}
};
let exhausted_directories = match collection_directories_input
.recv_many(&mut directory_buffer, capacity)
.await
{
0 => true,
_ => {
for directory in directory_buffer.drain(0..) {
index_builder.add_directory(directory);
}
false
}
};
if exhausted_directories && exhausted_songs {
break;
}
}
index_builder.build()
});
let index = tokio::join!(scanner.scan(), directory_task, song_task).2?;
let index = tokio::join!(scanner.scan(), index_task).1?;
self.index_manager.persist_index(&index).await?;
self.index_manager.replace_index(index).await;
@ -145,33 +138,6 @@ impl Updater {
start.elapsed().as_millis() as f32 / 1000.0
);
let start = Instant::now();
info!("Beginning collection DB update");
tokio::task::spawn({
let db = self.db.clone();
let vfs_manager = self.vfs_manager.clone();
let index_manager = self.index_manager.clone();
async move {
let cleaner = Cleaner::new(db.clone(), vfs_manager);
if let Err(e) = cleaner.clean().await {
error!("Error while cleaning up database: {}", e);
}
let mut song_inserter = Inserter::<Song>::new(db.clone());
for song in index_manager.get_songs().await {
song_inserter.insert(song).await;
}
song_inserter.flush().await;
}
})
.await?;
info!(
"Collection DB update took {} seconds",
start.elapsed().as_millis() as f32 / 1000.0
);
Ok(())
}
}
@ -181,7 +147,7 @@ mod test {
use std::path::PathBuf;
use crate::{
app::{collection::*, settings, test},
app::{settings, test},
test_name,
};
@ -197,71 +163,10 @@ mod test {
ctx.updater.update().await.unwrap();
ctx.updater.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts
let mut connection = ctx.db.connect().await.unwrap();
let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
.fetch_all(connection.as_mut())
.await
.unwrap();
let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
.fetch_all(connection.as_mut())
.await
.unwrap();
assert_eq!(all_directories.len(), 6);
assert_eq!(all_songs.len(), 13);
}
todo!();
#[tokio::test]
async fn scan_removes_missing_content() {
let builder = test::ContextBuilder::new(test_name!());
let original_collection_dir: PathBuf = ["test-data", "small-collection"].iter().collect();
let test_collection_dir: PathBuf = builder.test_directory.join("small-collection");
let copy_options = fs_extra::dir::CopyOptions::new();
fs_extra::dir::copy(
original_collection_dir,
&builder.test_directory,
&copy_options,
)
.unwrap();
let mut ctx = builder
.mount(TEST_MOUNT_NAME, test_collection_dir.to_str().unwrap())
.build()
.await;
ctx.updater.update().await.unwrap();
{
let mut connection = ctx.db.connect().await.unwrap();
let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
.fetch_all(connection.as_mut())
.await
.unwrap();
let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
.fetch_all(connection.as_mut())
.await
.unwrap();
assert_eq!(all_directories.len(), 6);
assert_eq!(all_songs.len(), 13);
}
let khemmis_directory = test_collection_dir.join("Khemmis");
std::fs::remove_dir_all(khemmis_directory).unwrap();
ctx.updater.update().await.unwrap();
{
let mut connection = ctx.db.connect().await.unwrap();
let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
.fetch_all(connection.as_mut())
.await
.unwrap();
let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
.fetch_all(connection.as_mut())
.await
.unwrap();
assert_eq!(all_directories.len(), 4);
assert_eq!(all_songs.len(), 8);
}
// assert_eq!(all_directories.len(), 6);
// assert_eq!(all_songs.len(), 13);
}
#[tokio::test]
@ -277,10 +182,7 @@ mod test {
let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3");
let song = ctx.browser.get_song(&song_virtual_path).await.unwrap();
assert_eq!(
song.artwork,
Some(song_virtual_path.to_string_lossy().into_owned())
);
assert_eq!(song.artwork, Some(song_virtual_path));
}
#[tokio::test]
@ -306,10 +208,7 @@ mod test {
[TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect();
let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg");
let song = &ctx.browser.flatten(&hunted_virtual_dir).await.unwrap()[0];
assert_eq!(
song.artwork,
Some(artwork_virtual_path.to_string_lossy().into_owned())
);
assert_eq!(song.artwork, Some(artwork_virtual_path));
}
}
}

40
src/app/playlist.rs
View file

@ -1,5 +1,6 @@
use core::clone::Clone;
use sqlx::{Acquire, QueryBuilder, Sqlite};
use std::path::PathBuf;
use crate::app::{collection::Song, vfs};
use crate::db::{self, DB};
@ -48,7 +49,7 @@ impl Manager {
&self,
playlist_name: &str,
owner: &str,
content: &[String],
content: &[PathBuf],
) -> Result<(), Error> {
let vfs = self.vfs_manager.get_vfs().await?;
@ -145,19 +146,20 @@ impl Manager {
.ok_or(Error::PlaylistNotFound)?;
// List songs
sqlx::query_as!(
Song,
r#"
SELECT s.*
FROM playlist_songs ps
INNER JOIN songs s ON ps.path = s.path
WHERE ps.playlist = $1
ORDER BY ps.ordering
"#,
playlist_id
)
.fetch_all(connection.as_mut())
.await?
todo!();
// sqlx::query_as!(
// Song,
// r#"
// SELECT s.*
// FROM playlist_songs ps
// INNER JOIN songs s ON ps.virtual_path = s.virtual_path
// WHERE ps.playlist = $1
// ORDER BY ps.ordering
// "#,
// playlist_id
// )
// .fetch_all(connection.as_mut())
// .await?
};
Ok(songs)
@ -230,14 +232,14 @@ mod test {
ctx.updater.update().await.unwrap();
let playlist_content: Vec<String> = ctx
let playlist_content = ctx
.browser
.flatten(Path::new(TEST_MOUNT_NAME))
.await
.unwrap()
.into_iter()
.map(|s| s.virtual_path)
.collect();
.collect::<Vec<_>>();
assert_eq!(playlist_content.len(), 13);
ctx.playlist_manager
@ -295,14 +297,14 @@ mod test {
ctx.updater.update().await.unwrap();
let playlist_content: Vec<String> = ctx
let playlist_content = ctx
.browser
.flatten(Path::new(TEST_MOUNT_NAME))
.await
.unwrap()
.into_iter()
.map(|s| s.virtual_path)
.collect();
.collect::<Vec<_>>();
assert_eq!(playlist_content.len(), 13);
ctx.playlist_manager
@ -327,6 +329,6 @@ mod test {
]
.iter()
.collect();
assert_eq!(songs[0].virtual_path, first_song_path.to_str().unwrap());
assert_eq!(songs[0].virtual_path, first_song_path);
}
}

3
src/app/test.rs
View file

@ -5,7 +5,6 @@ use crate::db::DB;
use crate::test::*;
pub struct Context {
pub db: DB,
pub browser: collection::Browser,
pub index_manager: collection::IndexManager,
pub updater: collection::Updater,
@ -70,7 +69,6 @@ impl ContextBuilder {
let browser = collection::Browser::new(db.clone(), vfs_manager.clone());
let index_manager = collection::IndexManager::new(db.clone()).await;
let updater = collection::Updater::new(
db.clone(),
index_manager.clone(),
settings_manager.clone(),
vfs_manager.clone(),
@ -82,7 +80,6 @@ impl ContextBuilder {
config_manager.apply(&self.config).await.unwrap();
Context {
db,
browser,
index_manager,
updater,

6
src/app/vfs.rs
View file

@ -52,12 +52,6 @@ impl VFS {
VFS { mounts }
}
pub fn exists<P: AsRef<Path>>(&self, virtual_path: P) -> bool {
self.mounts
.iter()
.any(|m| virtual_path.as_ref().starts_with(&m.name))
}
pub fn virtual_to_real<P: AsRef<Path>>(&self, virtual_path: P) -> Result<PathBuf, Error> {
for mount in &self.mounts {
let mount_path = Path::new(&mount.name);

32
src/db/20240711080449_init.sql
View file

@ -45,36 +45,6 @@ CREATE TABLE users (
UNIQUE(name)
);
CREATE TABLE directories (
id INTEGER PRIMARY KEY NOT NULL,
path TEXT NOT NULL,
virtual_path TEXT NOT NULL,
virtual_parent TEXT,
UNIQUE(path) ON CONFLICT REPLACE
);
CREATE TABLE songs (
id INTEGER PRIMARY KEY NOT NULL,
path TEXT NOT NULL,
virtual_path TEXT NOT NULL,
virtual_parent TEXT NOT NULL,
track_number INTEGER,
disc_number INTEGER,
title TEXT,
artists TEXT,
album_artists TEXT,
year INTEGER,
album TEXT,
artwork TEXT,
duration INTEGER,
lyricists TEXT,
composers TEXT,
genres TEXT,
labels TEXT,
date_added INTEGER DEFAULT 0 NOT NULL,
UNIQUE(path) ON CONFLICT REPLACE
);
CREATE TABLE collection_index (
id INTEGER PRIMARY KEY NOT NULL CHECK(id = 0),
content BLOB
@ -93,7 +63,7 @@ CREATE TABLE playlists (
CREATE TABLE playlist_songs (
id INTEGER PRIMARY KEY NOT NULL,
playlist INTEGER NOT NULL,
path TEXT NOT NULL,
virtual_path TEXT NOT NULL,
ordering INTEGER NOT NULL,
FOREIGN KEY(playlist) REFERENCES playlists(id) ON DELETE CASCADE ON UPDATE CASCADE,
UNIQUE(playlist, ordering) ON CONFLICT REPLACE

13
src/server/axum/api.rs
View file

@ -1,3 +1,5 @@
use std::path::PathBuf;
use axum::{
extract::{DefaultBodyLimit, Path, Query, State},
response::{Html, IntoResponse, Response},
@ -321,9 +323,9 @@ fn albums_to_response(albums: Vec<collection::Album>, api_version: APIMajorVersi
async fn get_browse_root(
_auth: Auth,
api_version: APIMajorVersion,
State(browser): State<collection::Browser>,
State(index_manager): State<collection::IndexManager>,
) -> Response {
let result = match browser.browse(std::path::Path::new("")).await {
let result = match index_manager.browse(PathBuf::new()).await {
Ok(r) => r,
Err(e) => return APIError::from(e).into_response(),
};
@ -333,11 +335,10 @@ async fn get_browse_root(
async fn get_browse(
_auth: Auth,
api_version: APIMajorVersion,
State(browser): State<collection::Browser>,
Path(path): Path<String>,
State(index_manager): State<collection::IndexManager>,
Path(path): Path<PathBuf>,
) -> Response {
let path = percent_decode_str(&path).decode_utf8_lossy();
let result = match browser.browse(std::path::Path::new(path.as_ref())).await {
let result = match index_manager.browse(path).await {
Ok(r) => r,
Err(e) => return APIError::from(e).into_response(),
};

47
src/server/dto/v7.rs
View file

@ -4,7 +4,7 @@ use crate::app::{
collection::{self, MultiString},
config, ddns, settings, thumbnail, user, vfs,
};
use std::convert::From;
use std::{convert::From, path::PathBuf};
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct Version {
@ -240,8 +240,29 @@ pub enum CollectionFile {
impl From<collection::File> for CollectionFile {
fn from(f: collection::File) -> Self {
match f {
collection::File::Directory(d) => Self::Directory(d.into()),
collection::File::Song(s) => Self::Song(s.into()),
collection::File::Directory(d) => Self::Directory(Directory {
path: d,
artist: None,
year: None,
album: None,
artwork: None,
}),
collection::File::Song(s) => Self::Song(Song {
path: s,
track_number: None,
disc_number: None,
title: None,
artist: None,
album_artist: None,
year: None,
album: None,
artwork: None,
duration: None,
lyricist: None,
composer: None,
genre: None,
label: None,
}),
}
}
}
@ -258,7 +279,7 @@ impl MultiString {
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Song {
pub path: String,
pub path: PathBuf,
pub track_number: Option<i64>,
pub disc_number: Option<i64>,
pub title: Option<String>,
@ -266,7 +287,7 @@ pub struct Song {
pub album_artist: Option<String>,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<String>,
pub artwork: Option<PathBuf>,
pub duration: Option<i64>,
pub lyricist: Option<String>,
pub composer: Option<String>,
@ -297,23 +318,11 @@ impl From<collection::Song> for Song {
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Directory {
pub path: String,
pub path: PathBuf,
pub artist: Option<String>,
pub year: Option<i64>,
pub album: Option<String>,
pub artwork: Option<String>,
}
impl From<collection::Directory> for Directory {
fn from(d: collection::Directory) -> Self {
Self {
path: d.virtual_path,
artist: None,
year: None,
album: None,
artwork: None,
}
}
pub artwork: Option<PathBuf>,
}
impl From<collection::Album> for Directory {

16
src/server/dto/v8.rs
View file

@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
use crate::app::{collection, config, ddns, settings, thumbnail, user, vfs};
use std::convert::From;
use std::{convert::From, path::PathBuf};
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
pub struct Version {
@ -73,7 +73,7 @@ pub struct ListPlaylistsEntry {
#[derive(Clone, Serialize, Deserialize)]
pub struct SavePlaylistInput {
pub tracks: Vec<String>,
pub tracks: Vec<std::path::PathBuf>,
}
#[derive(Serialize, Deserialize)]
@ -230,7 +230,7 @@ impl From<settings::Settings> for Settings {
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Song {
pub path: String,
pub path: PathBuf,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub track_number: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
@ -246,7 +246,7 @@ pub struct Song {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub album: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub artwork: Option<String>,
pub artwork: Option<PathBuf>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub duration: Option<i64>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
@ -282,7 +282,7 @@ impl From<collection::Song> for Song {
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct BrowserEntry {
pub path: String,
pub path: PathBuf,
pub is_directory: bool,
}
@ -291,11 +291,11 @@ impl From<collection::File> for BrowserEntry {
match file {
collection::File::Directory(d) => Self {
is_directory: true,
path: d.virtual_path,
path: d,
},
collection::File::Song(s) => Self {
is_directory: false,
path: s.virtual_path,
path: s,
},
}
}
@ -332,7 +332,7 @@ impl From<collection::Album> for AlbumHeader {
fn from(a: collection::Album) -> Self {
Self {
name: a.name,
artwork: a.artwork,
artwork: a.artwork.map(|a| a.to_string_lossy().to_string()),
artists: a.artists,
year: a.year,
}

2
src/server/test/collection.rs
View file

@ -317,5 +317,5 @@ async fn search_with_query() {
]
.iter()
.collect();
assert_eq!(results[0].path, path.to_string_lossy());
assert_eq!(results[0].path, path);
}

4
src/server/test/playlist.rs
View file

@ -1,3 +1,5 @@
use std::path::Path;
use http::StatusCode;
use crate::server::dto;
@ -53,7 +55,7 @@ async fn save_playlist_large() {
service.login().await;
let tracks = (0..100_000)
.map(|_| "My Super Cool Song".to_string())
.map(|_| Path::new("My Super Cool Song").to_owned())
.collect();
let my_playlist = dto::SavePlaylistInput { tracks };
let request = protocol::save_playlist(TEST_PLAYLIST_NAME, my_playlist);