diff --git a/src/app.rs b/src/app.rs
index d173b72..bde215f 100644
--- a/src/app.rs
+++ b/src/app.rs
@@ -9,6 +9,7 @@ pub mod ddns;
pub mod index;
pub mod lastfm;
pub mod playlist;
+pub mod scanner;
pub mod settings;
pub mod thumbnail;
pub mod user;
@@ -34,6 +35,7 @@ pub struct App {
pub port: u16,
pub web_dir_path: PathBuf,
pub swagger_dir_path: PathBuf,
+ pub scanner: scanner::Scanner,
pub index: index::Index,
pub config_manager: config::Manager,
pub ddns_manager: ddns::Manager,
@@ -62,7 +64,9 @@ impl App {
let auth_secret = settings_manager.get_auth_secret().await?;
let ddns_manager = ddns::Manager::new(db.clone());
let user_manager = user::Manager::new(db.clone(), auth_secret);
- let index = index::Index::new(db.clone(), vfs_manager.clone(), settings_manager.clone());
+ let scanner =
+ scanner::Scanner::new(db.clone(), vfs_manager.clone(), settings_manager.clone());
+ let index = index::Index::new(db.clone(), vfs_manager.clone());
let config_manager = config::Manager::new(
settings_manager.clone(),
user_manager.clone(),
@@ -82,6 +86,7 @@ impl App {
port,
web_dir_path: paths.web_dir_path,
swagger_dir_path: paths.swagger_dir_path,
+ scanner,
index,
config_manager,
ddns_manager,
diff --git a/src/app/index.rs b/src/app/index.rs
index b0b00cf..028d30a 100644
--- a/src/app/index.rs
+++ b/src/app/index.rs
@@ -1,74 +1,21 @@
-use log::error;
-use std::sync::Arc;
-use std::time::Duration;
-use tokio::sync::Notify;
-
-use crate::app::{settings, vfs};
+use crate::app::vfs;
use crate::db::DB;
-mod metadata;
mod query;
#[cfg(test)]
mod test;
mod types;
-mod update;
-pub use self::query::*;
pub use self::types::*;
#[derive(Clone)]
pub struct Index {
db: DB,
vfs_manager: vfs::Manager,
- settings_manager: settings::Manager,
- pending_reindex: Arc<Notify>,
}
impl Index {
- pub fn new(db: DB, vfs_manager: vfs::Manager, settings_manager: settings::Manager) -> Self {
- let index = Self {
- db,
- vfs_manager,
- settings_manager,
- pending_reindex: Arc::new(Notify::new()),
- };
-
- tokio::spawn({
- let index = index.clone();
- async move {
- loop {
- index.pending_reindex.notified().await;
- if let Err(e) = index.update().await {
- error!("Error while updating index: {}", e);
- }
- }
- }
- });
-
- index
- }
-
- pub fn trigger_reindex(&self) {
- self.pending_reindex.notify_one();
- }
-
- pub fn begin_periodic_updates(&self) {
- tokio::spawn({
- let index = self.clone();
- async move {
- loop {
- index.trigger_reindex();
- let sleep_duration = index
- .settings_manager
- .get_index_sleep_duration()
- .await
- .unwrap_or_else(|e| {
- error!("Could not retrieve index sleep duration: {}", e);
- Duration::from_secs(1800)
- });
- tokio::time::sleep(sleep_duration).await;
- }
- }
- });
+ pub fn new(db: DB, vfs_manager: vfs::Manager) -> Self {
+ Self { db, vfs_manager }
}
}
diff --git a/src/app/index/query.rs b/src/app/index/query.rs
index 95a5ff4..900bca2 100644
--- a/src/app/index/query.rs
+++ b/src/app/index/query.rs
@@ -1,22 +1,10 @@
-use std::path::{Path, PathBuf};
+use std::path::Path;
use super::*;
-use crate::db;
-
-#[derive(thiserror::Error, Debug)]
-pub enum QueryError {
- #[error(transparent)]
- Database(#[from] sqlx::Error),
- #[error(transparent)]
- DatabaseConnection(#[from] db::Error),
- #[error("Song was not found: `{0}`")]
- SongNotFound(PathBuf),
- #[error(transparent)]
- Vfs(#[from] vfs::Error),
-}
+use crate::app::scanner;
impl Index {
- pub async fn browse<P>(&self, virtual_path: P) -> Result<Vec<CollectionFile>, QueryError>
+ pub async fn browse<P>(&self, virtual_path: P) -> Result<Vec<CollectionFile>, Error>
where
P: AsRef<Path>,
{
@@ -26,10 +14,12 @@ impl Index {
if virtual_path.as_ref().components().count() == 0 {
// Browse top-level
- let real_directories =
- sqlx::query_as!(Directory, "SELECT * FROM directories WHERE parent IS NULL")
- .fetch_all(connection.as_mut())
- .await?;
+ let real_directories = sqlx::query_as!(
+ scanner::Directory,
+ "SELECT * FROM directories WHERE parent IS NULL"
+ )
+ .fetch_all(connection.as_mut())
+ .await?;
let virtual_directories = real_directories
.into_iter()
.filter_map(|d| d.virtualize(&vfs));
@@ -40,7 +30,7 @@ impl Index {
let real_path_string = real_path.as_path().to_string_lossy().into_owned();
let real_directories = sqlx::query_as!(
- Directory,
+ scanner::Directory,
"SELECT * FROM directories WHERE parent = $1 ORDER BY path COLLATE NOCASE ASC",
real_path_string
)
@@ -54,7 +44,7 @@ impl Index {
output.extend(virtual_directories.map(CollectionFile::Directory));
let real_songs = sqlx::query_as!(
- Song,
+ scanner::Song,
"SELECT * FROM songs WHERE parent = $1 ORDER BY path COLLATE NOCASE ASC",
real_path_string
)
@@ -68,7 +58,7 @@ impl Index {
Ok(output)
}
- pub async fn flatten<P>(&self, virtual_path: P) -> Result<Vec<Song>, QueryError>
+ pub async fn flatten<P>(&self, virtual_path: P) -> Result<Vec<scanner::Song>, Error>
where
P: AsRef<Path>,
{
@@ -83,28 +73,31 @@ impl Index {
path_buf.as_path().to_string_lossy().into_owned()
};
sqlx::query_as!(
- Song,
+ scanner::Song,
"SELECT * FROM songs WHERE path LIKE $1 ORDER BY path COLLATE NOCASE ASC",
song_path_filter
)
.fetch_all(connection.as_mut())
.await?
} else {
- sqlx::query_as!(Song, "SELECT * FROM songs ORDER BY path COLLATE NOCASE ASC")
- .fetch_all(connection.as_mut())
- .await?
+ sqlx::query_as!(
+ scanner::Song,
+ "SELECT * FROM songs ORDER BY path COLLATE NOCASE ASC"
+ )
+ .fetch_all(connection.as_mut())
+ .await?
};
let virtual_songs = real_songs.into_iter().filter_map(|s| s.virtualize(&vfs));
Ok(virtual_songs.collect::<Vec<_>>())
}
- pub async fn get_random_albums(&self, count: i64) -> Result<Vec<Directory>, QueryError> {
+ pub async fn get_random_albums(&self, count: i64) -> Result<Vec<scanner::Directory>, Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let mut connection = self.db.connect().await?;
let real_directories = sqlx::query_as!(
- Directory,
+ scanner::Directory,
"SELECT * FROM directories WHERE album IS NOT NULL ORDER BY RANDOM() DESC LIMIT $1",
count
)
@@ -117,12 +110,12 @@ impl Index {
Ok(virtual_directories.collect::<Vec<_>>())
}
- pub async fn get_recent_albums(&self, count: i64) -> Result<Vec<Directory>, QueryError> {
+ pub async fn get_recent_albums(&self, count: i64) -> Result<Vec<scanner::Directory>, Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let mut connection = self.db.connect().await?;
let real_directories = sqlx::query_as!(
- Directory,
+ scanner::Directory,
"SELECT * FROM directories WHERE album IS NOT NULL ORDER BY date_added DESC LIMIT $1",
count
)
@@ -135,7 +128,7 @@ impl Index {
Ok(virtual_directories.collect::<Vec<_>>())
}
- pub async fn search(&self, query: &str) -> Result<Vec<CollectionFile>, QueryError> {
+ pub async fn search(&self, query: &str) -> Result<Vec<CollectionFile>, Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let mut connection = self.db.connect().await?;
let like_test = format!("%{}%", query);
@@ -144,7 +137,7 @@ impl Index {
// Find dirs with matching path and parent not matching
{
let real_directories = sqlx::query_as!(
- Directory,
+ scanner::Directory,
"SELECT * FROM directories WHERE path LIKE $1 AND parent NOT LIKE $1",
like_test
)
@@ -161,7 +154,7 @@ impl Index {
// Find songs with matching title/album/artist and non-matching parent
{
let real_songs = sqlx::query_as!(
- Song,
+ scanner::Song,
r#"
SELECT * FROM songs
WHERE ( path LIKE $1
@@ -185,7 +178,7 @@ impl Index {
Ok(output)
}
- pub async fn get_song(&self, virtual_path: &Path) -> Result<Song, QueryError> {
+ pub async fn get_song(&self, virtual_path: &Path) -> Result<scanner::Song, Error> {
let vfs = self.vfs_manager.get_vfs().await?;
let mut connection = self.db.connect().await?;
@@ -193,7 +186,7 @@ impl Index {
let real_path_string = real_path.as_path().to_string_lossy();
let real_song = sqlx::query_as!(
- Song,
+ scanner::Song,
"SELECT * FROM songs WHERE path = $1",
real_path_string
)
@@ -202,7 +195,7 @@ impl Index {
match real_song.virtualize(&vfs) {
Some(s) => Ok(s),
- None => Err(QueryError::SongNotFound(real_path)),
+ None => Err(Error::SongNotFound(real_path)),
}
}
}
diff --git a/src/app/index/test.rs b/src/app/index/test.rs
index bf2dcfc..ad6850f 100644
--- a/src/app/index/test.rs
+++ b/src/app/index/test.rs
@@ -1,96 +1,18 @@
-use std::default::Default;
use std::path::{Path, PathBuf};
use super::*;
-use crate::app::test;
+use crate::app::{scanner, test};
use crate::test_name;
const TEST_MOUNT_NAME: &str = "root";
-#[tokio::test]
-async fn update_adds_new_content() {
- let ctx = test::ContextBuilder::new(test_name!())
- .mount(TEST_MOUNT_NAME, "test-data/small-collection")
- .build()
- .await;
-
- ctx.index.update().await.unwrap();
- ctx.index.update().await.unwrap(); // Validates that subsequent updates don't run into conflicts
-
- let mut connection = ctx.db.connect().await.unwrap();
- let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- assert_eq!(all_directories.len(), 6);
- assert_eq!(all_songs.len(), 13);
-}
-
-#[tokio::test]
-async fn update_removes_missing_content() {
- let builder = test::ContextBuilder::new(test_name!());
-
- let original_collection_dir: PathBuf = ["test-data", "small-collection"].iter().collect();
- let test_collection_dir: PathBuf = builder.test_directory.join("small-collection");
-
- let copy_options = fs_extra::dir::CopyOptions::new();
- fs_extra::dir::copy(
- original_collection_dir,
- &builder.test_directory,
- ©_options,
- )
- .unwrap();
-
- let ctx = builder
- .mount(TEST_MOUNT_NAME, test_collection_dir.to_str().unwrap())
- .build()
- .await;
-
- ctx.index.update().await.unwrap();
-
- {
- let mut connection = ctx.db.connect().await.unwrap();
- let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- assert_eq!(all_directories.len(), 6);
- assert_eq!(all_songs.len(), 13);
- }
-
- let khemmis_directory = test_collection_dir.join("Khemmis");
- std::fs::remove_dir_all(khemmis_directory).unwrap();
- ctx.index.update().await.unwrap();
- {
- let mut connection = ctx.db.connect().await.unwrap();
- let all_directories = sqlx::query_as!(Directory, "SELECT * FROM directories")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- let all_songs = sqlx::query_as!(Song, "SELECT * FROM songs")
- .fetch_all(connection.as_mut())
- .await
- .unwrap();
- assert_eq!(all_directories.len(), 4);
- assert_eq!(all_songs.len(), 8);
- }
-}
-
#[tokio::test]
async fn can_browse_top_level() {
let ctx = test::ContextBuilder::new(test_name!())
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let root_path = Path::new(TEST_MOUNT_NAME);
let files = ctx.index.browse(Path::new("")).await.unwrap();
@@ -110,7 +32,7 @@ async fn can_browse_directory() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let files = ctx.index.browse(Path::new(TEST_MOUNT_NAME)).await.unwrap();
@@ -132,7 +54,7 @@ async fn can_flatten_root() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let songs = ctx.index.flatten(Path::new(TEST_MOUNT_NAME)).await.unwrap();
assert_eq!(songs.len(), 13);
assert_eq!(songs[0].title, Some("Above The Water".to_owned()));
@@ -144,7 +66,7 @@ async fn can_flatten_directory() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao"].iter().collect();
let songs = ctx.index.flatten(path).await.unwrap();
assert_eq!(songs.len(), 8);
@@ -156,7 +78,7 @@ async fn can_flatten_directory_with_shared_prefix() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let path: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect(); // Prefix of '(Picnic Remixes)'
let songs = ctx.index.flatten(path).await.unwrap();
assert_eq!(songs.len(), 7);
@@ -168,7 +90,7 @@ async fn can_get_random_albums() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let albums = ctx.index.get_random_albums(1).await.unwrap();
assert_eq!(albums.len(), 1);
}
@@ -179,7 +101,7 @@ async fn can_get_recent_albums() {
.mount(TEST_MOUNT_NAME, "test-data/small-collection")
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let albums = ctx.index.get_recent_albums(2).await.unwrap();
assert_eq!(albums.len(), 2);
assert!(albums[0].date_added >= albums[1].date_added);
@@ -192,7 +114,7 @@ async fn can_get_a_song() {
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
let song_virtual_path = picnic_virtual_dir.join("05 - シャーベット (Sherbet).mp3");
@@ -203,8 +125,11 @@ async fn can_get_a_song() {
assert_eq!(song.track_number, Some(5));
assert_eq!(song.disc_number, None);
assert_eq!(song.title, Some("シャーベット (Sherbet)".to_owned()));
- assert_eq!(song.artists, MultiString(vec!["Tobokegao".to_owned()]));
- assert_eq!(song.album_artists, MultiString(vec![]));
+ assert_eq!(
+ song.artists,
+ scanner::MultiString(vec!["Tobokegao".to_owned()])
+ );
+ assert_eq!(song.album_artists, scanner::MultiString(vec![]));
assert_eq!(song.album, Some("Picnic".to_owned()));
assert_eq!(song.year, Some(2016));
assert_eq!(
@@ -212,51 +137,3 @@ async fn can_get_a_song() {
Some(artwork_virtual_path.to_string_lossy().into_owned())
);
}
-
-#[tokio::test]
-async fn indexes_embedded_artwork() {
- let ctx = test::ContextBuilder::new(test_name!())
- .mount(TEST_MOUNT_NAME, "test-data/small-collection")
- .build()
- .await;
-
- ctx.index.update().await.unwrap();
-
- let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
- let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3");
-
- let song = ctx.index.get_song(&song_virtual_path).await.unwrap();
- assert_eq!(
- song.artwork,
- Some(song_virtual_path.to_string_lossy().into_owned())
- );
-}
-
-#[tokio::test]
-async fn album_art_pattern_is_case_insensitive() {
- let ctx = test::ContextBuilder::new(test_name!())
- .mount(TEST_MOUNT_NAME, "test-data/small-collection")
- .build()
- .await;
-
- let patterns = vec!["folder", "FOLDER"];
-
- for pattern in patterns.into_iter() {
- ctx.settings_manager
- .amend(&settings::NewSettings {
- album_art_pattern: Some(pattern.to_owned()),
- ..Default::default()
- })
- .await
- .unwrap();
- ctx.index.update().await.unwrap();
-
- let hunted_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect();
- let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg");
- let song = &ctx.index.flatten(&hunted_virtual_dir).await.unwrap()[0];
- assert_eq!(
- song.artwork,
- Some(artwork_virtual_path.to_string_lossy().into_owned())
- );
- }
-}
diff --git a/src/app/index/types.rs b/src/app/index/types.rs
index 0c19dcb..136ef14 100644
--- a/src/app/index/types.rs
+++ b/src/app/index/types.rs
@@ -1,76 +1,24 @@
-use std::path::Path;
+use std::path::PathBuf;
-use crate::app::vfs::VFS;
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct MultiString(pub Vec<String>);
+use crate::{
+ app::{scanner, vfs},
+ db,
+};
#[derive(Debug, PartialEq, Eq)]
pub enum CollectionFile {
- Directory(Directory),
- Song(Song),
+ Directory(scanner::Directory),
+ Song(scanner::Song),
}
-#[derive(Debug, PartialEq, Eq)]
-pub struct Song {
- pub id: i64,
- pub path: String,
- pub parent: String,
- pub track_number: Option<i64>,
- pub disc_number: Option<i64>,
- pub title: Option<String>,
- pub artists: MultiString,
- pub album_artists: MultiString,
- pub year: Option<i64>,
- pub album: Option<String>,
- pub artwork: Option<String>,
- pub duration: Option<i64>,
- pub lyricists: MultiString,
- pub composers: MultiString,
- pub genres: MultiString,
- pub labels: MultiString,
-}
-
-impl Song {
- pub fn virtualize(mut self, vfs: &VFS) -> Option<Song> {
- self.path = match vfs.real_to_virtual(Path::new(&self.path)) {
- Ok(p) => p.to_string_lossy().into_owned(),
- _ => return None,
- };
- if let Some(artwork_path) = self.artwork {
- self.artwork = match vfs.real_to_virtual(Path::new(&artwork_path)) {
- Ok(p) => Some(p.to_string_lossy().into_owned()),
- _ => None,
- };
- }
- Some(self)
- }
-}
-
-#[derive(Debug, PartialEq, Eq)]
-pub struct Directory {
- pub id: i64,
- pub path: String,
- pub parent: Option<String>,
- pub artists: MultiString,
- pub year: Option<i64>,
- pub album: Option<String>,
- pub artwork: Option<String>,
- pub date_added: i64,
-}
-
-impl Directory {
- pub fn virtualize(mut self, vfs: &VFS) -> Option<Directory> {
- self.path = match vfs.real_to_virtual(Path::new(&self.path)) {
- Ok(p) => p.to_string_lossy().into_owned(),
- _ => return None,
- };
- if let Some(artwork_path) = self.artwork {
- self.artwork = match vfs.real_to_virtual(Path::new(&artwork_path)) {
- Ok(p) => Some(p.to_string_lossy().into_owned()),
- _ => None,
- };
- }
- Some(self)
- }
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error(transparent)]
+ Database(#[from] sqlx::Error),
+ #[error(transparent)]
+ DatabaseConnection(#[from] db::Error),
+ #[error("Song was not found: `{0}`")]
+ SongNotFound(PathBuf),
+ #[error(transparent)]
+ Vfs(#[from] vfs::Error),
}
diff --git a/src/app/lastfm.rs b/src/app/lastfm.rs
index f20a311..d0d47e2 100644
--- a/src/app/lastfm.rs
+++ b/src/app/lastfm.rs
@@ -3,7 +3,7 @@ use std::path::Path;
use user::AuthToken;
use crate::app::{
- index::{Index, QueryError},
+ index::{self, Index},
user,
};
@@ -19,7 +19,7 @@ pub enum Error {
#[error("Failed to emit last.fm now playing update")]
NowPlaying(rustfm_scrobble::ScrobblerError),
#[error(transparent)]
- Query(#[from] QueryError),
+ Query(#[from] index::Error),
#[error(transparent)]
User(#[from] user::Error),
}
diff --git a/src/app/playlist.rs b/src/app/playlist.rs
index 17baf6b..0a33cbd 100644
--- a/src/app/playlist.rs
+++ b/src/app/playlist.rs
@@ -1,7 +1,7 @@
use core::clone::Clone;
use sqlx::{Acquire, QueryBuilder, Sqlite};
-use crate::app::index::Song;
+use crate::app::scanner::Song;
use crate::app::vfs;
use crate::db::{self, DB};
@@ -237,7 +237,7 @@ mod test {
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let playlist_content: Vec<String> = ctx
.index
@@ -302,7 +302,7 @@ mod test {
.build()
.await;
- ctx.index.update().await.unwrap();
+ ctx.scanner.scan().await.unwrap();
let playlist_content: Vec<String> = ctx
.index
diff --git a/src/app/scanner.rs b/src/app/scanner.rs
new file mode 100644
index 0000000..77ca3f6
--- /dev/null
+++ b/src/app/scanner.rs
@@ -0,0 +1,123 @@
+use log::{error, info};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+use tokio::sync::Notify;
+
+use crate::app::{settings, vfs};
+use crate::db::DB;
+
+mod cleaner;
+mod collector;
+mod inserter;
+mod metadata;
+#[cfg(test)]
+mod test;
+mod traverser;
+mod types;
+
+pub use self::types::*;
+
+#[derive(Clone)]
+pub struct Scanner {
+ db: DB,
+ vfs_manager: vfs::Manager,
+ settings_manager: settings::Manager,
+ pending_scan: Arc<Notify>,
+}
+
+impl Scanner {
+ pub fn new(db: DB, vfs_manager: vfs::Manager, settings_manager: settings::Manager) -> Self {
+ let scanner = Self {
+ db,
+ vfs_manager,
+ settings_manager,
+ pending_scan: Arc::new(Notify::new()),
+ };
+
+ tokio::spawn({
+ let scanner = scanner.clone();
+ async move {
+ loop {
+ scanner.pending_scan.notified().await;
+ if let Err(e) = scanner.scan().await {
+ error!("Error while updating index: {}", e);
+ }
+ }
+ }
+ });
+
+ scanner
+ }
+
+ pub fn trigger_scan(&self) {
+ self.pending_scan.notify_one();
+ }
+
+ pub fn begin_periodic_scans(&self) {
+ tokio::spawn({
+ let index = self.clone();
+ async move {
+ loop {
+ index.trigger_scan();
+ let sleep_duration = index
+ .settings_manager
+ .get_index_sleep_duration()
+ .await
+ .unwrap_or_else(|e| {
+ error!("Could not retrieve index sleep duration: {}", e);
+ Duration::from_secs(1800)
+ });
+ tokio::time::sleep(sleep_duration).await;
+ }
+ }
+ });
+ }
+
+ pub async fn scan(&self) -> Result<(), types::Error> {
+ let start = Instant::now();
+ info!("Beginning library index update");
+
+ let album_art_pattern = self
+ .settings_manager
+ .get_index_album_art_pattern()
+ .await
+ .ok();
+
+ let cleaner = cleaner::Cleaner::new(self.db.clone(), self.vfs_manager.clone());
+ cleaner.clean().await?;
+
+ let (insert_sender, insert_receiver) = tokio::sync::mpsc::unbounded_channel();
+ let insertion = tokio::spawn({
+ let db = self.db.clone();
+ async {
+ let mut inserter = inserter::Inserter::new(db, insert_receiver);
+ inserter.insert().await;
+ }
+ });
+
+ let (collect_sender, collect_receiver) = crossbeam_channel::unbounded();
+ let collection = tokio::task::spawn_blocking(|| {
+ let collector =
+ collector::Collector::new(collect_receiver, insert_sender, album_art_pattern);
+ collector.collect();
+ });
+
+ let vfs = self.vfs_manager.get_vfs().await?;
+ let traversal = tokio::task::spawn_blocking(move || {
+ let mounts = vfs.mounts();
+ let traverser = traverser::Traverser::new(collect_sender);
+ traverser.traverse(mounts.iter().map(|p| p.source.clone()).collect());
+ });
+
+ traversal.await.unwrap();
+ collection.await.unwrap();
+ insertion.await.unwrap();
+
+ info!(
+ "Library index update took {} seconds",
+ start.elapsed().as_millis() as f32 / 1000.0
+ );
+
+ Ok(())
+ }
+}
diff --git a/src/app/index/update/cleaner.rs b/src/app/scanner/cleaner.rs
similarity index 100%
rename from src/app/index/update/cleaner.rs
rename to src/app/scanner/cleaner.rs
diff --git a/src/app/index/update/collector.rs b/src/app/scanner/collector.rs
similarity index 98%
rename from src/app/index/update/collector.rs
rename to src/app/scanner/collector.rs
index 1acf7d5..15d1fd9 100644
--- a/src/app/index/update/collector.rs
+++ b/src/app/scanner/collector.rs
@@ -1,9 +1,10 @@
use log::error;
use regex::Regex;
-use crate::app::index::MultiString;
+use crate::app::scanner::MultiString;
-use super::*;
+use super::inserter;
+use super::traverser;
pub struct Collector {
receiver: crossbeam_channel::Receiver<traverser::Directory>,
diff --git a/src/app/index/update/inserter.rs b/src/app/scanner/inserter.rs
similarity index 80%
rename from src/app/index/update/inserter.rs
rename to src/app/scanner/inserter.rs
index 09f3517..8d24c56 100644
--- a/src/app/index/update/inserter.rs
+++ b/src/app/scanner/inserter.rs
@@ -1,14 +1,8 @@
-use std::borrow::Cow;
-
use log::error;
-use sqlx::{
- encode::IsNull,
- sqlite::{SqliteArgumentValue, SqliteTypeInfo},
- QueryBuilder, Sqlite,
-};
+use sqlx::{QueryBuilder, Sqlite};
use tokio::sync::mpsc::UnboundedReceiver;
-use crate::{app::index::MultiString, db::DB};
+use crate::{app::scanner::MultiString, db::DB};
const INDEX_BUILDING_INSERT_BUFFER_SIZE: usize = 1000; // Insertions in each transaction
@@ -52,39 +46,6 @@ pub struct Inserter {
db: DB,
}
-static MULTI_STRING_SEPARATOR: &str = "\u{000C}";
-
-impl<'q> sqlx::Encode<'q, Sqlite> for MultiString {
- fn encode_by_ref(&self, args: &mut Vec<SqliteArgumentValue<'q>>) -> IsNull {
- if self.0.is_empty() {
- IsNull::Yes
- } else {
- let joined = self.0.join(MULTI_STRING_SEPARATOR);
- args.push(SqliteArgumentValue::Text(Cow::Owned(joined)));
- IsNull::No
- }
- }
-}
-
-impl From<Option<String>> for MultiString {
- fn from(value: Option<String>) -> Self {
- match value {
- None => MultiString(Vec::new()),
- Some(s) => MultiString(
- s.split(MULTI_STRING_SEPARATOR)
- .map(|s| s.to_string())
- .collect(),
- ),
- }
- }
-}
-
-impl sqlx::Type<Sqlite> for MultiString {
- fn type_info() -> SqliteTypeInfo {
- <&str as sqlx::Type<Sqlite>>::type_info()
- }
-}
-
impl Inserter {
pub fn new(db: DB, receiver: UnboundedReceiver<Item>) -> Self {
let new_directories = Vec::with_capacity(INDEX_BUILDING_INSERT_BUFFER_SIZE);
diff --git a/src/app/index/metadata.rs b/src/app/scanner/metadata.rs
similarity index 100%
rename from src/app/index/metadata.rs
rename to src/app/scanner/metadata.rs
diff --git a/src/app/index/update.rs b/src/app/scanner/scan.rs
similarity index 93%
rename from src/app/index/update.rs
rename to src/app/scanner/scan.rs
index 8064227..2e8e4a2 100644
--- a/src/app/index/update.rs
+++ b/src/app/scanner/scan.rs
@@ -1,11 +1,6 @@
use log::{error, info};
use std::time;
-mod cleaner;
-mod collector;
-mod inserter;
-mod traverser;
-
use crate::app::index::Index;
use crate::app::vfs;
use crate::db;
@@ -27,8 +22,8 @@ pub enum Error {
Vfs(#[from] vfs::Error),
}
-impl Index {
- pub async fn update(&self) -> Result<(), Error> {
+impl Scanner {
+ pub async fn scan(&self) -> Result<(), Error> {
let start = time::Instant::now();
info!("Beginning library index update");
diff --git a/src/app/scanner/test.rs b/src/app/scanner/test.rs
new file mode 100644
index 0000000..925d844
--- /dev/null
+++ b/src/app/scanner/test.rs
@@ -0,0 +1,133 @@
+use std::path::PathBuf;
+
+use crate::{
+ app::{scanner, settings, test},
+ test_name,
+};
+
+const TEST_MOUNT_NAME: &str = "root";
+
+#[tokio::test]
+async fn scan_adds_new_content() {
+ let ctx = test::ContextBuilder::new(test_name!())
+ .mount(TEST_MOUNT_NAME, "test-data/small-collection")
+ .build()
+ .await;
+
+ ctx.scanner.scan().await.unwrap();
+ ctx.scanner.scan().await.unwrap(); // Validates that subsequent updates don't run into conflicts
+
+ let mut connection = ctx.db.connect().await.unwrap();
+ let all_directories = sqlx::query_as!(scanner::Directory, "SELECT * FROM directories")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ let all_songs = sqlx::query_as!(scanner::Song, "SELECT * FROM songs")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ assert_eq!(all_directories.len(), 6);
+ assert_eq!(all_songs.len(), 13);
+}
+
+#[tokio::test]
+async fn scan_removes_missing_content() {
+ let builder = test::ContextBuilder::new(test_name!());
+
+ let original_collection_dir: PathBuf = ["test-data", "small-collection"].iter().collect();
+ let test_collection_dir: PathBuf = builder.test_directory.join("small-collection");
+
+ let copy_options = fs_extra::dir::CopyOptions::new();
+ fs_extra::dir::copy(
+ original_collection_dir,
+ &builder.test_directory,
+ ©_options,
+ )
+ .unwrap();
+
+ let ctx = builder
+ .mount(TEST_MOUNT_NAME, test_collection_dir.to_str().unwrap())
+ .build()
+ .await;
+
+ ctx.scanner.scan().await.unwrap();
+
+ {
+ let mut connection = ctx.db.connect().await.unwrap();
+ let all_directories = sqlx::query_as!(scanner::Directory, "SELECT * FROM directories")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ let all_songs = sqlx::query_as!(scanner::Song, "SELECT * FROM songs")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ assert_eq!(all_directories.len(), 6);
+ assert_eq!(all_songs.len(), 13);
+ }
+
+ let khemmis_directory = test_collection_dir.join("Khemmis");
+ std::fs::remove_dir_all(khemmis_directory).unwrap();
+ ctx.scanner.scan().await.unwrap();
+ {
+ let mut connection = ctx.db.connect().await.unwrap();
+ let all_directories = sqlx::query_as!(scanner::Directory, "SELECT * FROM directories")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ let all_songs = sqlx::query_as!(scanner::Song, "SELECT * FROM songs")
+ .fetch_all(connection.as_mut())
+ .await
+ .unwrap();
+ assert_eq!(all_directories.len(), 4);
+ assert_eq!(all_songs.len(), 8);
+ }
+}
+
+#[tokio::test]
+async fn finds_embedded_artwork() {
+ let ctx = test::ContextBuilder::new(test_name!())
+ .mount(TEST_MOUNT_NAME, "test-data/small-collection")
+ .build()
+ .await;
+
+ ctx.scanner.scan().await.unwrap();
+
+ let picnic_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Tobokegao", "Picnic"].iter().collect();
+ let song_virtual_path = picnic_virtual_dir.join("07 - なぜ (Why).mp3");
+
+ let song = ctx.index.get_song(&song_virtual_path).await.unwrap();
+ assert_eq!(
+ song.artwork,
+ Some(song_virtual_path.to_string_lossy().into_owned())
+ );
+}
+
+#[tokio::test]
+async fn album_art_pattern_is_case_insensitive() {
+ let ctx = test::ContextBuilder::new(test_name!())
+ .mount(TEST_MOUNT_NAME, "test-data/small-collection")
+ .build()
+ .await;
+
+ let patterns = vec!["folder", "FOLDER"];
+
+ for pattern in patterns.into_iter() {
+ ctx.settings_manager
+ .amend(&settings::NewSettings {
+ album_art_pattern: Some(pattern.to_owned()),
+ ..Default::default()
+ })
+ .await
+ .unwrap();
+ ctx.scanner.scan().await.unwrap();
+
+ let hunted_virtual_dir: PathBuf = [TEST_MOUNT_NAME, "Khemmis", "Hunted"].iter().collect();
+ let artwork_virtual_path = hunted_virtual_dir.join("Folder.jpg");
+ let song = &ctx.index.flatten(&hunted_virtual_dir).await.unwrap()[0];
+ assert_eq!(
+ song.artwork,
+ Some(artwork_virtual_path.to_string_lossy().into_owned())
+ );
+ }
+}
diff --git a/src/app/index/update/traverser.rs b/src/app/scanner/traverser.rs
similarity index 98%
rename from src/app/index/update/traverser.rs
rename to src/app/scanner/traverser.rs
index 9084fe4..e79650d 100644
--- a/src/app/index/update/traverser.rs
+++ b/src/app/scanner/traverser.rs
@@ -9,7 +9,7 @@ use std::sync::Arc;
use std::thread;
use std::time::Duration;
-use crate::app::index::metadata::{self, SongMetadata};
+use crate::app::scanner::metadata::{self, SongMetadata};
#[derive(Debug)]
pub struct Song {
diff --git a/src/app/scanner/types.rs b/src/app/scanner/types.rs
new file mode 100644
index 0000000..47a7577
--- /dev/null
+++ b/src/app/scanner/types.rs
@@ -0,0 +1,124 @@
+use std::{borrow::Cow, path::Path};
+
+use sqlx::{
+ encode::IsNull,
+ sqlite::{SqliteArgumentValue, SqliteTypeInfo},
+ Sqlite,
+};
+
+use crate::{
+ app::vfs::{self, VFS},
+ db,
+};
+
+#[derive(thiserror::Error, Debug)]
+pub enum Error {
+ #[error(transparent)]
+ IndexClean(#[from] super::cleaner::Error),
+ #[error(transparent)]
+ Database(#[from] sqlx::Error),
+ #[error(transparent)]
+ DatabaseConnection(#[from] db::Error),
+ #[error(transparent)]
+ Vfs(#[from] vfs::Error),
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct MultiString(pub Vec<String>);
+
+static MULTI_STRING_SEPARATOR: &str = "\u{000C}";
+
+impl<'q> sqlx::Encode<'q, Sqlite> for MultiString {
+ fn encode_by_ref(&self, args: &mut Vec<SqliteArgumentValue<'q>>) -> IsNull {
+ if self.0.is_empty() {
+ IsNull::Yes
+ } else {
+ let joined = self.0.join(MULTI_STRING_SEPARATOR);
+ args.push(SqliteArgumentValue::Text(Cow::Owned(joined)));
+ IsNull::No
+ }
+ }
+}
+
+impl From<Option<String>> for MultiString {
+ fn from(value: Option<String>) -> Self {
+ match value {
+ None => MultiString(Vec::new()),
+ Some(s) => MultiString(
+ s.split(MULTI_STRING_SEPARATOR)
+ .map(|s| s.to_string())
+ .collect(),
+ ),
+ }
+ }
+}
+
+impl sqlx::Type<Sqlite> for MultiString {
+ fn type_info() -> SqliteTypeInfo {
+ <&str as sqlx::Type<Sqlite>>::type_info()
+ }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Song {
+ pub id: i64,
+ pub path: String,
+ pub parent: String,
+ pub track_number: Option<i64>,
+ pub disc_number: Option<i64>,
+ pub title: Option<String>,
+ pub artists: MultiString,
+ pub album_artists: MultiString,
+ pub year: Option<i64>,
+ pub album: Option<String>,
+ pub artwork: Option<String>,
+ pub duration: Option<i64>,
+ pub lyricists: MultiString,
+ pub composers: MultiString,
+ pub genres: MultiString,
+ pub labels: MultiString,
+}
+
+impl Song {
+ pub fn virtualize(mut self, vfs: &VFS) -> Option<Song> {
+ self.path = match vfs.real_to_virtual(Path::new(&self.path)) {
+ Ok(p) => p.to_string_lossy().into_owned(),
+ _ => return None,
+ };
+ if let Some(artwork_path) = self.artwork {
+ self.artwork = match vfs.real_to_virtual(Path::new(&artwork_path)) {
+ Ok(p) => Some(p.to_string_lossy().into_owned()),
+ _ => None,
+ };
+ }
+ Some(self)
+ }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct Directory {
+ pub id: i64,
+ pub path: String,
+ pub parent: Option<String>,
+ pub artists: MultiString,
+ pub year: Option<i64>,
+ pub album: Option<String>,
+ pub artwork: Option<String>,
+ pub date_added: i64,
+}
+
+impl Directory {
+ pub fn virtualize(mut self, vfs: &VFS) -> Option<Directory> {
+ self.path = match vfs.real_to_virtual(Path::new(&self.path)) {
+ Ok(p) => p.to_string_lossy().into_owned(),
+ _ => return None,
+ };
+ if let Some(artwork_path) = self.artwork {
+ self.artwork = match vfs.real_to_virtual(Path::new(&artwork_path)) {
+ Ok(p) => Some(p.to_string_lossy().into_owned()),
+ _ => None,
+ };
+ }
+ Some(self)
+ }
+}
diff --git a/src/app/test.rs b/src/app/test.rs
index a00628c..1c25089 100644
--- a/src/app/test.rs
+++ b/src/app/test.rs
@@ -1,11 +1,12 @@
use std::path::PathBuf;
-use crate::app::{config, ddns, index::Index, playlist, settings, user, vfs};
+use crate::app::{config, ddns, index::Index, playlist, scanner::Scanner, settings, user, vfs};
use crate::db::DB;
use crate::test::*;
pub struct Context {
pub db: DB,
+ pub scanner: Scanner,
pub index: Index,
pub config_manager: config::Manager,
pub ddns_manager: ddns::Manager,
@@ -65,13 +66,15 @@ impl ContextBuilder {
vfs_manager.clone(),
ddns_manager.clone(),
);
- let index = Index::new(db.clone(), vfs_manager.clone(), settings_manager.clone());
+ let scanner = Scanner::new(db.clone(), vfs_manager.clone(), settings_manager.clone());
+ let index = Index::new(db.clone(), vfs_manager.clone());
let playlist_manager = playlist::Manager::new(db.clone(), vfs_manager.clone());
config_manager.apply(&self.config).await.unwrap();
Context {
db,
+ scanner,
index,
config_manager,
ddns_manager,
diff --git a/src/main.rs b/src/main.rs
index eb8ffe7..ab02616 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -144,7 +144,7 @@ fn main() -> Result<(), Error> {
async fn async_main(cli_options: CLIOptions, paths: paths::Paths) -> Result<(), Error> {
// Create and run app
let app = app::App::new(cli_options.port.unwrap_or(5050), paths).await?;
- app.index.begin_periodic_updates();
+ app.scanner.begin_periodic_scans();
app.ddns_manager.begin_periodic_updates();
// Start server
diff --git a/src/server/axum.rs b/src/server/axum.rs
index 58bd4cd..a0a9352 100644
--- a/src/server/axum.rs
+++ b/src/server/axum.rs
@@ -62,6 +62,12 @@ impl FromRef<App> for app::user::Manager {
}
}
+impl FromRef<App> for app::scanner::Scanner {
+ fn from_ref(app: &App) -> Self {
+ app.scanner.clone()
+ }
+}
+
impl FromRef<App> for app::settings::Manager {
fn from_ref(app: &App) -> Self {
app.settings_manager.clone()
diff --git a/src/server/axum/api.rs b/src/server/axum/api.rs
index b6d203c..0cdad7c 100644
--- a/src/server/axum/api.rs
+++ b/src/server/axum/api.rs
@@ -11,7 +11,7 @@ use base64::{prelude::BASE64_STANDARD_NO_PAD, Engine};
use percent_encoding::percent_decode_str;
use crate::{
- app::{config, ddns, index, lastfm, playlist, settings, thumbnail, user, vfs, App},
+ app::{config, ddns, index, lastfm, playlist, scanner, settings, thumbnail, user, vfs, App},
server::{dto, error::APIError},
};
@@ -246,9 +246,9 @@ async fn put_preferences(
async fn post_trigger_index(
_admin_rights: AdminRights,
- State(index): State<index::Index>,
+ State(scanner): State<scanner::Scanner>,
) -> Result<(), APIError> {
- index.trigger_reindex();
+ scanner.trigger_scan();
Ok(())
}
diff --git a/src/server/dto.rs b/src/server/dto.rs
index 566be2f..e052c9c 100644
--- a/src/server/dto.rs
+++ b/src/server/dto.rs
@@ -1,6 +1,6 @@
use serde::{Deserialize, Serialize};
-use crate::app::{config, ddns, index, settings, thumbnail, user, vfs};
+use crate::app::{config, ddns, index, scanner, settings, thumbnail, user, vfs};
use std::convert::From;
pub const API_MAJOR_VERSION: i32 = 8;
@@ -277,8 +277,8 @@ pub struct Song {
pub labels: Vec<String>,
}
-impl From<index::Song> for Song {
- fn from(s: index::Song) -> Self {
+impl From<scanner::Song> for Song {
+ fn from(s: scanner::Song) -> Self {
Self {
path: s.path,
track_number: s.track_number,
@@ -312,8 +312,8 @@ pub struct Directory {
pub date_added: i64,
}
-impl From<index::Directory> for Directory {
- fn from(d: index::Directory) -> Self {
+impl From<scanner::Directory> for Directory {
+ fn from(d: scanner::Directory) -> Self {
Self {
path: d.path,
artists: d.artists.0,
diff --git a/src/server/error.rs b/src/server/error.rs
index af19618..3c2941c 100644
--- a/src/server/error.rs
+++ b/src/server/error.rs
@@ -1,8 +1,7 @@
use std::path::PathBuf;
use thiserror::Error;
-use crate::app::index::QueryError;
-use crate::app::{config, ddns, lastfm, playlist, settings, thumbnail, user, vfs};
+use crate::app::{config, ddns, index, lastfm, playlist, settings, thumbnail, user, vfs};
use crate::db;
#[derive(Error, Debug)]
@@ -102,13 +101,13 @@ impl From<playlist::Error> for APIError {
}
}
-impl From<QueryError> for APIError {
- fn from(error: QueryError) -> APIError {
+impl From<index::Error> for APIError {
+ fn from(error: index::Error) -> APIError {
match error {
- QueryError::Database(e) => APIError::Database(e),
- QueryError::DatabaseConnection(e) => e.into(),
- QueryError::SongNotFound(_) => APIError::SongMetadataNotFound,
- QueryError::Vfs(e) => e.into(),
+ index::Error::Database(e) => APIError::Database(e),
+ index::Error::DatabaseConnection(e) => e.into(),
+ index::Error::SongNotFound(_) => APIError::SongMetadataNotFound,
+ index::Error::Vfs(e) => e.into(),
}
}
}