Index songs and directories to DB

.gitignore

@@ -2,5 +2,8 @@ target
 release
 *.dll
 *.res
+*.sqlite
+*.sqlite-wal
+*.sqlite-shm
 tmp
 TestConfig.toml

src/collection.rs

@@ -1,7 +1,6 @@
 use std::fs;
 use std::path::Path;
 use std::path::PathBuf;
-use id3::Tag;
 use regex::Regex;
 use std::sync::Arc;
 
@@ -28,16 +27,6 @@ pub struct Song {
     artist: Option<String>,
 }
 
-#[derive(Debug)]
-pub struct SongTags {
-    track_number: Option<u32>,
-    title: Option<String>,
-    artist: Option<String>,
-    album_artist: Option<String>,
-    album: Option<String>,
-    year: Option<i32>,
-}
-
 #[derive(Clone, Debug)]
 pub struct User {
     name: String,
@@ -66,31 +55,6 @@ impl Album {
             Some(a) => a.to_str().map(|p| p.to_string()),
         };
 
-        let mut song_path = None;
-        if real_path.is_file() {
-            song_path = Some(real_path.to_path_buf());
-        } else {
-            let find_song = try!(fs::read_dir(real_path)).find(|f| {
-                match *f {
-                    Ok(ref dir_entry) => is_song(dir_entry.path().as_path()),
-                    _ => false,
-                }
-            });
-            if let Some(dir_entry) = find_song {
-                song_path = Some(try!(dir_entry).path());
-            }
-        };
-
-        let song_tags = song_path.map(|p| SongTags::read(p.as_path()));
-        if let Some(Ok(t)) = song_tags {
-            let artist = t.album_artist.or(t.artist);
-            Ok(Album {
-                album_art: album_art,
-                title: t.album,
-                year: t.year,
-                artist: artist,
-            })
-        } else {
         Ok(Album {
             album_art: album_art,
             title: None,
@@ -99,49 +63,14 @@ impl Album {
         })
     }
 }
-}
-
-impl SongTags {
-    fn read(path: &Path) -> Result<SongTags, PError> {
-        let tag = try!(Tag::read_from_path(path));
-
-        let artist = tag.artist().map(|s| s.to_string());
-        let album_artist = tag.album_artist().map(|s| s.to_string());
-        let album = tag.album().map(|s| s.to_string());
-        let title = tag.title().map(|s| s.to_string());
-        let track_number = tag.track();
-        let year = tag.year()
-            .map(|y| y as i32)
-            .or(tag.date_released().and_then(|d| d.year))
-            .or(tag.date_recorded().and_then(|d| d.year));
-        Ok(SongTags {
-            artist: artist,
-            album_artist: album_artist,
-            album: album,
-            title: title,
-            track_number: track_number,
-            year: year,
-        })
-    }
-}
 
 impl Song {
     fn read(collection: &Collection, path: &Path) -> Result<Song, PError> {
         let virtual_path = try!(collection.vfs.real_to_virtual(path));
         let path_string = try!(virtual_path.to_str().ok_or(PError::PathDecoding));
 
-        let tags = SongTags::read(path).ok();
         let album = try!(Album::read(collection, path));
 
-        if let Some(t) = tags {
-            Ok(Song {
-                path: path_string.to_string(),
-                album: album,
-                artist: t.artist,
-                title: t.title,
-                track_number: t.track_number,
-            })
-        } else {
         Ok(Song {
             path: path_string.to_string(),
             album: album,
@@ -150,7 +79,6 @@ impl Song {
             track_number: None,
         })
     }
-    }
 
     
 }

src/index.rs

@@ -1,7 +1,8 @@
 use sqlite;
 use core::ops::Deref;
+use id3::Tag;
+use regex::Regex;
 use std::fs;
-use std::fs::DirBuilder;
 use std::path::Path;
 use std::sync::Arc;
 use std::thread;
@@ -12,97 +13,280 @@ use vfs::Vfs;
 
 pub struct Index {
 	path: String,
+	vfs: Arc<Vfs>,
+	album_art_pattern: Option<Regex>,
+}
+
+struct SongTags {
+    track_number: Option<u32>,
+    title: Option<String>,
+    artist: Option<String>,
+    album_artist: Option<String>,
+    album: Option<String>,
+    year: Option<i32>,
+}
+
+impl SongTags {
+    fn read(path: &Path) -> Result<SongTags, PError> {
+        let tag = try!(Tag::read_from_path(path));
+
+        let artist = tag.artist().map(|s| s.to_string());
+        let album_artist = tag.album_artist().map(|s| s.to_string());
+        let album = tag.album().map(|s| s.to_string());
+        let title = tag.title().map(|s| s.to_string());
+        let track_number = tag.track();
+        let year = tag.year()
+            .map(|y| y as i32)
+            .or(tag.date_released().and_then(|d| d.year))
+            .or(tag.date_recorded().and_then(|d| d.year));
+		
+        Ok(SongTags {
+            artist: artist,
+            album_artist: album_artist,
+            album: album,
+            title: title,
+            track_number: track_number,
+            year: year,
+        })
+    }
 }
 
 impl Index {
 
-	pub fn new(path: &Path) -> Result<Index, PError> {
+	pub fn new(path: &Path, vfs: Arc<Vfs>, album_art_pattern: &Option<Regex>) -> Result<Index, PError> {
 
-		// Create target directory
-		let mut dir_path = path.to_path_buf();
-		if dir_path.components().count() > 1 {
-			dir_path.pop();
-		}
-		let mut dir_builder = DirBuilder::new();
-		dir_builder.recursive(true);
-		dir_builder.create(dir_path).unwrap();
-
-		// Init Index
 		let index = Index {
 			path: path.to_string_lossy().deref().to_string(),
+			vfs: vfs,
+			album_art_pattern: album_art_pattern.clone(),
 		};
 
-		// Setup DB
 		if path.exists() {
-			match fs::remove_file(&index.path) {
+			// Migration
-				Err(_) => return Err(PError::CannotClearExistingIndex),
+		} else {
-				_ => (),
+			index.init();
-			}
 		}
 
-		let db = index.connect();
+		Ok(index)
+	}
+
+	fn init(&self) {
+
+		println!("Initializing index database");
+
+		let db = self.connect();
+		db.execute("PRAGMA synchronous = NORMAL").unwrap();
+		db.execute("PRAGMA journal_mode = WAL").unwrap();
 		db.execute("
 
-			CREATE TABLE artists
+			CREATE TABLE version
 			(	id INTEGER PRIMARY KEY NOT NULL
-			,	name TEXT NOT NULL
+			,	number INTEGER NULL
-			,	UNIQUE(name)
-			);
-
-			CREATE TABLE albums
-			(	id INTEGER PRIMARY KEY NOT NULL	
-			,	title TEXT NOT NULL
-			,	year INTEGER
-			,	artwork TEXT
-			,	artist INTEGER NOT NULL
-			,	FOREIGN KEY(artist) REFERENCES artists(id)
-			,	UNIQUE(artist, title)
 			);
+			INSERT INTO version (number) VALUES(1);
 
 			CREATE TABLE directories
-			(	path TEXT PRIMARY KEY NOT NULL
+			(	id INTEGER PRIMARY KEY NOT NULL
-			,	name TEXT NOT NULL
+			,	path NOT NULL
-			,	album INTEGER
+			,	artist TEXT
+			,	year INTEGER
+			,	album TEXT
 			,	artwork TEXT
-			,	FOREIGN KEY(album) REFERENCES albums(id)	
+			,	UNIQUE(path)
 			);
 
     		CREATE TABLE songs
-			(	path TEXT PRIMARY KEY NOT NULL
+			(	id INTEGER PRIMARY KEY NOT NULL
+			,	path NOT NULL
 			,	track_number INTEGER
 			,	title TEXT
-			,	artist INTEGER
+			,	artist TEXT
-			,	album INTEGER
+			,	album_artist TEXT
-			,	FOREIGN KEY(artist) REFERENCES artists(id)
+			,	year INTEGER
-			,	FOREIGN KEY(album) REFERENCES albums(id)
+			,	album TEXT
+			,	artwork TEXT
+			,	UNIQUE(path)
 			);
 
 		").unwrap();
-
-		Ok(index)
 	}
 
 	fn connect(&self) -> sqlite::Connection {
 		sqlite::open(self.path.clone()).unwrap()
 	}
 
-	fn refresh(&self, vfs: &Vfs) {
+	fn update_index(&self, db: &sqlite::Connection) {
-		let db = self.connect();
+		let start = time::Instant::now();
+		println!("Indexing library");
+		self.clean(db);
+		self.populate(db);
+		println!("Indexing library took {} seconds", start.elapsed().as_secs());	
+	}
+
+	fn clean(&self, db: &sqlite::Connection) {
+		{
+			let mut cursor = db.prepare("SELECT path FROM songs").unwrap().cursor();
+			let mut delete = db.prepare("DELETE FROM songs WHERE path = ?").unwrap();
+			while let Some(row) = cursor.next().unwrap() {
+				let path_string = row[0].as_string().unwrap();
+				let path = Path::new(path_string);
+				if !path.exists() {
+					delete.reset().ok();
+					delete.bind(1, &sqlite::Value::String(path_string.to_owned())).ok();
+					delete.next().ok();
+				}
+			}
+		}
+
+		{
+			let mut cursor = db.prepare("SELECT path FROM directories").unwrap().cursor();
+			let mut delete = db.prepare("DELETE FROM directories WHERE path = ?").unwrap();
+			while let Some(row) = cursor.next().unwrap() {
+				let path_string = row[0].as_string().unwrap();
+				let path = Path::new(path_string);
+				if !path.exists() {
+					delete.reset().ok();
+					delete.bind(1, &sqlite::Value::String(path_string.to_owned())).ok();
+					delete.next().ok();
+				}
+			}
+		}
+	}
+
+	fn populate(&self, db: &sqlite::Connection) {
+		let vfs = self.vfs.deref();
 		let mount_points = vfs.get_mount_points();
+
 		for (_, target) in mount_points {
 			self.populate_directory(&db, target.as_path());
 		}
 	}
 
+	fn get_artwork(&self, dir: &Path) -> Option<String> {
+		let pattern = match self.album_art_pattern {
+			Some(ref p) => p,
+			_ => return None,
+		};
+
+		if let Ok(dir_content) = fs::read_dir(dir) {
+			for file in dir_content {
+				if let Ok(file) = file {
+					if let Some(name_string) = file.file_name().to_str() {
+						if pattern.is_match(name_string) {
+							return file.path().to_str().map(|p| p.to_owned());
+						}
+					}
+				}
+			}
+		}
+
+		None
+	}
+
 	fn populate_directory(&self, db: &sqlite::Connection, path: &Path) {
 		
+		// Find artwork
+		let artwork = self.get_artwork(path).map_or(sqlite::Value::Null, |t| sqlite::Value::String(t));
+		
+		let mut directory_album = None;
+		let mut directory_year = None;
+		let mut directory_artist = None;
+		let mut inconsistent_directory_album = false;
+		let mut inconsistent_directory_year = false;
+		let mut inconsistent_directory_artist = false;
+
+		// Prepare statements
+		let mut insert_directory = db.prepare("
+			INSERT OR REPLACE INTO directories (path, artwork, year, artist, album)
+			VALUES (?, ?, ?, ?, ?)
+		").unwrap();
+
+		let mut insert_song = db.prepare("
+			INSERT OR REPLACE INTO songs (path, track_number, title, year, album_artist, artist, album, artwork)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+		").unwrap();
+
+		// Insert content
+		if let Ok(dir_content) = fs::read_dir(path) {
+			for file in dir_content {
+				let file_path = match file {
+					Ok(f) => f.path(),
+					_ => continue,
+				};
+
+				if file_path.is_dir() {
+					self.populate_directory(db, file_path.as_path());
+				} else {
+					if let Some(file_path_string) = file_path.to_str() {
+						if let Ok(tags) = SongTags::read(file_path.as_path()) {
+							if tags.year.is_some() {
+								inconsistent_directory_year |= directory_year.is_some() && directory_year != tags.year;
+								directory_year = tags.year;
 							}
 
+							if tags.album.is_some() {
+								inconsistent_directory_album |= directory_album.is_some() && directory_album != tags.album;
+								directory_album = Some(tags.album.as_ref().unwrap().clone());
 							}
 
-pub fn run(vfs: Arc<Vfs>, index: Arc<Index>) {
+							if tags.album_artist.is_some() {
+								inconsistent_directory_artist |= directory_artist.is_some() && directory_artist != tags.album_artist;
+								directory_artist = Some(tags.album_artist.as_ref().unwrap().clone());
+							} else if tags.artist.is_some() {
+								inconsistent_directory_artist |= directory_artist.is_some() && directory_artist != tags.artist;
+								directory_artist = Some(tags.artist.as_ref().unwrap().clone());
+							}
+
+							insert_song.reset().ok();
+							insert_song.bind(1, &sqlite::Value::String(file_path_string.to_owned())).unwrap();
+							insert_song.bind(2, &tags.track_number.map_or(sqlite::Value::Null, |t| sqlite::Value::Integer(t as i64))).unwrap();
+							insert_song.bind(3, &tags.title.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+							insert_song.bind(4, &tags.year.map_or(sqlite::Value::Null, |t| sqlite::Value::Integer(t as i64))).unwrap();
+							insert_song.bind(5, &tags.album_artist.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+							insert_song.bind(6, &tags.artist.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+							insert_song.bind(7, &tags.album.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+							insert_song.bind(8, &artwork).unwrap();
+							insert_song.next().ok();
+						}
+					}
+				}
+			}
+		}
+
+		// Insert directory
+		if inconsistent_directory_year {
+			directory_year = None; 
+		}
+		if inconsistent_directory_album {
+			directory_album = None; 
+		}
+		if inconsistent_directory_artist {
+			directory_artist = None; 
+		}
+		if let Some(path_string) = path.to_str() {
+			insert_directory.reset().ok();
+			insert_directory.bind(1, &sqlite::Value::String(path_string.to_owned())).unwrap();
+			insert_directory.bind(2, &artwork).unwrap();
+			insert_directory.bind(3, &directory_year.map_or(sqlite::Value::Null, |t| sqlite::Value::Integer(t as i64))).unwrap();
+			insert_directory.bind(4, &directory_artist.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+			insert_directory.bind(5, &directory_album.map_or(sqlite::Value::Null, |t| sqlite::Value::String(t))).unwrap();
+			insert_directory.next().ok();
+		}
+	}
+
+	pub fn run(&self)
+	{
 		loop {
-		index.deref().refresh(vfs.deref());
+			let db = self.connect();
-		thread::sleep(time::Duration::from_secs(60 * 30)); // TODO expose in configuration
+			if let Err(e) = db.execute("BEGIN TRANSACTION") {
+				print!("Error while beginning transaction for index update: {}", e);
+			} else {
+				self.update_index(&db);
+				if let Err(e) = db.execute("END TRANSACTION") {
+					print!("Error while ending transaction for index update: {}", e);
+				}
+			}
+			thread::sleep(time::Duration::from_secs(60 * 20)); // TODO expose in configuration
+		}
 	}
 }

src/main.rs

@@ -44,6 +44,7 @@ mod thumbnails;
 mod vfs;
 
 const DEFAULT_CONFIG_FILE_NAME: &'static str = "polaris.toml";
+const INDEX_FILE_NAME: &'static str = "index.sqlite";
 
 fn main() {
 
@@ -67,13 +68,10 @@ fn main() {
 
     // Init index
     println!("Starting up index");
-    let index_path = path::Path::new("tmp/index.sqlite"); // TODO: base off constant and wipe on startup/exit
+    let index_path = path::Path::new(INDEX_FILE_NAME);
-    let index = Arc::new(index::Index::new(&index_path).unwrap());
+    let index = Arc::new(index::Index::new(&index_path, vfs.clone(), &config.album_art_pattern).unwrap());
     let index_ref = index.clone();
-    let vfs_ref = vfs.clone();
+    std::thread::spawn(move || index_ref.run());
-    std::thread::spawn(|| {
-        index::run(vfs_ref, index_ref);
-    });
 
     // Start server
     println!("Starting up server");