From dd2cd8551d3e39335556a9d204a1d3fb50bbcef5 Mon Sep 17 00:00:00 2001 From: Mark <> Date: Mon, 26 Aug 2024 20:16:01 +0200 Subject: [PATCH] store file last modified date for each song and add find-songs-with-changed-files to get api --- musicdb-filldb/src/main.rs | 54 ++++++-- musicdb-lib/src/data/database.rs | 7 +- musicdb-lib/src/data/song.rs | 8 +- musicdb-lib/src/server/get.rs | 217 ++++++++++++++++++++++++++++++- 4 files changed, 268 insertions(+), 18 deletions(-) diff --git a/musicdb-filldb/src/main.rs b/musicdb-filldb/src/main.rs index 48ab7f3..e9982f7 100755 --- a/musicdb-filldb/src/main.rs +++ b/musicdb-filldb/src/main.rs @@ -4,7 +4,7 @@ use std::{ fs, io::Write, path::{Path, PathBuf}, - sync::{Arc, Mutex}, + sync::{Arc, Mutex}, time::SystemTime, }; use id3::TagLike; @@ -12,7 +12,7 @@ use musicdb_lib::data::{ album::Album, artist::Artist, database::{Cover, Database}, - song::{CachedData, Song}, + song::Song, CoverId, DatabaseLocation, GeneralData, }; @@ -218,18 +218,45 @@ fn main() { .to_string_lossy() .into_owned() }); - database.add_song_new(Song { - id: 0, - title: title.clone(), - location: DatabaseLocation { + database.add_song_new(Song::new( + DatabaseLocation { rel_path: path.to_path_buf(), }, - album: album_id, - artist: artist_id, - more_artists: vec![], - cover: None, - file_size: song_file_metadata.len(), - duration_millis: if let Some(dur) = song_tags.duration() { + match path.metadata() { + Ok(v) => match v.modified() { + Ok(v) => if let Ok(time) = v.duration_since(SystemTime::UNIX_EPOCH) { + Some(time.as_secs()) + } else { + eprintln!( + "LastModified time of song {:?} is before the UNIX-EPOCH, setting `None`.", + song_path + ); + None + }, + Err(e) => { + eprintln!( + "LastModified time of song {:?} not available: {e}.", + song_path + ); + None + } + } + Err(e) => { + eprintln!( + "LastModified time of song {:?} could not be read: {e}.", + song_path + ); + None + + } + }, + title.clone(), + album_id, + artist_id, + vec![], + None, + song_file_metadata.len(), + if let Some(dur) = song_tags.duration() { dur as u64 } else { if skip_duration { @@ -249,8 +276,7 @@ fn main() { } }, general, - cached_data: CachedData(Arc::new(Mutex::new((None, None)))), - }); + )); } eprintln!("searching for covers..."); let mut multiple_cover_options = vec![]; diff --git a/musicdb-lib/src/data/database.rs b/musicdb-lib/src/data/database.rs index 0b93d73..2d2d15d 100755 --- a/musicdb-lib/src/data/database.rs +++ b/musicdb-lib/src/data/database.rs @@ -3,7 +3,7 @@ use std::{ collections::{BTreeSet, HashMap}, fs::{self, File}, io::{BufReader, Read, Write}, - path::PathBuf, + path::{Path, PathBuf}, sync::{mpsc, Arc, Mutex}, time::{Duration, Instant}, }; @@ -73,7 +73,10 @@ impl Database { self.client_is_init } pub fn get_path(&self, location: &DatabaseLocation) -> PathBuf { - self.lib_directory.join(&location.rel_path) + Self::get_path_nodb(&self.lib_directory, location) + } + pub fn get_path_nodb(lib_directory: &impl AsRef, location: &DatabaseLocation) -> PathBuf { + lib_directory.as_ref().join(&location.rel_path) } fn modified_data(&mut self) { let now = Instant::now(); diff --git a/musicdb-lib/src/data/song.rs b/musicdb-lib/src/data/song.rs index bb1468b..32c63db 100755 --- a/musicdb-lib/src/data/song.rs +++ b/musicdb-lib/src/data/song.rs @@ -21,6 +21,7 @@ use super::{ pub struct Song { pub id: SongId, pub location: DatabaseLocation, + pub file_last_modified_unix_timestamp: Option, pub title: String, pub album: Option, pub artist: ArtistId, @@ -38,6 +39,7 @@ pub struct Song { impl Song { pub fn new( location: DatabaseLocation, + file_last_modified_unix_timestamp: Option, title: String, album: Option, artist: ArtistId, @@ -45,10 +47,12 @@ impl Song { cover: Option, file_size: u64, duration_millis: u64, + general: GeneralData, ) -> Self { Self { id: 0, location, + file_last_modified_unix_timestamp, title, album, artist, @@ -56,7 +60,7 @@ impl Song { cover, file_size, duration_millis, - general: GeneralData::default(), + general, cached_data: CachedData(Arc::new(Mutex::new((Err(None), None)))), } } @@ -258,6 +262,7 @@ impl ToFromBytes for Song { { self.id.to_bytes(s)?; self.location.to_bytes(s)?; + self.file_last_modified_unix_timestamp.to_bytes(s)?; self.title.to_bytes(s)?; self.album.to_bytes(s)?; self.artist.to_bytes(s)?; @@ -275,6 +280,7 @@ impl ToFromBytes for Song { Ok(Self { id: ToFromBytes::from_bytes(s)?, location: ToFromBytes::from_bytes(s)?, + file_last_modified_unix_timestamp: ToFromBytes::from_bytes(s)?, title: ToFromBytes::from_bytes(s)?, album: ToFromBytes::from_bytes(s)?, artist: ToFromBytes::from_bytes(s)?, diff --git a/musicdb-lib/src/server/get.rs b/musicdb-lib/src/server/get.rs index 1a0041e..ac8a475 100755 --- a/musicdb-lib/src/server/get.rs +++ b/musicdb-lib/src/server/get.rs @@ -3,7 +3,7 @@ use std::{ io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, sync::{Arc, Mutex}, - time::Instant, + time::{Instant, SystemTime}, }; use crate::data::{database::Database, CoverId, SongId}; @@ -102,6 +102,134 @@ impl Client { Ok(Err(response)) } } + /// find songs which have no last-modified time, whose time has changed, whose files have been removed or whose files could not be read due to another error. + pub fn find_songs_with_changed_files( + &mut self, + ) -> Result< + Result< + ( + Vec, + Vec<(SongId, u64)>, + Vec, + Vec<(SongId, String)>, + ), + String, + >, + std::io::Error, + > { + writeln!( + self.0.get_mut(), + "{}", + con_get_encode_string("find-songs-with-changed-files") + )?; + self.0.get_mut().flush()?; + loop { + let mut response = String::new(); + self.0.read_line(&mut response)?; + let len_line = response.trim(); + if len_line.starts_with('%') { + eprintln!( + "[Find Songs With Changed Files] Status: {}", + len_line[1..].trim() + ); + } else { + let mut read_list = || -> std::io::Result, String>> { + if len_line.starts_with("len: ") { + if let Ok(len) = len_line[4..].trim().parse() { + let mut out = Vec::with_capacity(len); + for _ in 0..len { + let mut line = String::new(); + self.0.read_line(&mut line)?; + let line = line.trim_end_matches(['\n', '\r']); + out.push(line.trim().to_owned()); + } + Ok(Ok(out)) + } else { + Ok(Err(format!("bad len in len-line: {len_line}"))) + } + } else { + Ok(Err(format!("bad len-line: {len_line}"))) + } + }; + break Ok(Ok(( + match read_list()? { + Ok(v) => match v + .into_iter() + .map(|v| v.trim().parse::().map_err(|e| (v, e.to_string()))) + .collect() + { + Ok(v) => v, + Err((s, e)) => { + return Ok(Err(format!("error parsing songid(notime) '{s}': {e}"))) + } + }, + Err(e) => return Ok(Err(e)), + }, + match read_list()? { + Ok(v) => match v + .into_iter() + .map(|v| { + v.trim() + .split_once(':') + .ok_or_else(|| format!("missing colon")) + .and_then(|(i, t)| { + Ok(( + i.parse::().map_err(|e| e.to_string())?, + t.parse::().map_err(|e| e.to_string())?, + )) + }) + .map_err(|e| (v, e)) + }) + .collect() + { + Ok(v) => v, + Err((s, e)) => { + return Ok(Err(format!("error parsing songid+time '{s}': {e}"))) + } + }, + Err(e) => return Ok(Err(e)), + }, + match read_list()? { + Ok(v) => match v + .into_iter() + .map(|v| v.trim().parse::().map_err(|e| (v, e))) + .collect() + { + Ok(v) => v, + Err((s, e)) => { + return Ok(Err(format!("error parsing songid(deleted) '{s}': {e}"))) + } + }, + Err(e) => return Ok(Err(e)), + }, + match read_list()? { + Ok(v) => match v + .into_iter() + .map(|v| { + v.trim() + .split_once(':') + .ok_or_else(|| format!("missing colon")) + .and_then(|(i, t)| { + Ok(( + i.parse::().map_err(|e| e.to_string())?, + t.to_owned(), + )) + }) + .map_err(|e| (v, e)) + }) + .collect() + { + Ok(v) => v, + Err((s, e)) => { + return Ok(Err(format!("error parsing songid+error '{s}': {e}"))) + } + }, + Err(e) => return Ok(Err(e)), + }, + ))); + }; + } + } /// tell the server to search for files that are not in its song database. /// /// ## `extensions`: @@ -109,6 +237,8 @@ impl Client { /// If `Some([])`, allow all extensions, even ones like `.jpg` and files without extensions. /// If `Some(...)`, only allow the specified extensions. Note: These are actually suffixes, for example `mp3` would allow a file named `test_mp3`, while `.mp3` would only allow `test.mp3`. /// Because of this, you usually want to include the `.` before the extension, and double extensions like `.tar.gz` are also supported. + /// + /// For each file, returns a boolean error flag indicating, if `true`, that the path was invalid (not UTF-8 or contained a newline). pub fn find_unused_song_files( &mut self, extensions: Option<&[&str]>, @@ -317,6 +447,91 @@ pub fn handle_one_connection_as_get( writeln!(connection.get_mut(), "no data")?; } } + "find-songs-with-changed-files" => { + let db_lock = db.lock().unwrap(); + let lib_directory = db_lock.lib_directory.clone(); + let all_songs = db_lock + .songs() + .iter() + .map(|(id, song)| { + ( + *id, + song.location.clone(), + song.file_last_modified_unix_timestamp.clone(), + ) + }) + .collect::>(); + drop(db_lock); + let ( + mut songs_no_time, + mut songs_new_time, + mut songs_removed, + mut songs_err, + ) = (vec![], vec![], vec![], vec![]); + for (id, location, last_modified) in all_songs { + let path = Database::get_path_nodb(&lib_directory, &location); + match path.try_exists() { + Ok(true) => match path.metadata() { + Ok(metadata) => { + let time = metadata.modified().ok().and_then(|time| { + time.duration_since(SystemTime::UNIX_EPOCH) + .ok() + .map(|v| v.as_secs()) + }); + if last_modified.is_none() || time != last_modified { + if let Some(time) = time { + songs_new_time.push((id, time)); + } else { + songs_no_time.push(id); + } + } + } + Err(e) => songs_err.push((id, e.to_string())), + }, + Ok(false) => songs_removed.push(id), + Err(e) => songs_err.push((id, e.to_string())), + } + } + write_list( + connection.get_mut(), + songs_no_time.len(), + songs_no_time.into_iter().map(|id| (id, None)), + )?; + write_list( + connection.get_mut(), + songs_new_time.len(), + songs_new_time + .into_iter() + .map(|(id, t)| (id, Some(format!("{t}")))), + )?; + write_list( + connection.get_mut(), + songs_removed.len(), + songs_removed.into_iter().map(|id| (id, None)), + )?; + write_list( + connection.get_mut(), + songs_err.len(), + songs_err + .into_iter() + .map(|(id, e)| (id, Some(format!("{e}")))), + )?; + fn write_list( + connection: &mut impl Write, + len: usize, + list: impl IntoIterator)>, + ) -> std::io::Result<()> { + writeln!(connection, "len: {}", len)?; + for (song, data) in list { + if let Some(data) = data { + writeln!(connection, "{song}:{data}")?; + } else { + writeln!(connection, "{song}")?; + } + } + Ok(()) + } + } "find-unused-song-files" => { // configure search let mut extensions = None;