mark/musicdb
1
0
Fork 0
mirror of https://github.com/Dummi26/musicdb.git synced 2026-04-28 09:39:58 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: more warnings and statistics for filldb

Browse Source
This commit is contained in:
Mark
2026-03-29 21:46:57 +02:00
parent 1138365180
commit 3b544d834a
2 changed files with 211 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
musicdb-filldb/Cargo.toml
View File

@@ -1,7 +1,7 @@
[package] [package]
name = "musicdb-filldb" name = "musicdb-filldb"
version = "0.1.0" version = "0.1.0"
edition = "2021" edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

251
musicdb-filldb/src/main.rs
View File

@@ -1,6 +1,6 @@
use std::{ use std::{
cmp::Ordering, cmp::Ordering,
collections::HashMap, collections::{BTreeMap, HashMap},
fs, fs,
io::Write, io::Write,
path::{Path, PathBuf}, path::{Path, PathBuf},
@@ -10,11 +10,11 @@ use std::{
use id3::TagLike; use id3::TagLike;
use musicdb_lib::data::{ use musicdb_lib::data::{
CoverId, DatabaseLocation, GeneralData,
album::Album, album::Album,
artist::Artist, artist::Artist,
database::{Cover, Database}, database::{Cover, Database},
song::Song, song::Song,
CoverId, DatabaseLocation, GeneralData,
}; };
fn main() { fn main() {
@@ -23,27 +23,50 @@ fn main() {
let lib_dir = if let Some(arg) = args.next() { let lib_dir = if let Some(arg) = args.next() {
arg arg
} else { } else {
eprintln!("usage: musicdb-filldb <library root> [--help] [--skip-duration] [--custom-files <path>] [... (see --help)]"); eprintln!(
"usage: musicdb-filldb <library root> [--help] [--skip-duration] [--custom-files <path>] [... (see --help)]"
);
std::process::exit(1); std::process::exit(1);
}; };
let mut bad_arg = false; let mut bad_arg = false;
let mut dbdir = ".".to_owned(); let mut dbdir = ".".to_owned();
let mut skip_duration = false; let mut skip_duration = false;
let mut verbosity = 0;
let mut custom_files = None; let mut custom_files = None;
let mut artist_img = false; let mut artist_img = false;
let mut export_custom_files = None; let mut export_custom_files = None;
let mut stats = false;
let mut year_counts = BTreeMap::new();
let mut genre_counts = BTreeMap::new();
loop { loop {
match args.next() { match args.next() {
None => break, None => break,
Some(arg) => match arg.as_str() { Some(arg) => match arg.as_str() {
"-v" | "--verbose" => verbosity += 1,
"--help" => { "--help" => {
eprintln!(
"-v, --verbose: Generate more warnings (can be specified multiple times)"
);
eprintln!("--dbdir <path>: Save dbfile in the <path> directory (default: `.`)"); eprintln!("--dbdir <path>: Save dbfile in the <path> directory (default: `.`)");
eprintln!("--skip-duration: Don't try to figure out the songs duration from file contents. This means mp3 files with the Duration field unset will have a duration of 0."); eprintln!(
eprintln!("--custom-files <path>: server will use <path> as its custom-files directory. Additional data is loaded from here."); "--skip-duration: Don't try to figure out the songs duration from file contents. This means mp3 files with the Duration field unset will have a duration of 0."
eprintln!("--cf-artist-img: For each artist, check for an <artist>.{{jpg,png,...}} file. If it exists, add ImageExt=<extension> tag to the artist, so the image can be loaded by clients later."); );
eprintln!("--export-custom-files <path>: Create <path> as a directory containing metadata from the *existing* dbfile, so that it can be loaded again using --custom-files <same-path>."); eprintln!(
"--custom-files <path>: Server will use <path> as its custom-files directory. Additional data is loaded from here."
);
eprintln!(
"--cf-artist-img: For each artist, check for an <artist>.{{jpg,png,...}} file. If it exists, add ImageExt=<extension> tag to the artist, so the image can be loaded by clients later."
);
eprintln!(
"--export-custom-files <path>: Create <path> as a directory containing metadata from the *existing* dbfile, so that it can be loaded again using --custom-files <same-path>."
);
eprintln!("--stats, --statistics: Output statistics before exiting.");
return; return;
} }
"--stats" | "--statistics" => stats = true,
"--dbdir" => { "--dbdir" => {
if let Some(dir) = args.next() { if let Some(dir) = args.next() {
dbdir = dir; dbdir = dir;
@@ -77,12 +100,10 @@ fn main() {
}, },
} }
} }
if export_custom_files.is_some() { if export_custom_files.is_some() && (skip_duration || custom_files.is_some() || artist_img) {
if skip_duration || custom_files.is_some() || artist_img {
bad_arg = true; bad_arg = true;
eprintln!("--export-custom-files :: incompatible with other arguments except --dbdir!"); eprintln!("--export-custom-files :: incompatible with other arguments except --dbdir!");
} }
}
if bad_arg { if bad_arg {
return; return;
} }
@@ -121,7 +142,7 @@ fn main() {
let mut database = Database::new_empty_in_dir(PathBuf::from(dbdir), PathBuf::from(&lib_dir)); let mut database = Database::new_empty_in_dir(PathBuf::from(dbdir), PathBuf::from(&lib_dir));
let unknown_artist = database.add_artist_new(Artist { let unknown_artist = database.add_artist_new(Artist {
id: 0, id: 0,
name: format!("<unknown>"), name: "<unknown>".to_owned(),
cover: None, cover: None,
albums: vec![], albums: vec![],
singles: vec![], singles: vec![],
@@ -148,7 +169,7 @@ fn main() {
(None, Some(_)) => Ordering::Less, (None, Some(_)) => Ordering::Less,
(None, None) => Ordering::Equal, (None, None) => Ordering::Equal,
} }
.then_with(|| path1.cmp(&path2)) .then_with(|| path1.cmp(path2))
}) })
}); });
for (i, (song_path, song_file_metadata, song_tags)) in songs.into_iter().enumerate() { for (i, (song_path, song_file_metadata, song_tags)) in songs.into_iter().enumerate() {
@@ -173,9 +194,19 @@ fn main() {
} }
if let Some(year) = song_tags.year() { if let Some(year) = song_tags.year() {
general.tags.push(format!("SRCFILE:Year={year}")); general.tags.push(format!("SRCFILE:Year={year}"));
if stats {
*year_counts.entry(year).or_insert(0) += 1;
}
} else if verbosity > 0 {
eprintln!("Missing year tag for file {}.", song_path.display());
} }
if let Some(genre) = song_tags.genre_parsed() { if let Some(genre) = song_tags.genre_parsed() {
general.tags.push(format!("SRCFILE:Genre={genre}")); general.tags.push(format!("SRCFILE:Genre={genre}"));
if stats {
*genre_counts.entry(genre.into_owned()).or_insert(0) += 1;
}
} else if verbosity > 0 {
eprintln!("Missing genre tag for file {}.", song_path.display());
} }
let (artist_id, album_id) = if let Some(artist) = song_tags let (artist_id, album_id) = if let Some(artist) = song_tags
.album_artist() .album_artist()
@@ -234,7 +265,7 @@ fn main() {
let path = song_path.strip_prefix(&lib_dir).unwrap(); let path = song_path.strip_prefix(&lib_dir).unwrap();
let title = song_tags let title = song_tags
.title() .title()
.map_or(None, |title| { .and_then(|title| {
if title.trim().is_empty() { if title.trim().is_empty() {
None None
} else { } else {
@@ -289,10 +320,12 @@ fn main() {
dur as u64 dur as u64
} else { } else {
if skip_duration { if skip_duration {
if verbosity > 0 {
eprintln!( eprintln!(
"Duration of song {:?} not found in tags, using 0 instead!", "Duration of song {:?} not found in tags, using 0 instead!",
song_path song_path
); );
}
0 0
} else { } else {
match mp3_duration::from_path(&song_path) { match mp3_duration::from_path(&song_path) {
@@ -336,18 +369,18 @@ fn main() {
let mut single_images = HashMap::new(); let mut single_images = HashMap::new();
for (i1, (_artist, (artist_id, albums))) in artists.iter().enumerate() { for (i1, (_artist, (artist_id, albums))) in artists.iter().enumerate() {
eprint!("\rartist {}/{}", i1 + 1, artists.len()); eprint!("\rartist {}/{}", i1 + 1, artists.len());
for (_album, (album_id, album_dir)) in albums { for (album_id, album_dir) in albums.values() {
if let Some(album_dir) = album_dir { if let Some(album_dir) = album_dir
if let Some(cover_id) = get_cover( && let Some(cover_id) = get_cover(
&mut database, &mut database,
&lib_dir, &lib_dir,
album_dir, album_dir,
&mut multiple_cover_options, &mut multiple_cover_options,
) { )
{
database.albums_mut().get_mut(album_id).unwrap().cover = Some(cover_id); database.albums_mut().get_mut(album_id).unwrap().cover = Some(cover_id);
} }
} }
}
if let Some(artist) = database.artists().get(artist_id) { if let Some(artist) = database.artists().get(artist_id) {
for song in artist.singles.clone() { for song in artist.singles.clone() {
if let Some(dir) = AsRef::<Path>::as_ref(&lib_dir) if let Some(dir) = AsRef::<Path>::as_ref(&lib_dir)
@@ -394,24 +427,21 @@ fn main() {
} }
Ok(ls) => { Ok(ls) => {
let mut files = HashMap::new(); let mut files = HashMap::new();
for entry in ls { for entry in ls.flatten() {
if let Ok(entry) = entry {
let p = entry.path(); let p = entry.path();
if let Some(base) = p.file_stem().and_then(|v| v.to_str()) { if let Some(base) = p.file_stem().and_then(|v| v.to_str())
if let Some(ext) = entry && let Some(ext) = entry
.path() .path()
.extension() .extension()
.and_then(|v| v.to_str()) .and_then(|v| v.to_str())
.filter(|v| { .filter(|v| {
matches!(v.to_lowercase().as_str(), "png" | "jpg" | "jpeg") matches!(v.to_lowercase().as_str(), "png" | "jpg" | "jpeg")
}) })
&& let Some(old) = files.insert(base.to_owned(), ext.to_owned())
{ {
if let Some(old) = files.insert(base.to_owned(), ext.to_owned()) eprintln!(
{ "[warn] Not using file {base}.{old}, because {base}.{ext} was found."
eprintln!("[warn] Not using file {base}.{old}, because {base}.{ext} was found."); );
}
}
}
} }
} }
for artist in database.artists_mut().values_mut() { for artist in database.artists_mut().values_mut() {
@@ -423,7 +453,9 @@ fn main() {
} }
} }
} }
eprintln!("[info] Searching for <artist>.tags, <artist>.d/<album>.tags, <artist>.d/singles.d/<song>.tags, <artist>.d/<album>.d/<song>.tags in custom-files dir..."); eprintln!(
"[info] Searching for <artist>.tags, <artist>.d/<album>.tags, <artist>.d/singles.d/<song>.tags, <artist>.d/<album>.d/<song>.tags in custom-files dir..."
);
let l = database.artists().len() + database.albums().len() + database.songs().len(); let l = database.artists().len() + database.albums().len() + database.songs().len();
let mut cc = 0; let mut cc = 0;
let mut c = 0; let mut c = 0;
@@ -458,17 +490,17 @@ fn main() {
for song in artist.singles.iter() { for song in artist.singles.iter() {
// <artist>.d/singles/<song>.tags // <artist>.d/singles/<song>.tags
cc += 1; cc += 1;
if let Some(song) = songs.get_mut(song) { if let Some(song) = songs.get_mut(song)
if let Ok(info) = fs::read_to_string(dir.join(format!( && let Ok(info) = fs::read_to_string(dir.join(format!(
"{}.tags", "{}.tags",
normalize_to_file_path_component_for_custom_files(&song.title) normalize_to_file_path_component_for_custom_files(&song.title)
))) { )))
{
c += 1; c += 1;
push_tags(&info, &mut song.general.tags); push_tags(&info, &mut song.general.tags);
} }
} }
} }
}
for album in artist.albums.iter() { for album in artist.albums.iter() {
eprint!(" {cc}/{l} ({c})\r"); eprint!(" {cc}/{l} ({c})\r");
cc += 1; cc += 1;
@@ -516,6 +548,64 @@ fn main() {
eprintln!("saving dbfile..."); eprintln!("saving dbfile...");
database.save_database(None).unwrap(); database.save_database(None).unwrap();
eprintln!("done!"); eprintln!("done!");
if stats {
eprintln!();
eprintln!("=== Genre Statistics ===");
let mut genre_counts = genre_counts
.iter()
.map(|(genre, count)| (genre, *count))
.collect::<Vec<_>>();
genre_counts.sort_by(|(_, count_1), (_, count_2)| count_2.cmp(count_1));
for (genre, count) in genre_counts {
eprintln!("{genre}: {count}");
}
eprintln!();
eprintln!("=== Year Statistics ===");
if let Some((&min_year, _)) = year_counts.first_key_value()
&& let Some((&max_year, _)) = year_counts.last_key_value()
{
let width = year_counts
.values()
.copied()
.max()
.unwrap_or(0)
.to_string()
.len();
for i in min_year / 10..=max_year / 10 {
let start = 10 * i;
let end = 10 * (1 + i);
let total = year_counts
.range(start..end)
.map(|(_, c)| *c)
.sum::<usize>()
.to_string();
eprint!(
"{}-{} | ∑: {}{}",
start,
end - 1,
" ".repeat(width + 1 - total.len()),
total,
);
for y in start..end {
let count = year_counts.get(&y).copied().unwrap_or(0);
let (pre, post) = if count == 0 {
("\x1b[90m", "\x1b[0m")
} else {
("", "")
};
let count = count.to_string();
eprint!(
", {}{y}: {}{}{}",
pre,
" ".repeat(width - count.len()),
count,
post
);
}
eprintln!();
}
}
}
} }
fn get_all_files_in_dir(dir: impl AsRef<Path>) -> Vec<PathBuf> { fn get_all_files_in_dir(dir: impl AsRef<Path>) -> Vec<PathBuf> {
@@ -556,18 +646,18 @@ fn get_cover(
let mut multiple = false; let mut multiple = false;
let mut cover = None; let mut cover = None;
if let Ok(files) = fs::read_dir(&abs_dir) { if let Ok(files) = fs::read_dir(&abs_dir) {
for file in files { for file in files.flatten() {
if let Ok(file) = file {
let path = file.path(); let path = file.path();
if let Ok(metadata) = path.metadata() { if let Ok(metadata) = path.metadata()
if metadata.is_file() { && metadata.is_file()
if path.extension().and_then(|v| v.to_str()).is_some_and(|v| { && path
matches!(v.to_lowercase().as_str(), "png" | "jpg" | "jpeg") .extension()
}) { .and_then(|v| v.to_str())
if cover.is_none() .is_some_and(|v| matches!(v.to_lowercase().as_str(), "png" | "jpg" | "jpeg"))
&& (cover.is_none()
|| cover || cover
.as_ref() .as_ref()
.is_some_and(|(_, size)| *size < metadata.len()) .is_some_and(|(_, size)| *size < metadata.len()))
{ {
if cover.is_some() { if cover.is_some() {
multiple = true; multiple = true;
@@ -576,15 +666,11 @@ fn get_cover(
} }
} }
} }
}
}
}
}
if multiple { if multiple {
multiple_options_list.push(abs_dir.as_ref().to_path_buf()); multiple_options_list.push(abs_dir.as_ref().to_path_buf());
} }
if let Some((path, _)) = cover { if let Some((path, _)) = cover {
let rel_path = path.strip_prefix(&lib_dir).unwrap().to_path_buf(); let rel_path = path.strip_prefix(lib_dir).unwrap().to_path_buf();
Some(database.add_cover_new(Cover { Some(database.add_cover_new(Cover {
location: DatabaseLocation { location: DatabaseLocation {
rel_path: rel_path.clone(), rel_path: rel_path.clone(),
@@ -606,6 +692,8 @@ fn normalize_to_file_path_component_for_custom_files(str: &str) -> String {
.replace('\n', "%n") .replace('\n', "%n")
} }
// may NOT set \! to a valid escape sequence, as this is used to
// identify db-internal "tags" such as the Song-/Album-/Artist-ID
fn normalize_tag_to_str(str: &str) -> String { fn normalize_tag_to_str(str: &str) -> String {
str.replace('\\', "\\S") str.replace('\\', "\\S")
.replace('\n', "\\n") .replace('\n', "\\n")
@@ -620,9 +708,10 @@ fn normalized_str_to_tag(str: &str) -> String {
fn export_to_custom_files_dir(dbdir: String, path: PathBuf) { fn export_to_custom_files_dir(dbdir: String, path: PathBuf) {
let database = Database::load_database_from_dir(dbdir.into(), PathBuf::new()).unwrap(); let database = Database::load_database_from_dir(dbdir.into(), PathBuf::new()).unwrap();
for artist in database.artists().values() { for (artist_id, artist) in database.artists().iter() {
export_custom_files_tags( export_custom_files_tags(
&artist.general.tags, &artist.general.tags,
&gen_internals(Some(*artist_id), artist.cover),
&path.join(format!( &path.join(format!(
"{}.tags", "{}.tags",
normalize_to_file_path_component_for_custom_files(&artist.name) normalize_to_file_path_component_for_custom_files(&artist.name)
@@ -634,10 +723,11 @@ fn export_to_custom_files_dir(dbdir: String, path: PathBuf) {
)); ));
{ {
let dir = dir.join("singles"); let dir = dir.join("singles");
for song in artist.singles.iter() { for song_id in artist.singles.iter() {
if let Some(song) = database.songs().get(song) { if let Some(song) = database.songs().get(song_id) {
export_custom_files_tags( export_custom_files_tags(
&song.general.tags, &song.general.tags,
&gen_internals(Some(*song_id), song.cover),
&dir.join(format!( &dir.join(format!(
"{}.tags", "{}.tags",
normalize_to_file_path_component_for_custom_files(&song.title,) normalize_to_file_path_component_for_custom_files(&song.title,)
@@ -646,10 +736,11 @@ fn export_to_custom_files_dir(dbdir: String, path: PathBuf) {
} }
} }
} }
for album in artist.albums.iter() { for album_id in artist.albums.iter() {
if let Some(album) = database.albums().get(album) { if let Some(album) = database.albums().get(album_id) {
export_custom_files_tags( export_custom_files_tags(
&album.general.tags, &album.general.tags,
&gen_internals(Some(*album_id), album.cover),
&dir.join(format!( &dir.join(format!(
"{}.tags", "{}.tags",
normalize_to_file_path_component_for_custom_files(&album.name,) normalize_to_file_path_component_for_custom_files(&album.name,)
@@ -659,10 +750,11 @@ fn export_to_custom_files_dir(dbdir: String, path: PathBuf) {
"{}.d", "{}.d",
normalize_to_file_path_component_for_custom_files(&album.name,) normalize_to_file_path_component_for_custom_files(&album.name,)
)); ));
for song in album.songs.iter() { for song_id in album.songs.iter() {
if let Some(song) = database.songs().get(song) { if let Some(song) = database.songs().get(song_id) {
export_custom_files_tags( export_custom_files_tags(
&song.general.tags, &song.general.tags,
&gen_internals(Some(*song_id), song.cover),
&dir.join(format!( &dir.join(format!(
"{}.tags", "{}.tags",
normalize_to_file_path_component_for_custom_files(&song.title,) normalize_to_file_path_component_for_custom_files(&song.title,)
@@ -674,15 +766,16 @@ fn export_to_custom_files_dir(dbdir: String, path: PathBuf) {
} }
} }
} }
fn export_custom_files_tags(tags: &Vec<String>, path: &Path) { fn export_custom_files_tags(tags: &[String], internals: &[String], path: &Path) {
if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) {
let mut normalized_tags = None; let mut normalized_tags = None;
fn mk_normalized_tags<'a>( fn mk_normalized_tags<'a>(
normalized_tags: &'a mut Option<Vec<String>>, normalized_tags: &'a mut Option<String>,
tags: &'_ Vec<String>, tags: &[String],
) -> &'a Vec<String> { internals: &[String],
) -> &'a String {
&*normalized_tags.get_or_insert_with(|| { &*normalized_tags.get_or_insert_with(|| {
let mut tags = tags.clone(); let mut tags = Vec::from(tags);
let mut rm = Vec::new(); let mut rm = Vec::new();
for (i, srcfile_tag_stripped) in tags for (i, srcfile_tag_stripped) in tags
.iter() .iter()
@@ -710,7 +803,14 @@ fn export_custom_files_tags(tags: &Vec<String>, path: &Path) {
for i in rm.into_iter().rev() { for i in rm.into_iter().rev() {
tags.remove(i); tags.remove(i);
} }
tags tags.iter()
.map(|tag| normalize_tag_to_str(tag) + "\n")
.chain(
internals
.iter()
.map(|tag| format!("\\!{}\n", normalize_tag_to_str(tag))),
)
.collect::<String>()
}) })
} }
let allow_write = match fs::exists(path) { let allow_write = match fs::exists(path) {
@@ -721,10 +821,7 @@ fn export_custom_files_tags(tags: &Vec<String>, path: &Path) {
Ok(false) => true, Ok(false) => true,
Ok(true) => { Ok(true) => {
if fs::read_to_string(path).is_ok_and(|file| { if fs::read_to_string(path).is_ok_and(|file| {
file.lines() file == *mk_normalized_tags(&mut normalized_tags, tags, internals)
.map(|str| normalized_str_to_tag(str))
.collect::<Vec<String>>()
== *mk_normalized_tags(&mut normalized_tags, tags)
}) { }) {
// file contains the same tags as database, don't write, // file contains the same tags as database, don't write,
// but don't create backup either // but don't create backup either
@@ -764,30 +861,36 @@ fn export_custom_files_tags(tags: &Vec<String>, path: &Path) {
} }
} }
}; };
if allow_write { if allow_write && !mk_normalized_tags(&mut normalized_tags, tags, internals).is_empty() {
if !mk_normalized_tags(&mut normalized_tags, tags).is_empty() { if let Some(p) = path.parent()
if let Some(p) = path.parent() { && let Err(e) = fs::create_dir_all(p)
if let Err(e) = fs::create_dir_all(p) { {
eprintln!( eprintln!(
"Could not create directory to contain {}: {e}", "Could not create directory to contain {}: {e}",
path.display() path.display()
); );
} }
}
if let Err(e) = fs::write( if let Err(e) = fs::write(
path, path,
mk_normalized_tags(&mut normalized_tags, tags) mk_normalized_tags(&mut normalized_tags, tags, internals),
.iter()
.map(|tag| normalize_tag_to_str(tag) + "\n")
.collect::<String>(),
) { ) {
eprintln!("Could not save {}: {e}", path.display()); eprintln!("Could not save {}: {e}", path.display());
} }
} }
}
} else { } else {
eprintln!( eprintln!(
"[ERR] Somehow created a non-unicode path {path:?}! This should not have happened!" "[ERR] Somehow created a non-unicode path {path:?}! This should not have happened!"
); );
} }
} }
// TODO: load these tags, infer album and artist id from parent directories in the structure (will probably happen with no further changes required)
fn gen_internals(id: Option<u64>, cover: Option<CoverId>) -> Vec<String> {
[
id.map(|id| format!("id={id}")),
cover.map(|id| format!("cover={id}")),
]
.into_iter()
.flatten()
.collect()
}