fix(library): multi-genre local index with track_genre and backfill (#1059)

* fix(library): multi-genre local index with track_genre and backfill

Restore atomic genre browse, filters, and counts via track_genre:
OpenSubsonic genres[] first with Navidrome-default split fallback, sync
write path, read-path query switches, blocking startup backfill with
progress, and v12 repair migration for DBs that recorded legacy 002–011.
TS fallback adds genreTagsFor and migration gate i18n across locales.

* fix(library): address multi-genre review — robust TS genres and scope join

genreTagsFor routes raw genres through parseItemGenres (single-object
Subsonic quirk and bare strings). Library-scoped genre browse/counts join
track for raw_json library_id fallback. Statistics keeps empty-genre bucket.

* docs: CHANGELOG and credits for multi-genre local index (PR #1059)

* docs(changelog): credit HiveMind on Discord for multi-genre report (PR #1059)
This commit is contained in:
cucadmuh
2026-06-11 01:02:35 +03:00
committed by GitHub
parent 8593858f3a
commit 5cd01c90ac
55 changed files with 1322 additions and 145 deletions
@@ -248,6 +248,23 @@ CREATE TABLE play_session (
CHECK (completion IN ('partial', 'full'))
);
CREATE TABLE track_genre (
server_id TEXT NOT NULL,
track_id TEXT NOT NULL,
genre TEXT NOT NULL,
album_id TEXT,
library_id TEXT,
PRIMARY KEY (server_id, track_id, genre COLLATE NOCASE),
FOREIGN KEY (server_id, track_id) REFERENCES track(server_id, id) ON DELETE CASCADE
);
CREATE TABLE library_data_migration (
id TEXT PRIMARY KEY,
cursor_rowid INTEGER NOT NULL DEFAULT 0,
completed_at INTEGER,
started_at INTEGER
);
CREATE INDEX idx_track_album ON track(server_id, album_id) WHERE deleted = 0;
CREATE INDEX idx_track_artist ON track(server_id, artist_id) WHERE deleted = 0;
CREATE INDEX idx_track_updated ON track(server_id, server_updated_at DESC) WHERE deleted = 0;
@@ -293,3 +310,7 @@ CREATE INDEX idx_play_session_started
CREATE INDEX idx_track_fact_mood_tag
ON track_fact(server_id, fact_kind, value_text, track_id)
WHERE fact_kind = 'mood_tag';
CREATE INDEX idx_track_genre_browse
ON track_genre(server_id, genre COLLATE NOCASE, album_id, track_id)
WHERE album_id IS NOT NULL AND album_id != '';
@@ -1,6 +0,0 @@
-- psysonic-library schema v2 — large-library ingest policy (R7-15).
-- Per-server learned flag: when N1 (`/api/song`) returns HTTP 500 beyond a
-- deep offset on a large catalog, the strategy selector stops choosing N1 for
-- that server on future initial syncs (spec §6.3 / R7-15 Q1/Q5). Additive
-- column, DEFAULT 0 → existing rows keep N1 eligible until they hit the wall.
ALTER TABLE sync_state ADD COLUMN n1_bulk_unreliable INTEGER NOT NULL DEFAULT 0;
@@ -1,10 +0,0 @@
-- Remap detection (§6.9) and unstable-id servers: without these indexes
-- each upsert in a 500-row batch can scan the whole track table.
CREATE INDEX IF NOT EXISTS idx_track_remap_path
ON track(server_id, server_path)
WHERE deleted = 0 AND server_path IS NOT NULL AND server_path != '';
CREATE INDEX IF NOT EXISTS idx_track_remap_hash
ON track(server_id, content_hash)
WHERE deleted = 0 AND content_hash IS NOT NULL AND content_hash != '';
@@ -1,4 +0,0 @@
-- Browse / sort-by-title without sorting the full server slice on every page.
CREATE INDEX IF NOT EXISTS idx_track_title
ON track(server_id, title COLLATE NOCASE)
WHERE deleted = 0;
@@ -1,8 +0,0 @@
-- Advanced search filters on genre and year (partial indexes — only non-null rows).
CREATE INDEX IF NOT EXISTS idx_track_genre
ON track(server_id, genre COLLATE NOCASE)
WHERE deleted = 0 AND genre IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_track_year
ON track(server_id, year)
WHERE deleted = 0 AND year IS NOT NULL;
@@ -1,22 +0,0 @@
-- Player listening history — see workdocs player-stats spec §3.1
CREATE TABLE play_session (
id INTEGER PRIMARY KEY AUTOINCREMENT,
server_id TEXT NOT NULL,
track_id TEXT NOT NULL,
started_at_ms INTEGER NOT NULL,
listened_sec REAL NOT NULL,
position_max_sec REAL NOT NULL,
completion TEXT NOT NULL,
end_reason TEXT NOT NULL,
FOREIGN KEY (server_id, track_id) REFERENCES track(server_id, id),
CHECK (completion IN ('partial', 'full'))
);
CREATE INDEX idx_play_session_server_time
ON play_session(server_id, started_at_ms DESC);
CREATE INDEX idx_play_session_track
ON play_session(server_id, track_id, started_at_ms DESC);
CREATE INDEX idx_play_session_started
ON play_session(started_at_ms DESC);
@@ -1,4 +0,0 @@
-- Full-resync orphan sweep (mark-and-sweep via generation stamp).
-- Rows ingested during a resync pass carry the active `resync_gen`; after
-- IS-6 succeeds, live rows with a stale generation are soft-deleted.
ALTER TABLE track ADD COLUMN resync_gen INTEGER NOT NULL DEFAULT 0;
@@ -1,4 +0,0 @@
-- Atomic mood tags for Advanced Search (EXISTS on track_fact).
CREATE INDEX IF NOT EXISTS idx_track_fact_mood_tag
ON track_fact(server_id, fact_kind, value_text, track_id)
WHERE fact_kind = 'mood_tag';
@@ -1,3 +0,0 @@
-- Oximedia mood heuristics were misleading; drop accumulated mood facts.
DELETE FROM track_fact
WHERE fact_kind IN ('mood_tag', 'moods', 'valence', 'arousal', 'mood_labels');
@@ -1,8 +0,0 @@
-- Genre album browse: filter by (server, genre) then group by album_id.
CREATE INDEX IF NOT EXISTS idx_track_genre_album_browse
ON track(server_id, genre COLLATE NOCASE, album_id)
WHERE deleted = 0
AND genre IS NOT NULL
AND TRIM(genre) != ''
AND album_id IS NOT NULL
AND album_id != '';
@@ -1,8 +0,0 @@
-- Genre album browse sort: (server, genre, album name, album_id) covering walk.
CREATE INDEX IF NOT EXISTS idx_track_genre_album_name_browse
ON track(server_id, genre COLLATE NOCASE, album COLLATE NOCASE, album_id)
WHERE deleted = 0
AND genre IS NOT NULL
AND TRIM(genre) != ''
AND album_id IS NOT NULL
AND album_id != '';
@@ -0,0 +1,22 @@
-- Repair for DBs that recorded legacy migrations 002011 (removed) before
-- multi-genre tables shipped. Safe on fresh installs (IF NOT EXISTS).
CREATE TABLE IF NOT EXISTS track_genre (
server_id TEXT NOT NULL,
track_id TEXT NOT NULL,
genre TEXT NOT NULL,
album_id TEXT,
library_id TEXT,
PRIMARY KEY (server_id, track_id, genre COLLATE NOCASE),
FOREIGN KEY (server_id, track_id) REFERENCES track(server_id, id) ON DELETE CASCADE
);
CREATE INDEX IF NOT EXISTS idx_track_genre_browse
ON track_genre(server_id, genre COLLATE NOCASE, album_id, track_id)
WHERE album_id IS NOT NULL AND album_id != '';
CREATE TABLE IF NOT EXISTS library_data_migration (
id TEXT PRIMARY KEY,
cursor_rowid INTEGER NOT NULL DEFAULT 0,
completed_at INTEGER,
started_at INTEGER
);
@@ -900,8 +900,25 @@ fn resolve_clause(
if c.field == "genre" {
let v = json_to_text(&c.field, c.value.as_ref())?;
let sql = match entity {
EntityKind::Track => {
"EXISTS (SELECT 1 FROM track_genre tg \
WHERE tg.server_id = t.server_id AND tg.track_id = t.id \
AND tg.genre = ? COLLATE NOCASE)"
.to_string()
}
EntityKind::Album => {
"EXISTS (SELECT 1 FROM track_genre tg \
WHERE tg.server_id = a.server_id AND tg.album_id = a.id \
AND tg.genre = ? COLLATE NOCASE)"
.to_string()
}
_ => {
return Err(filter::FilterError::NotQueryable(c.field.clone()).to_string());
}
};
return Ok(Some(SqlFragment {
sql: format!("{col} = ? COLLATE NOCASE"),
sql,
params: vec![v],
}));
}
@@ -113,13 +113,26 @@ pub(crate) fn genre_album_counts_for_server(
) -> Result<Vec<GenreAlbumCountDto>, String> {
store
.with_read_conn(|conn| {
let mut sql = String::from(
"SELECT t.genre, COUNT(DISTINCT t.album_id) AS album_count, COUNT(*) AS song_count \
FROM track t \
WHERE t.server_id = ?1 AND t.deleted = 0 \
AND t.genre IS NOT NULL AND TRIM(t.genre) != '' \
AND t.album_id IS NOT NULL AND t.album_id != ''",
);
let scoped = library_scope.is_some_and(|s| !s.trim().is_empty());
let mut sql = if scoped {
String::from(
"SELECT tg.genre, COUNT(DISTINCT tg.album_id) AS album_count, \
COUNT(DISTINCT tg.track_id) AS song_count \
FROM track_genre tg \
INNER JOIN track t \
ON t.server_id = tg.server_id AND t.id = tg.track_id AND t.deleted = 0 \
WHERE tg.server_id = ?1 \
AND tg.album_id IS NOT NULL AND tg.album_id != ''",
)
} else {
String::from(
"SELECT tg.genre, COUNT(DISTINCT tg.album_id) AS album_count, \
COUNT(DISTINCT tg.track_id) AS song_count \
FROM track_genre tg \
WHERE tg.server_id = ?1 \
AND tg.album_id IS NOT NULL AND tg.album_id != ''",
)
};
let mut params: Vec<rusqlite::types::Value> =
vec![rusqlite::types::Value::Text(server_id.to_string())];
if let Some(scope) = library_scope.filter(|s| !s.trim().is_empty()) {
@@ -127,8 +140,8 @@ pub(crate) fn genre_album_counts_for_server(
params.push(rusqlite::types::Value::Text(scope.to_string()));
}
sql.push_str(
" GROUP BY t.genre COLLATE NOCASE \
ORDER BY album_count DESC, t.genre COLLATE NOCASE ASC",
" GROUP BY tg.genre COLLATE NOCASE \
ORDER BY album_count DESC, tg.genre COLLATE NOCASE ASC",
);
let mut stmt = conn.prepare(&sql)?;
let rows = stmt
@@ -337,6 +350,26 @@ mod tests {
assert_eq!(counts[0].song_count, 1);
}
#[test]
fn genre_album_counts_scope_reads_library_id_from_track_raw_json() {
let store = Arc::new(LibraryStore::open_in_memory());
let mut scoped = make_row("s1", "r1", "al_a", 1);
scoped.genre = Some("Rock".into());
scoped.library_id = None;
scoped.raw_json = r#"{"libraryId":"lib1"}"#.into();
let mut other = make_row("s1", "r2", "al_b", 1);
other.genre = Some("Rock".into());
other.library_id = None;
other.raw_json = r#"{"libraryId":"lib2"}"#.into();
TrackRepository::new(&store)
.upsert_batch(&[scoped, other])
.unwrap();
let counts = genre_album_counts_for_server(&store, "s1", Some("lib1")).unwrap();
assert_eq!(counts.len(), 1);
assert_eq!(counts[0].album_count, 1);
}
#[test]
fn reconcile_album_stars_clears_all_when_server_list_empty() {
let store = Arc::new(LibraryStore::open_in_memory());
@@ -526,6 +526,23 @@ pub async fn library_list_albums_by_genre(
.await
}
#[tauri::command]
pub fn library_genre_tags_inspect(
runtime: State<'_, LibraryRuntime>,
) -> Result<crate::genre_tags_backfill::GenreTagsInspectDto, String> {
crate::genre_tags_backfill::inspect_genre_tags_backfill(&runtime.store)
}
#[tauri::command]
pub async fn library_genre_tags_run(
app: tauri::AppHandle,
runtime: State<'_, LibraryRuntime>,
) -> Result<(), String> {
let store = Arc::clone(&runtime.store);
library_spawn_blocking(move || crate::genre_tags_backfill::run_genre_tags_backfill(&store, &app))
.await
}
#[tauri::command]
pub async fn library_get_artist_lossless_browse(
runtime: State<'_, LibraryRuntime>,
@@ -45,10 +45,16 @@ fn count_genre_albums(
conn: &rusqlite::Connection,
where_sql: &str,
params: &[SqlValue],
library_scoped: bool,
) -> Result<u32, rusqlite::Error> {
let count_sql = format!(
"SELECT COUNT(DISTINCT t.album_id) FROM track t WHERE {where_sql}"
);
let from = if library_scoped {
"FROM track_genre tg \
INNER JOIN track t \
ON t.server_id = tg.server_id AND t.id = tg.track_id AND t.deleted = 0"
} else {
"FROM track_genre tg"
};
let count_sql = format!("SELECT COUNT(DISTINCT tg.album_id) {from} WHERE {where_sql}");
let n: i64 = conn.query_row(
&count_sql,
rusqlite::params_from_iter(params.iter()),
@@ -108,16 +114,16 @@ pub fn list_albums_by_genre(
let order_sql = genre_album_order_sql(&req.sort);
let mut where_clauses = vec![
"t.deleted = 0".to_string(),
"t.server_id = ?1".to_string(),
"t.album_id IS NOT NULL AND t.album_id != ''".to_string(),
"t.genre = ?2 COLLATE NOCASE".to_string(),
"tg.server_id = ?1".to_string(),
"tg.album_id IS NOT NULL AND tg.album_id != ''".to_string(),
"tg.genre = ?2 COLLATE NOCASE".to_string(),
];
let mut params: Vec<SqlValue> = vec![
SqlValue::Text(req.server_id.clone()),
SqlValue::Text(genre.to_string()),
];
let library_scoped = trimmed_nonempty(req.library_scope.as_deref()).is_some();
if let Some(scope) = trimmed_nonempty(req.library_scope.as_deref()) {
where_clauses.push(library_scope_equals_sql("t"));
params.push(SqlValue::Text(scope));
@@ -142,8 +148,8 @@ pub fn list_albums_by_genre(
a.raw_json \
FROM ( \
SELECT \
t.server_id, \
t.album_id, \
tg.server_id, \
tg.album_id, \
MAX(t.album) AS album_name, \
MAX(t.artist) AS artist, \
MAX(t.album_artist) AS album_artist, \
@@ -155,9 +161,11 @@ pub fn list_albums_by_genre(
MAX(t.synced_at) AS synced_at, \
COUNT(*) AS track_count, \
COALESCE(SUM(t.duration_sec), 0) AS duration_sec \
FROM track t \
FROM track_genre tg \
INNER JOIN track t \
ON t.server_id = tg.server_id AND t.id = tg.track_id AND t.deleted = 0 \
WHERE {where_sql} \
GROUP BY t.server_id, t.album_id \
GROUP BY tg.server_id, tg.album_id \
) la \
LEFT JOIN album a ON a.server_id = la.server_id AND a.id = la.album_id \
{order_sql} \
@@ -170,7 +178,7 @@ pub fn list_albums_by_genre(
store.with_read_conn(|conn| {
let total = if req.include_total {
Some(count_genre_albums(conn, &where_sql, &count_params)?)
Some(count_genre_albums(conn, &where_sql, &count_params, library_scoped)?)
} else {
None
};
@@ -285,4 +293,49 @@ mod tests {
assert_eq!(all.total, Some(3));
assert!(all.has_more);
}
#[test]
fn list_albums_by_atomic_genre_from_compound_tag() {
let store = LibraryStore::open_in_memory();
TrackRepository::new(&store)
.upsert_batch(&[track(
"s1",
"t1",
"al_a",
"Noise Metal/Dark Ambient/Experimental Black Metal",
)])
.unwrap();
let dark = list_albums_by_genre(
&store,
&LibraryGenreAlbumsRequest {
server_id: "s1".into(),
genre: "Dark Ambient".into(),
library_scope: None,
sort: vec![],
limit: 10,
offset: 0,
include_total: true,
},
)
.unwrap();
assert_eq!(dark.total, Some(1));
assert_eq!(dark.albums.len(), 1);
assert_eq!(dark.albums[0].id, "al_a");
let noise = list_albums_by_genre(
&store,
&LibraryGenreAlbumsRequest {
server_id: "s1".into(),
genre: "Noise Metal".into(),
library_scope: None,
sort: vec![],
limit: 10,
offset: 0,
include_total: true,
},
)
.unwrap();
assert_eq!(noise.total, Some(1));
}
}
@@ -0,0 +1,226 @@
//! Atomic genre resolution for multi-value tags (OpenSubsonic `genres[]` first,
//! Navidrome-default string split as fallback).
use std::collections::HashSet;
use rusqlite::{params, Transaction};
use serde_json::Value;
const GENRE_SEPARATORS: [&str; 3] = [";", "/", ","];
/// Fallback split when the server sent no `genres[]` array (legacy Subsonic).
pub fn split_genre_tags(raw: &str) -> Vec<String> {
let trimmed = raw.trim();
if trimmed.is_empty() {
return Vec::new();
}
let mut parts = vec![trimmed.to_string()];
for sep in GENRE_SEPARATORS {
let mut next = Vec::new();
for part in parts {
for sub in part.split(sep) {
next.push(sub.to_string());
}
}
parts = next;
}
dedupe_genres(parts)
}
fn dedupe_genres(genres: Vec<String>) -> Vec<String> {
let mut seen = HashSet::new();
let mut out = Vec::new();
for g in genres {
let t = g.trim();
if t.is_empty() {
continue;
}
let key = t.to_ascii_lowercase();
if seen.insert(key) {
out.push(t.to_string());
}
}
out
}
fn parse_genres_array_value(value: &Value) -> Option<Vec<String>> {
let arr = value.as_array()?;
if arr.is_empty() {
return None;
}
let mut out = Vec::new();
for item in arr {
if let Some(name) = item.get("name").and_then(|v| v.as_str()) {
let t = name.trim();
if !t.is_empty() {
out.push(t.to_string());
}
} else if let Some(s) = item.as_str() {
let t = s.trim();
if !t.is_empty() {
out.push(t.to_string());
}
}
}
if out.is_empty() {
None
} else {
Some(dedupe_genres(out))
}
}
fn parse_genres_json_str(genres_json: &str) -> Option<Vec<String>> {
let trimmed = genres_json.trim();
if trimmed.is_empty() {
return None;
}
let value: Value = serde_json::from_str(trimmed).ok()?;
parse_genres_array_value(&value)
}
/// Source-priority resolver (§2.0): `genres[]` from parsed payload, else split `genre`.
pub fn genres_for_track_value(raw_json: &Value, genre: Option<&str>) -> Vec<String> {
if let Some(genres) = raw_json.get("genres").and_then(parse_genres_array_value) {
return genres;
}
genre
.map(str::trim)
.filter(|s| !s.is_empty())
.map(split_genre_tags)
.unwrap_or_default()
}
/// Backfill path: `genres_json` from `json_extract(raw_json, '$.genres')`.
pub fn genres_for_track_extracted(genres_json: Option<&str>, genre: Option<&str>) -> Vec<String> {
if let Some(json) = genres_json {
if let Some(genres) = parse_genres_json_str(json) {
return genres;
}
}
genre
.map(str::trim)
.filter(|s| !s.is_empty())
.map(split_genre_tags)
.unwrap_or_default()
}
pub fn genres_for_track_raw_json(raw_json: &str, genre: Option<&str>) -> Vec<String> {
if let Ok(value) = serde_json::from_str::<Value>(raw_json) {
return genres_for_track_value(&value, genre);
}
genre
.map(str::trim)
.filter(|s| !s.is_empty())
.map(split_genre_tags)
.unwrap_or_default()
}
pub fn replace_track_genre_rows(
tx: &Transaction<'_>,
server_id: &str,
track_id: &str,
album_id: Option<&str>,
library_id: Option<&str>,
genres: &[String],
) -> rusqlite::Result<()> {
tx.execute(
"DELETE FROM track_genre WHERE server_id = ?1 AND track_id = ?2",
params![server_id, track_id],
)?;
if genres.is_empty() {
return Ok(());
}
let mut insert = tx.prepare_cached(
"INSERT OR IGNORE INTO track_genre (server_id, track_id, genre, album_id, library_id) \
VALUES (?1, ?2, ?3, ?4, ?5)",
)?;
for genre in genres {
insert.execute(params![server_id, track_id, genre, album_id, library_id])?;
}
Ok(())
}
pub fn delete_track_genre_for_track(
conn: &rusqlite::Connection,
server_id: &str,
track_id: &str,
) -> rusqlite::Result<()> {
conn.execute(
"DELETE FROM track_genre WHERE server_id = ?1 AND track_id = ?2",
params![server_id, track_id],
)?;
Ok(())
}
pub fn delete_track_genre_for_server_tracks(
conn: &rusqlite::Connection,
server_id: &str,
track_ids: &[String],
) -> rusqlite::Result<()> {
if track_ids.is_empty() {
return Ok(());
}
for id in track_ids {
delete_track_genre_for_track(conn, server_id, id)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn split_separators_and_dedupe() {
assert_eq!(
split_genre_tags("Rock/Jazz"),
vec!["Rock".to_string(), "Jazz".to_string()]
);
assert_eq!(
split_genre_tags("Rock; Jazz, Electronic"),
vec![
"Rock".to_string(),
"Jazz".to_string(),
"Electronic".to_string()
]
);
assert_eq!(split_genre_tags("Rock/rock/ROCK"), vec!["Rock".to_string()]);
assert!(split_genre_tags("").is_empty());
}
#[test]
fn array_wins_over_genre_string() {
let raw = json!({
"genres": [{"name": "A"}, {"name": "B"}],
"genre": "A/B/C"
});
assert_eq!(
genres_for_track_value(&raw, Some("A/B/C")),
vec!["A".to_string(), "B".to_string()]
);
}
#[test]
fn bare_string_array_and_empty_array_fallback() {
let bare = json!({ "genres": ["A", "B"] });
assert_eq!(
genres_for_track_value(&bare, None),
vec!["A".to_string(), "B".to_string()]
);
let empty = json!({ "genres": [], "genre": "A/B" });
assert_eq!(
genres_for_track_value(&empty, Some("A/B")),
vec!["A".to_string(), "B".to_string()]
);
}
#[test]
fn extracted_json_matches_value_path() {
let genres_json = r#"[{"name":"Jazz"},{"name":"Rock"}]"#;
assert_eq!(
genres_for_track_extracted(Some(genres_json), Some("Noise/Metal")),
vec!["Jazz".to_string(), "Rock".to_string()]
);
}
}
@@ -0,0 +1,318 @@
//! One-time blocking backfill: populate `track_genre` from existing `track` rows.
use std::time::{SystemTime, UNIX_EPOCH};
use rusqlite::{params, Connection, OptionalExtension};
use tauri::{AppHandle, Emitter};
use crate::genre_tags::{genres_for_track_extracted, replace_track_genre_rows};
use crate::store::LibraryStore;
pub const GENRE_TAGS_MIGRATION_ID: &str = "genre_tags_v1";
const BATCH_SIZE: i64 = 10_000;
type BackfillTrackRow = (
i64,
String,
String,
Option<String>,
Option<String>,
Option<String>,
Option<String>,
);
fn ensure_genre_tags_tables(conn: &mut Connection) -> rusqlite::Result<()> {
crate::store::ensure_genre_tags_schema(conn)
}
#[derive(Debug, Clone, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GenreTagsInspectDto {
pub needed: bool,
pub total_tracks: u64,
pub done_tracks: u64,
}
#[derive(Debug, Clone, serde::Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GenreTagsProgressEvent {
pub done: u64,
pub total: u64,
}
fn now_unix() -> i64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs() as i64)
.unwrap_or(0)
}
fn migration_completed(conn: &Connection) -> Result<bool, rusqlite::Error> {
let completed: Option<Option<i64>> = conn
.query_row(
"SELECT completed_at FROM library_data_migration WHERE id = ?1",
params![GENRE_TAGS_MIGRATION_ID],
|r| r.get(0),
)
.optional()?;
Ok(completed.flatten().is_some())
}
fn count_live_tracks(conn: &Connection) -> Result<u64, rusqlite::Error> {
let n: i64 = conn.query_row(
"SELECT COUNT(*) FROM track WHERE deleted = 0",
[],
|r| r.get(0),
)?;
Ok(n.max(0) as u64)
}
fn cursor_rowid(conn: &Connection) -> Result<i64, rusqlite::Error> {
let rowid: Option<i64> = conn
.query_row(
"SELECT cursor_rowid FROM library_data_migration WHERE id = ?1",
params![GENRE_TAGS_MIGRATION_ID],
|r| r.get(0),
)
.optional()?;
Ok(rowid.unwrap_or(0))
}
pub fn inspect_genre_tags_backfill(store: &LibraryStore) -> Result<GenreTagsInspectDto, String> {
store.with_conn_mut("genre_tags.ensure_schema", ensure_genre_tags_tables)?;
store.with_read_conn(|conn| {
let total_tracks = count_live_tracks(conn)?;
if total_tracks == 0 {
return Ok(GenreTagsInspectDto {
needed: false,
total_tracks: 0,
done_tracks: 0,
});
}
if migration_completed(conn)? {
return Ok(GenreTagsInspectDto {
needed: false,
total_tracks,
done_tracks: total_tracks,
});
}
let cursor = cursor_rowid(conn)?;
let done: i64 = conn.query_row(
"SELECT COUNT(*) FROM track WHERE deleted = 0 AND rowid <= ?1",
params![cursor],
|r| r.get(0),
)?;
Ok(GenreTagsInspectDto {
needed: true,
total_tracks,
done_tracks: done.max(0) as u64,
})
})
}
fn emit_progress(app: &AppHandle, done: u64, total: u64) -> Result<(), String> {
app.emit(
"genre_tags:progress",
GenreTagsProgressEvent { done, total },
)
.map_err(|e| e.to_string())
}
pub fn run_genre_tags_backfill(store: &LibraryStore, app: &AppHandle) -> Result<(), String> {
run_genre_tags_backfill_impl(store, Some(app))
}
fn run_genre_tags_backfill_impl(
store: &LibraryStore,
app: Option<&AppHandle>,
) -> Result<(), String> {
let inspect = inspect_genre_tags_backfill(store)?;
if !inspect.needed {
return Ok(());
}
let total = inspect.total_tracks;
loop {
let (batch_done, finished) = store.with_conn_mut("genre_tags.backfill", |conn| {
if migration_completed(conn)? {
return Ok::<(i64, bool), rusqlite::Error>((total as i64, true));
}
conn.execute(
"INSERT INTO library_data_migration (id, cursor_rowid, started_at) \
VALUES (?1, 0, ?2) \
ON CONFLICT(id) DO UPDATE SET \
started_at = COALESCE(library_data_migration.started_at, excluded.started_at)",
params![GENRE_TAGS_MIGRATION_ID, now_unix()],
)?;
let cursor = cursor_rowid(conn)?;
let mut stmt = conn.prepare(
"SELECT rowid, server_id, id, genre, \
CASE WHEN json_valid(raw_json) THEN json_extract(raw_json, '$.genres') END, \
album_id, library_id \
FROM track \
WHERE deleted = 0 AND rowid > ?1 \
ORDER BY rowid \
LIMIT ?2",
)?;
let rows: Vec<BackfillTrackRow> =
stmt
.query_map(params![cursor, BATCH_SIZE], |r| {
Ok((
r.get(0)?,
r.get(1)?,
r.get(2)?,
r.get(3)?,
r.get(4)?,
r.get(5)?,
r.get(6)?,
))
})?
.collect::<rusqlite::Result<Vec<_>>>()?;
if rows.is_empty() {
conn.execute(
"UPDATE library_data_migration SET completed_at = ?2, cursor_rowid = \
(SELECT COALESCE(MAX(rowid), 0) FROM track WHERE deleted = 0) \
WHERE id = ?1",
params![GENRE_TAGS_MIGRATION_ID, now_unix()],
)?;
return Ok((total as i64, true));
}
let tx = conn.unchecked_transaction()?;
let mut last_rowid = cursor;
for (rowid, server_id, track_id, genre, genres_json, album_id, library_id) in rows {
let genres = genres_for_track_extracted(
genres_json.as_deref(),
genre.as_deref(),
);
replace_track_genre_rows(
&tx,
&server_id,
&track_id,
album_id.as_deref(),
library_id.as_deref(),
&genres,
)?;
last_rowid = rowid;
}
tx.commit()?;
conn.execute(
"UPDATE library_data_migration SET cursor_rowid = ?2 WHERE id = ?1",
params![GENRE_TAGS_MIGRATION_ID, last_rowid],
)?;
let done: i64 = conn.query_row(
"SELECT COUNT(*) FROM track WHERE deleted = 0 AND rowid <= ?1",
params![last_rowid],
|r| r.get(0),
)?;
Ok((done, false))
})?;
if let Some(app) = app {
emit_progress(app, batch_done.max(0) as u64, total)?;
}
if finished {
break;
}
}
// Belt-and-suspenders: all live tracks processed but `completed_at` not set
// (can happen when rowid gaps from soft-deletes make done == total early).
store.with_conn_mut("genre_tags.backfill.finalize", |conn| {
if migration_completed(conn)? {
return Ok(());
}
let cursor = cursor_rowid(conn)?;
let pending: i64 = conn.query_row(
"SELECT COUNT(*) FROM track WHERE deleted = 0 AND rowid > ?1",
params![cursor],
|r| r.get(0),
)?;
if pending == 0 {
conn.execute(
"UPDATE library_data_migration SET completed_at = ?2, cursor_rowid = \
(SELECT COALESCE(MAX(rowid), 0) FROM track WHERE deleted = 0) \
WHERE id = ?1",
params![GENRE_TAGS_MIGRATION_ID, now_unix()],
)?;
}
Ok(())
})?;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::repos::track::{TrackRepository, TrackRow};
use crate::store::LibraryStore;
fn track(server_id: &str, id: &str, genre: &str, deleted: bool) -> TrackRow {
TrackRow {
server_id: server_id.into(),
id: id.into(),
title: id.into(),
title_sort: None,
artist: Some("Artist".into()),
artist_id: None,
album: "Album".into(),
album_id: Some("al1".into()),
album_artist: None,
duration_sec: 100,
track_number: Some(1),
disc_number: Some(1),
year: None,
genre: Some(genre.into()),
suffix: None,
bit_rate: None,
size_bytes: None,
cover_art_id: None,
starred_at: None,
user_rating: None,
play_count: None,
played_at: None,
server_path: None,
library_id: Some("lib1".into()),
isrc: None,
mbid_recording: None,
bpm: None,
replay_gain_track_db: None,
replay_gain_album_db: None,
content_hash: None,
server_updated_at: None,
server_created_at: None,
deleted,
synced_at: 1,
raw_json: "{}".into(),
}
}
#[test]
fn backfill_marks_complete_when_rowid_gaps_leave_pending_rows() {
let store = LibraryStore::open_in_memory();
let live: Vec<TrackRow> = (1..=5)
.map(|n| track("s1", &format!("t{n}"), "Rock", false))
.collect();
let mut batch = live;
for n in 6..=20 {
batch.push(track("s1", &format!("del{n}"), "Rock", true));
}
batch.push(track("s1", "t6", "Jazz", false));
TrackRepository::new(&store).upsert_batch(&batch).unwrap();
run_genre_tags_backfill_impl(&store, None).unwrap();
let inspect = inspect_genre_tags_backfill(&store).unwrap();
assert!(!inspect.needed, "backfill should complete despite rowid gaps");
}
}
@@ -25,6 +25,8 @@ pub mod dto;
pub mod enrichment;
pub mod filter;
pub mod genre_album_browse;
pub mod genre_tags;
pub mod genre_tags_backfill;
pub mod mood_groups;
pub mod live_search;
pub mod lossless_albums;
@@ -1,7 +1,23 @@
use rusqlite::{params, OptionalExtension};
use rusqlite::{params, OptionalExtension, Transaction};
use crate::genre_tags::{self, genres_for_track_raw_json};
use crate::store::{LibraryStore, WriteOpTiming};
fn sync_track_genre_row(tx: &Transaction<'_>, row: &TrackRow) -> rusqlite::Result<()> {
if row.deleted {
return genre_tags::delete_track_genre_for_track(tx, &row.server_id, &row.id);
}
let genres = genres_for_track_raw_json(&row.raw_json, row.genre.as_deref());
genre_tags::replace_track_genre_rows(
tx,
&row.server_id,
&row.id,
row.album_id.as_deref(),
row.library_id.as_deref(),
&genres,
)
}
/// One row of the `track` table — every hot column from spec §5.1 plus
/// `raw_json` (the full normalized SubsonicSong). Sync code (PR-2/PR-3) is
/// expected to project ingested payloads into this shape, not to talk SQL
@@ -181,6 +197,7 @@ impl<'a> TrackRepository<'a> {
r.raw_json,
])?;
}
sync_track_genre_row(&tx, r)?;
}
drop(upsert);
tx.commit()?;
@@ -204,6 +221,14 @@ impl<'a> TrackRepository<'a> {
pub fn sweep_resync_orphans(&self, server_id: &str, resync_gen: i64) -> Result<u32, String> {
let now = now_unix_ms();
let changed = self.store.with_conn_mut("track.sweep_resync_orphans", |c| {
c.execute(
"DELETE FROM track_genre \
WHERE server_id = ?1 AND track_id IN ( \
SELECT id FROM track \
WHERE server_id = ?1 AND deleted = 0 AND resync_gen != ?2 \
)",
params![server_id, resync_gen],
)?;
c.execute(
"UPDATE track SET deleted = 1, synced_at = ?3 \
WHERE server_id = ?1 AND deleted = 0 AND resync_gen != ?2",
@@ -477,6 +502,7 @@ impl<'a> TrackRepository<'a> {
r.synced_at,
r.raw_json,
])?;
sync_track_genre_row(&tx, r)?;
if let Some(old_id) = detected_old {
remap_existing_to_new(
@@ -1043,8 +1069,9 @@ mod tests {
repo.upsert_batch_initial_ingest(&rows).unwrap();
let elapsed = start.elapsed();
assert!(
elapsed < std::time::Duration::from_millis(500),
"initial ingest batch(500) took {elapsed:?}"
elapsed < std::time::Duration::from_millis(1000),
"initial ingest batch(500) took {elapsed:?}; includes per-row track_genre \
maintenance and large raw_json payloads"
);
}
+59 -7
View File
@@ -9,7 +9,7 @@ use tauri::Manager;
/// Current head of the embedded migrations. Bump each time a new
/// `migrations/NNN_*.sql` is added.
pub const LIBRARY_DB_SCHEMA_VERSION: i64 = 1;
pub const LIBRARY_DB_SCHEMA_VERSION: i64 = 12;
/// Lowest applied schema version the current code can advance from purely
/// additively. If a DB carries a version below this, the breaking-bump hook
@@ -22,10 +22,20 @@ pub const LIBRARY_DB_SCHEMA_VERSION: i64 = 1;
pub const LIBRARY_DB_MIN_COMPATIBLE_VERSION: i64 = 1;
pub(crate) const INITIAL_SQL: &str = include_str!("../migrations/001_initial.sql");
/// Version 12 is above the removed legacy migrations 002011 so existing DBs
/// still pick up `track_genre` + `library_data_migration`.
pub(crate) const MIGRATION_012_TRACK_GENRE_LEGACY: &str =
include_str!("../migrations/012_track_genre_legacy_repair.sql");
/// Embedded migrations. Ordered ascending by `version`; the runner sorts
/// defensively before applying so the source order can stay readable.
const MIGRATIONS: &[(i64, &str)] = &[(1, INITIAL_SQL)];
const MIGRATIONS: &[(i64, &str)] = &[(1, INITIAL_SQL), (12, MIGRATION_012_TRACK_GENRE_LEGACY)];
/// Idempotent repair — also runs after the migration runner on every open so
/// DBs that recorded the wrong version numbers still get the tables.
pub(crate) fn ensure_genre_tags_schema(conn: &Connection) -> rusqlite::Result<()> {
conn.execute_batch(MIGRATION_012_TRACK_GENRE_LEGACY)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum MigrationOutcome {
@@ -67,6 +77,7 @@ impl LibraryStore {
let write_conn = Connection::open(db_path).map_err(|e| e.to_string())?;
configure_write_connection(&write_conn).map_err(|e| e.to_string())?;
run_migrations(&write_conn).map_err(|e| e.to_string())?;
ensure_genre_tags_schema(&write_conn).map_err(|e| e.to_string())?;
checkpoint_wal_conn(&write_conn, "open").map_err(|e| e.to_string())?;
let read_conn = Connection::open_with_flags(db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)
.map_err(|e| e.to_string())?;
@@ -84,6 +95,7 @@ impl LibraryStore {
let write_conn = Connection::open(&uri).expect("in-memory write connection");
configure_write_connection(&write_conn).expect("write pragmas");
run_migrations(&write_conn).expect("schema migration");
ensure_genre_tags_schema(&write_conn).expect("genre tags schema");
let read_conn = Connection::open(&uri).expect("in-memory read connection");
configure_read_connection(&read_conn).expect("read pragmas");
Self {
@@ -556,8 +568,7 @@ mod tests {
rows
})
.unwrap();
// Embedded migrations are numbered 1..=head, all applied on a fresh DB.
let expected: Vec<i64> = (1..=LIBRARY_DB_SCHEMA_VERSION).collect();
let expected: Vec<i64> = MIGRATIONS.iter().map(|(version, _)| *version).collect();
assert_eq!(versions, expected);
}
@@ -574,11 +585,53 @@ mod tests {
})
.unwrap();
assert_eq!(
count, LIBRARY_DB_SCHEMA_VERSION,
count,
MIGRATIONS.len() as i64,
"one schema_migrations row per embedded migration, no duplicates"
);
}
#[test]
fn migration_012_repairs_db_that_recorded_legacy_versions_without_genre_tables() {
let uri = in_memory_uri();
let conn = Connection::open(&uri).expect("connection");
configure_write_connection(&conn).expect("pragmas");
conn.execute_batch(INITIAL_SQL).expect("initial");
conn.execute("DROP TABLE IF EXISTS track_genre", [])
.expect("drop track_genre");
conn.execute("DROP TABLE IF EXISTS library_data_migration", [])
.expect("drop cursor table");
for version in 1..=11_i64 {
conn.execute(
"INSERT INTO schema_migrations (version, applied_at) VALUES (?1, ?1)",
params![version],
)
.expect("seed legacy versions");
}
let outcome = run_migrations_with(
&conn,
MIGRATIONS,
LIBRARY_DB_MIN_COMPATIBLE_VERSION,
no_op_hook,
)
.expect("apply v12 repair");
assert_eq!(outcome, MigrationOutcome::Applied);
ensure_genre_tags_schema(&conn).expect("ensure");
for table in ["track_genre", "library_data_migration"] {
let exists: i64 = conn
.query_row(
"SELECT COUNT(*) FROM sqlite_master \
WHERE type = 'table' AND name = ?1",
params![table],
|r| r.get(0),
)
.expect("table probe");
assert_eq!(exists, 1, "missing table {table}");
}
}
#[test]
fn fts_virtual_table_exists() {
let store = LibraryStore::open_in_memory();
@@ -656,8 +709,7 @@ mod tests {
rows
})
.unwrap();
// Real embedded migrations (1..=head) plus the additive fixture.
let mut expected: Vec<i64> = (1..=LIBRARY_DB_SCHEMA_VERSION).collect();
let mut expected: Vec<i64> = MIGRATIONS.iter().map(|(version, _)| *version).collect();
expected.push(FIXTURE_ADD_BIO_VERSION);
assert_eq!(versions, expected);
}
@@ -139,6 +139,10 @@ impl<'a> TombstoneReconciler<'a> {
WHERE server_id = ?1 AND id = ?2",
rusqlite::params![self.server_id, id, now_unix_ms()],
)?;
c.execute(
"DELETE FROM track_genre WHERE server_id = ?1 AND track_id = ?2",
rusqlite::params![self.server_id, id],
)?;
Ok(())
})
.map_err(SyncError::Storage)
+2
View File
@@ -768,6 +768,8 @@ pub fn run() {
psysonic_library::commands::library_advanced_search,
psysonic_library::commands::library_list_lossless_albums,
psysonic_library::commands::library_list_albums_by_genre,
psysonic_library::commands::library_genre_tags_inspect,
psysonic_library::commands::library_genre_tags_run,
psysonic_library::commands::library_get_artist_lossless_browse,
psysonic_library::commands::library_search_cross_server,
psysonic_library::commands::library_get_track,