fix(analysis): persist library backfill scan phase across coordinator ticks (#882)

* fix(analysis): persist backfill scan phase and cursor across coordinator ticks

Keep HashBpmGaps progress when the candidate SQL page is empty only because
id > cursor; store scan phase in the native worker so each tick does not
restart from Candidates and rescan the first ~10k ready tracks.

* docs: CHANGELOG and credits for PR #882 backfill scan phase fix

* fix(analysis): move backfill tests below production code for clippy

Clippy items_after_test_module requires all non-test items before mod tests.
This commit is contained in:
cucadmuh
2026-05-28 11:26:24 +03:00
committed by GitHub
parent b24a7fc5cb
commit 8004ec559c
5 changed files with 115 additions and 23 deletions
@@ -29,23 +29,64 @@ pub struct LibraryAnalysisProgressDto {
pub done_tracks: i64,
}
enum ScanMode {
/// Persisted across native coordinator ticks (see `library_analysis_backfill` worker).
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, serde::Serialize, serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum AnalysisBackfillScanPhase {
#[default]
Candidates,
/// Tracks with hash + BPM that may still need waveform/LUFS/enrichment gaps.
HashBpmGaps,
}
fn begin_hash_bpm_gap_scan() -> (ScanMode, Option<String>) {
enum ScanMode {
Candidates,
HashBpmGaps,
}
impl From<AnalysisBackfillScanPhase> for ScanMode {
fn from(p: AnalysisBackfillScanPhase) -> Self {
match p {
AnalysisBackfillScanPhase::Candidates => ScanMode::Candidates,
AnalysisBackfillScanPhase::HashBpmGaps => ScanMode::HashBpmGaps,
}
}
}
impl From<ScanMode> for AnalysisBackfillScanPhase {
fn from(m: ScanMode) -> Self {
match m {
ScanMode::Candidates => AnalysisBackfillScanPhase::Candidates,
ScanMode::HashBpmGaps => AnalysisBackfillScanPhase::HashBpmGaps,
}
}
}
/// End of the cheap candidate SQL pass — start hash/BPM gap scan from the first id.
fn begin_hash_bpm_gap_scan_from_start() -> (ScanMode, Option<String>) {
(ScanMode::HashBpmGaps, None)
}
/// Candidate page empty only because `id > cursor`; continue the id walk in hash/BPM phase.
fn begin_hash_bpm_gap_scan_from_cursor(cursor: String) -> (ScanMode, Option<String>) {
(ScanMode::HashBpmGaps, Some(cursor))
}
fn advance_after_empty_candidate_page(after: Option<String>) -> (ScanMode, Option<String>) {
match after {
None => begin_hash_bpm_gap_scan_from_start(),
Some(cursor) => begin_hash_bpm_gap_scan_from_cursor(cursor),
}
}
pub fn collect_analysis_backfill_batch(
app: &AppHandle,
runtime: &LibraryRuntime,
server_id: &str,
phase: AnalysisBackfillScanPhase,
cursor: Option<&str>,
limit: Option<u32>,
) -> Result<LibraryAnalysisBackfillBatchDto, String> {
) -> Result<(LibraryAnalysisBackfillBatchDto, AnalysisBackfillScanPhase), String> {
let want = limit.unwrap_or(DEFAULT_BATCH).min(MAX_BATCH) as usize;
let needs_work = app
.try_state::<TrackAnalysisNeedsWorkQuery>()
@@ -54,7 +95,7 @@ pub fn collect_analysis_backfill_batch(
let repo = TrackRepository::new(&runtime.store);
let mut found = Vec::with_capacity(want);
let mut after = cursor.map(str::to_string);
let mut mode = ScanMode::Candidates;
let mut mode = ScanMode::from(phase);
let mut scanned = 0usize;
while found.len() < want && scanned < MAX_SCAN_IDS_PER_CALL {
@@ -70,15 +111,16 @@ pub fn collect_analysis_backfill_batch(
if page.is_empty() {
match mode {
ScanMode::Candidates => {
(mode, after) = begin_hash_bpm_gap_scan();
(mode, after) = advance_after_empty_candidate_page(after);
continue;
}
ScanMode::HashBpmGaps => {
return Ok(LibraryAnalysisBackfillBatchDto {
let dto = LibraryAnalysisBackfillBatchDto {
track_ids: found,
next_cursor: after,
exhausted: true,
});
};
return Ok((dto, AnalysisBackfillScanPhase::HashBpmGaps));
}
}
}
@@ -105,24 +147,26 @@ pub fn collect_analysis_backfill_batch(
if page_len < SCAN_CHUNK {
match mode {
ScanMode::Candidates => {
(mode, after) = begin_hash_bpm_gap_scan();
(mode, after) = begin_hash_bpm_gap_scan_from_start();
}
ScanMode::HashBpmGaps => {
return Ok(LibraryAnalysisBackfillBatchDto {
let dto = LibraryAnalysisBackfillBatchDto {
track_ids: found,
next_cursor: after,
exhausted: true,
});
};
return Ok((dto, AnalysisBackfillScanPhase::HashBpmGaps));
}
}
}
}
Ok(LibraryAnalysisBackfillBatchDto {
let dto = LibraryAnalysisBackfillBatchDto {
track_ids: found,
next_cursor: after,
exhausted: false,
})
};
Ok((dto, ScanMode::into(mode)))
}
pub fn collect_analysis_progress(
@@ -170,3 +214,29 @@ pub fn collect_analysis_progress(
done_tracks: done,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn empty_candidate_page_with_cursor_keeps_id_walk() {
let (mode, after) = advance_after_empty_candidate_page(Some("track-z".to_string()));
assert!(matches!(mode, ScanMode::HashBpmGaps));
assert_eq!(after.as_deref(), Some("track-z"));
}
#[test]
fn empty_candidate_page_without_cursor_starts_gap_scan_at_beginning() {
let (mode, after) = advance_after_empty_candidate_page(None);
assert!(matches!(mode, ScanMode::HashBpmGaps));
assert!(after.is_none());
}
#[test]
fn finished_candidate_phase_resets_gap_scan_cursor() {
let (mode, after) = begin_hash_bpm_gap_scan_from_start();
assert!(matches!(mode, ScanMode::HashBpmGaps));
assert!(after.is_none());
}
}
@@ -82,13 +82,15 @@ pub fn library_analysis_backfill_batch(
cursor: Option<String>,
limit: Option<u32>,
) -> Result<LibraryAnalysisBackfillBatchDto, String> {
analysis_backfill::collect_analysis_backfill_batch(
let (dto, _) = analysis_backfill::collect_analysis_backfill_batch(
&app,
&runtime,
server_id.trim(),
analysis_backfill::AnalysisBackfillScanPhase::Candidates,
cursor.as_deref().filter(|s| !s.is_empty()),
limit,
)
)?;
Ok(dto)
}
#[tauri::command]