From d353482ac5d8918d26e024e3268cd76be1ff2b1c Mon Sep 17 00:00:00 2001 From: Frank Stellmacher <171614930+Psychotoxical@users.noreply.github.com> Date: Wed, 27 May 2026 00:21:51 +0200 Subject: [PATCH] fix(analysis): cap HTTP backfill on CPU-seed pipeline load (#873) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(analysis): cap HTTP backfill on CPU-seed pipeline load Aggressive library analytics with multiple workers grew RAM unbounded: HTTP downloads finish in ~500 ms, but Symphonia decode + R128 loudness take seconds, so decoded `Vec` track buffers piled up in the CPU-seed queue while the HTTP worker kept fetching. On large libraries the process eventually saturated memory and forced the system into swap. Backpressure: the HTTP backfill worker now checks the CPU-seed pipeline depth (queued + running) against a `workers * 2` cap before popping the next job. High-priority (now-playing) jobs bypass the cap so playback prefetch is never starved. The CPU-seed worker pings the HTTP queue after every completed decode, so the gate releases the instant decode catches up. The frontend backfill loop already idles at its own watermark when the HTTP queue is satisfied — this change keeps the pipeline self-limiting end-to-end even when callers (library top-up, playlist enqueue) submit faster than decode drains. Tests cover cap scaling, the floor for workers=1, the idle decision, and the high-priority bypass. * docs(changelog): analytics aggressive scan no longer eats memory (#873) --- CHANGELOG.md | 9 ++ .../psysonic-analysis/src/analysis_runtime.rs | 91 ++++++++++++++++++- 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b0d260..ba580a13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -365,6 +365,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 +### Analytics — aggressive scan no longer eats memory on big libraries + +**By [@Psychotoxical](https://github.com/Psychotoxical), PR [#873](https://github.com/Psychotoxical/psysonic/pull/873)** + +* **Settings → Library → Analytics → Aggressive** on multi-server or 100k+ track libraries no longer climbs in memory until the system swaps (Linux) or runs out (Windows OOM mid-scan): the HTTP download stage now waits for Symphonia decode + loudness to catch up instead of buffering tracks faster than they can be processed. +* Now-playing prefetch still bypasses the cap, so starting a track during a background scan stays instant. + + + ## [1.46.0] - 2026-05-18 > **🙏 Special thanks to [@zz5zz](https://github.com/zz5zz)** for his tireless quirk-spotting and bug reports on the [Psysonic Discord](https://discord.gg/AMnDRErm4u) — several of the polish fixes in this release landed directly off the back of his messages. diff --git a/src-tauri/crates/psysonic-analysis/src/analysis_runtime.rs b/src-tauri/crates/psysonic-analysis/src/analysis_runtime.rs index 2e169a99..164817b0 100644 --- a/src-tauri/crates/psysonic-analysis/src/analysis_runtime.rs +++ b/src-tauri/crates/psysonic-analysis/src/analysis_runtime.rs @@ -554,20 +554,66 @@ async fn analysis_backfill_worker_loop( } } +/// Queued + currently-decoding CPU-seed jobs. Each retains the full track +/// byte buffer, so this counter approximates pipeline memory pressure. +fn cpu_seed_pipeline_load() -> usize { + let Some(shared) = ANALYSIS_CPU_SEED.get() else { + return 0; + }; + let st = shared.state.lock().unwrap_or_else(|e| e.into_inner()); + st.queued_len() + st.running.len() +} + +/// Soft cap on in-flight CPU-seed jobs (queued + running). When reached, the +/// HTTP backfill worker idles to keep decoded `Vec` buffers from piling up +/// faster than Symphonia + R128 can drain them. Floor of 2 covers `workers=1`. +fn cpu_seed_pipeline_cap(max_parallel: usize) -> usize { + max_parallel.saturating_mul(2).max(2) +} + +/// Decide whether the HTTP backfill worker should idle right now. High-tier +/// (now-playing) jobs always bypass the cap so playback is never starved. +fn should_idle_for_cpu_backpressure( + cpu_load: usize, + cpu_cap: usize, + high_pending: bool, +) -> bool { + !high_pending && cpu_load >= cpu_cap +} + async fn spawn_backfill_slots(app: &tauri::AppHandle, shared: &Arc) { loop { let max = shared.max_parallel(); + // Backpressure against the CPU-seed pipeline: downloaded track bytes + // (Vec, tens of MB for FLAC) sit in `AnalysisCpuSeedJob.bytes` until + // Symphonia decode + R128 finish — much slower than HTTP. Without a cap, + // aggressive library backfill on large libraries grows RAM unbounded. + // High-tier (now-playing) jobs always proceed. + let cpu_load = cpu_seed_pipeline_load(); + let cpu_cap = cpu_seed_pipeline_cap(max); let job_bundle = { let mut st = shared .state .lock() .unwrap_or_else(|e| e.into_inner()); - st.try_pop_next(max).map(|job| { - let worker_slot = st.in_progress.len(); - (job, worker_slot) - }) + let high_pending = !st.high.is_empty(); + if should_idle_for_cpu_backpressure(cpu_load, cpu_cap, high_pending) { + None + } else { + st.try_pop_next(max).map(|job| { + let worker_slot = st.in_progress.len(); + (job, worker_slot) + }) + } }; let Some(((track_id, url, server_id), worker_slot)) = job_bundle else { + if cpu_load >= cpu_cap { + crate::app_deprintln!( + "[analysis] backfill idle: cpu_seed pipeline_load={} cap={} (waiting for decode catch-up)", + cpu_load, + cpu_cap + ); + } break; }; crate::app_deprintln!( @@ -1112,6 +1158,11 @@ async fn spawn_cpu_seed_slots(app: &tauri::AppHandle, shared: &Arc { @@ -1560,4 +1611,36 @@ mod tests { let result = rx.blocking_recv().expect("sender side should have closed cleanly"); assert!(result.is_err(), "pruned job must yield Err, got {result:?}"); } + + // ── CPU-seed backpressure ───────────────────────────────────────────────── + + #[test] + fn cpu_seed_pipeline_cap_scales_with_workers() { + assert_eq!(cpu_seed_pipeline_cap(1), 2); + assert_eq!(cpu_seed_pipeline_cap(3), 6); + assert_eq!(cpu_seed_pipeline_cap(6), 12); + assert_eq!(cpu_seed_pipeline_cap(20), 40); + } + + #[test] + fn cpu_seed_pipeline_cap_has_floor_of_two() { + assert_eq!(cpu_seed_pipeline_cap(0), 2); + } + + #[test] + fn backpressure_idles_when_cpu_load_meets_cap_and_no_high() { + assert!(should_idle_for_cpu_backpressure(12, 12, false)); + assert!(should_idle_for_cpu_backpressure(20, 12, false)); + } + + #[test] + fn backpressure_allows_pop_when_cpu_load_below_cap() { + assert!(!should_idle_for_cpu_backpressure(11, 12, false)); + assert!(!should_idle_for_cpu_backpressure(0, 12, false)); + } + + #[test] + fn backpressure_bypassed_for_high_priority_jobs() { + assert!(!should_idle_for_cpu_backpressure(100, 12, true)); + } }