From 10c362a94a1709258bf0a2dfe0dc35142b9c74c1 Mon Sep 17 00:00:00 2001 From: Edouard Vanbelle Date: Sun, 30 Aug 2026 19:41:41 +0200 Subject: [PATCH] fix(jobs): flush the checkpoint tail, so progress reflects reality MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All three import jobs only checkpointed on a full batch, so the remainder after the last one was never counted. A run shorter than BATCH_SIZE never checkpointed at all: `scanned_count` stayed 0 against a known `total_rows`, and the admin progress bar sat at zero for the whole run and finished there. Seen on a transcode_import run over 20 entries — 13 imported, 5 negatives, 2 already present, progress 0/20 throughout. The thumbnail imports had it too, just less visibly: a 105-file run reported `scanned_count: 100`, losing the tail rather than all of it. Cursor-wise the final checkpoint is a no-op — the walk is finished, so nothing resumes from it — but the scanned delta is what the progress display reads, and it has to include the last partial batch. Co-Authored-By: Claude Opus 5 (1M context) --- .../services/thumb_attached_import_service.rs | 12 +++++++++ .../services/thumb_derived_import_service.rs | 13 ++++++++++ .../services/transcode_import_service.rs | 26 +++++++++++++++++++ 3 files changed, 51 insertions(+) diff --git a/src/infrastructure/services/thumb_attached_import_service.rs b/src/infrastructure/services/thumb_attached_import_service.rs index 14748df6..92917d55 100644 --- a/src/infrastructure/services/thumb_attached_import_service.rs +++ b/src/infrastructure/services/thumb_attached_import_service.rs @@ -476,6 +476,18 @@ impl RecoverableJobHandler for ThumbAttachedImport { } } + // Flush the tail — see the derived twin. The loop only checkpoints + // on a full batch, so the remainder went uncounted: a 105-file run + // reported `scanned_count: 100`, and a run shorter than one batch + // reported zero and left the progress bar at zero throughout. + if since_checkpoint > 0 + && let Err(e) = store.checkpoint(Vec::new(), since_checkpoint as u64).await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + // Both jobs attempt the teardown, and it no-ops unless the tree is // drained of files EITHER of them claims. Without this, whichever // job runs last leaves an empty `.thumbnails/` behind until the diff --git a/src/infrastructure/services/thumb_derived_import_service.rs b/src/infrastructure/services/thumb_derived_import_service.rs index 490f272a..37e67200 100644 --- a/src/infrastructure/services/thumb_derived_import_service.rs +++ b/src/infrastructure/services/thumb_derived_import_service.rs @@ -734,6 +734,19 @@ impl RecoverableJobHandler for ThumbDerivedImport { } } + // Flush the tail. The loop only checkpoints on a full batch, so the + // remainder after the last one was never counted — a run of fewer + // than BATCH_SIZE files reported `scanned_count: 0` against a known + // total and left the admin progress bar at zero for its whole life. + // Same fix in both imports and in transcode_import. + if since_checkpoint > 0 + && let Err(e) = store.checkpoint(Vec::new(), since_checkpoint as u64).await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + // Remove the size directories once genuinely empty, because ABSENCE // is what step 10e gates the fallback removal on — not emptiness. // Empty is momentary: an on-demand render can repopulate it the next diff --git a/src/infrastructure/services/transcode_import_service.rs b/src/infrastructure/services/transcode_import_service.rs index 83c29d38..27c6963d 100644 --- a/src/infrastructure/services/transcode_import_service.rs +++ b/src/infrastructure/services/transcode_import_service.rs @@ -253,6 +253,8 @@ impl RecoverableJobHandler for TranscodeImport { let mut unverified = 0u64; let mut failed = 0u64; let mut since_checkpoint = 0usize; + // Last entry visited, for the tail flush below. + let mut last_name: Option = None; for name in Self::entry_names(&dir).await { if let Some(c) = &cursor @@ -464,11 +466,35 @@ impl RecoverableJobHandler for TranscodeImport { } since_checkpoint += 1; + last_name = Some(name.clone()); if let Some(failure) = checkpoint_if_due(store, &name, &mut since_checkpoint).await { return failure; } } + // Flush the tail. + // + // `checkpoint_if_due` only fires on a full batch, so a run shorter + // than BATCH_SIZE never checkpointed at all and reported + // `scanned_count: 0` against a known `total_rows` — the admin + // progress bar sat at zero through the whole run and finished + // there. Longer runs were wrong too, just less visibly: the + // remainder after the last full batch was never counted. + // + // Cursor-wise this is a no-op — the walk is finished, so nothing + // will resume from it — but the scanned delta is what the progress + // display reads, and it has to include the last partial batch. + if since_checkpoint > 0 + && let Some(name) = last_name + && let Err(e) = store + .checkpoint(name.into_bytes(), since_checkpoint as u64) + .await + { + return RunOutcome::Failed { + message: format!("final checkpoint: {e}"), + }; + } + // Remove the tree once drained. Deletion first, rename only if a // non-cache file is in the way — same rule as `.thumbnails/`, and // for the same reason: absence is what the read path tests, and a