import { test } from "node:test"; import assert from "node:assert/strict"; import { createDiarizeProgressParser, createDownloadProgressParser, formatRsyncProgressDetail, isRsyncProgressLine, parseRsyncProgress, rsyncProgressFraction, } from "./progressParsers"; import { formatBytes } from "../lib/format"; // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test jobs/progressParsers.test.ts // Subtitle-only (youtube --skip-download) progress lines, as seen in real job // logs: total=NA while downloading, a real total only on the finished line. const DL_NA = "DLOM_PROGRESS status=downloading downloaded=7168 total=NA frag=NA/NA speed=3442662 eta=NA"; const DL_FIN = "DLOM_PROGRESS status=finished downloaded=177273 total=177273 frag=NA/NA speed=745217 eta=NA"; test("subtitle-only download advances one step per completed track", () => { const p = createDownloadProgressParser(); // Without a total, byte fraction can't move — but once the track count is // known the bar steps per finished track. assert.equal(p.feed("[info] vid: Downloading subtitles: en-orig, en"), null); let u = p.feed(DL_NA); // first track, downloading assert.equal(u?.fraction, 0); assert.match(u?.detail ?? "", /subs 0\/2/); u = p.feed(DL_FIN); // first track done assert.equal(u?.fraction, 0.5); assert.match(u?.detail ?? "", /subs 1\/2/); u = p.feed(DL_NA); // second track, downloading assert.equal(u?.fraction, 0.5); u = p.feed(DL_FIN); // second track done assert.equal(u?.fraction, 1); assert.match(u?.detail ?? "", /subs 2\/2/); }); test("without the subtitle announcement, behaviour is unchanged (jumps at finish)", () => { const p = createDownloadProgressParser(); const downloading = p.feed(DL_NA); // total=NA, no track count → no fraction, just whatever detail it can build. assert.equal(downloading?.fraction, undefined); const finished = p.feed(DL_FIN); assert.equal(finished?.fraction, 1); // finished line still reports 100% }); test("a real media download is byte-based, not stepped, even after a subs announce", () => { const p = createDownloadProgressParser(); // youtube no-subs fallback: subs announced, then it switches to an audio // download that DOES report a byte total — subtitle-step mode must release. p.feed("[info] vid: Downloading subtitles: en-orig, en"); const u = p.feed( "DLOM_PROGRESS status=downloading downloaded=500 total=1000 frag=NA/NA speed=1000 eta=1", ); assert.equal(u?.fraction, 0.5); // 500/1000, not a subtitle step assert.doesNotMatch(u?.detail ?? "", /subs /); }); test("media download fraction is byte-based from the start", () => { const p = createDownloadProgressParser(); const u = p.feed( "DLOM_PROGRESS status=downloading downloaded=250 total=1000 frag=NA/NA speed=1000 eta=3", ); assert.equal(u?.fraction, 0.25); }); // --------------------------------------------------------------------------- // Diarization. The lines are real output from scripts/diarize-sherpa.py. test("diarization reports window progress with a measured ETA", () => { const p = createDiarizeProgressParser(); // The plan line makes the bar determinate before the first window finishes — // which on a 6-hour file is several minutes of otherwise-blank spinner. const planned = p.feed( "diarize-sherpa: 6.20 h -> 9 window(s) of 45 min (15s overlap), windowed mode", ); assert.equal(planned?.fraction, 0); assert.equal(planned?.detail, "0/9 windows"); const first = p.feed( "diarize-sherpa: window 1/9 (0-45 min): 412 turns, 6 local speaker(s) in 300.0s", ); // Printed AFTER the window is processed, so 1/9 is one window DONE. assert.equal(first?.fraction, 1 / 9); // 8 windows left at the one measured time: 2400s = 40:00. assert.equal(first?.detail, "window 1/9 · ETA 40:00"); // The mean moves with the second sample: (300+180)/2 = 240 × 7 = 1680 = 28:00. const second = p.feed( "diarize-sherpa: window 2/9 (45-90 min): 300 turns, 4 local speaker(s) in 180.0s", ); assert.equal(second?.fraction, 2 / 9); assert.equal(second?.detail, "window 2/9 · ETA 28:00"); }); test("diarization drops the ETA on the last window rather than showing 0:00", () => { const p = createDiarizeProgressParser(); p.feed("diarize-sherpa: window 1/2 (0-45 min): 10 turns, 2 local speaker(s) in 60.0s"); const last = p.feed( "diarize-sherpa: window 2/2 (45-90 min): 10 turns, 2 local speaker(s) in 60.0s", ); assert.equal(last?.fraction, 1); assert.equal(last?.detail, "window 2/2"); }); test("a skipped window leaves the bar alone rather than inventing a step", () => { const p = createDiarizeProgressParser(); const first = p.feed( "diarize-sherpa: window 1/4 (0-45 min): 10 turns, 2 local speaker(s) in 100.0s", ); assert.equal(first?.fraction, 0.25); // The REAL zero-samples line (diarize-sherpa.py) prints a bare index with no // "/N" and no elapsed time: "window 2 decoded zero samples; skipping". So it // matches nothing here and the bar holds until window 3 reports — which is // the correct outcome, and the reason this asserts the actual string rather // than a plausible-looking one. assert.equal( p.feed("diarize-sherpa: window 2 decoded zero samples; skipping"), null, ); // The mean is untouched by the skip: still 100s, 2 windows left = 3:20. const third = p.feed( "diarize-sherpa: window 3/4 (90-135 min): 10 turns, 2 local speaker(s) in 100.0s", ); assert.equal(third?.detail, "window 3/4 · ETA 1:40"); }); test("the diarize parser ignores everything that is not its own output", () => { const p = createDiarizeProgressParser(); // Non-windowed short files print no per-window line: indeterminate is honest. assert.equal(p.feed("diarize-sherpa: 3.2 min decoded, diarizing…"), null); // And a backfill task that is re-acquiring media must fall through to the // download parser untouched — this parser must not claim yt-dlp's lines. assert.equal( p.feed( "DLOM_PROGRESS status=downloading downloaded=500 total=1000 frag=NA/NA speed=1000 eta=1", ), null, ); assert.equal(p.feed("[download] 50.0% of 1.00GiB at 2.31MiB/s ETA 00:30"), null); }); // --- rsync --info=progress2 ------------------------------------------------- // // The exact bytes rsync 3.x writes, captured off a real copy (two files, 35 MB) // and pasted verbatim: one \r-separated buffer, the early frame with no xfr# // suffix, a mid frame, and the two identical final frames rsync always prints. const RSYNC_CHUNK = "\r 32,768 0% 0.00kB/s 0:00:00 " + "\r 30,000,000 85% 1.03GB/s 0:00:03 (xfr#1, to-chk=1/3)" + "\r 35,000,000 100% 1.05GB/s 0:00:00 (xfr#2, to-chk=0/3)"; test("rsync progress: bytes, percent, rate and ETA off one frame", () => { const p = parseRsyncProgress( " 30,000,000 85% 1.03GB/s 0:01:03 (xfr#1, to-chk=1/3)", ); assert.deepEqual(p, { bytes: 30_000_000, percent: 85, rate: "1.03GB/s", etaSeconds: 63, }); }); test("rsync progress: a whole \\r buffer splits into frames", () => { const frames = RSYNC_CHUNK.split(/[\r\n]+/) .filter((l) => l.trim() !== "") .map(parseRsyncProgress); assert.equal(frames.length, 3); assert.deepEqual( frames.map((f) => f?.bytes), [32_768, 30_000_000, 35_000_000], ); assert.deepEqual( frames.map((f) => f?.percent), [0, 85, 100], ); }); test("rsync progress: an hours-long ETA is seconds, not a string", () => { assert.equal( parseRsyncProgress(" 1,000 10% 1.00MB/s 2:03:04")?.etaSeconds, 2 * 3600 + 3 * 60 + 4, ); }); test("rsync progress: everything that is not a frame is not a frame", () => { for (const line of [ "sending incremental file list", "20240101_test1234567/", "sent 35,012,345 bytes received 4,321 bytes 70,033,332.00 bytes/sec", "total size is 35,000,000 speedup is 1.00", ".d..t...... 20240101_test1234567/", "", "rsync: [sender] change_dir failed: No such file or directory (2)", ]) { assert.equal(parseRsyncProgress(line), null, line); assert.equal(isRsyncProgressLine(line), false, line); } }); // THE FRACTION IS AGAINST THE MEASURED TREE, not rsync's percentage — which // under incremental recursion is a percentage of what it has enumerated so far // and walks backwards. The controller has already measured the whole tree for // its space check, so it has a denominator that only moves one way. test("rsync progress: the fraction divides by the measured tree", () => { const p = parseRsyncProgress(" 30,000,000 85% 1.03GB/s 0:00:03")!; assert.equal(rsyncProgressFraction(p, 60_000_000), 0.5); // No measurement (a caller that never walked the tree) falls back to rsync's // own percentage rather than reporting zero. assert.equal(rsyncProgressFraction(p, 0), 0.85); // Clamped: a tree that grew under the copy must not report 140 %. assert.equal(rsyncProgressFraction(p, 10_000_000), 1); }); test("rsync progress: the detail line is the one wording", () => { const p = parseRsyncProgress(" 30,000,000 85% 1.03GB/s 0:05:32")!; assert.equal( formatRsyncProgressDetail(p, 60_000_000, formatBytes), "28.6 MB of 57.2 MB · 50 % · 1.03GB/s · ETA 5:32", ); // A finished frame prints 0:00:00, and an ETA of zero is noise. const done = parseRsyncProgress(" 60,000,000 100% 1.05GB/s 0:00:00")!; assert.equal( formatRsyncProgressDetail(done, 60_000_000, formatBytes), "57.2 MB of 57.2 MB · 100 % · 1.05GB/s", ); }); // A TORN FRAME MUST NEVER PARSE AS A FRAME. A chunk boundary falls wherever the // pipe decides, and it lands inside a progress line often enough to matter on a // long copy. The halves are the hazard, not the loss: ` 30,000,` still // matches nothing, but the NEXT chunk's leading ` 30` can parse as a frame // reporting thirty bytes — the bar jumps back to 0 % and the decile latch has // already spent its line. relocateDir.ts holds the trailing segment back; this // pins what each half does on its own so that buffering is provably necessary. test("rsync progress: half a frame is not a frame, and two halves are one", () => { const whole = " 30,000,000 85% 1.03GB/s 0:00:03 (xfr#1, to-chk=1/3)"; const cut = 12; const head = whole.slice(0, cut); const tail = whole.slice(cut); // The head alone has no percentage, so it cannot parse. assert.equal(parseRsyncProgress(head), null); // The tail alone is the dangerous one: rejoined wrongly it would be read as a // frame about a handful of bytes. const strayTail = parseRsyncProgress(tail); if (strayTail !== null) { assert.notEqual(strayTail.bytes, 30_000_000); } // Rejoined, it is the frame it always was. assert.deepEqual(parseRsyncProgress(head + tail), { bytes: 30_000_000, percent: 85, rate: "1.03GB/s", etaSeconds: 3, }); });