commit 2333e8af60b20aca916589954c471023238e3f6f
parent 158b0603e7e563bbcc745397b21e70b735289fe1
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 02:20:06 -0400
common: persistKept counts a failed forced media pass as failed, not persisted
O3 review low 2 (pre-existing since release 10 slice N). A failed forced
pass over a transcript leaves the download ok, and persistKept counted every
returned download as persisted. It now reads sourceFetchFailure.
persistKeptForceMedia.test.ts: the setup is one helper with a process-wide
settings file (getPaths caches the first SETTINGS_FILE), plus a failing
media pass case.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 122 insertions(+), 65 deletions(-)
diff --git a/common/controller/persistKept.ts b/common/controller/persistKept.ts
@@ -7,7 +7,10 @@ import { resolveCookiePolicy } from "../lib/cookiePolicy";
import { isSavedVideo } from "../lib/savedVideo-server";
import { computeKeptVideoIds } from "./keptVideos";
import { findVideoSourceUrl } from "./undownloadedVideos";
-import { downloadOneManaged } from "../ytdlp/downloadOneManaged";
+import {
+ downloadOneManaged,
+ sourceFetchFailure,
+} from "../ytdlp/downloadOneManaged";
// Bulk "persist kept now" pass (Phase 5). Ensures every video currently in the
// channel's keep-latest window has its source container saved to the store, for
@@ -119,7 +122,7 @@ export async function persistKept({
}
log(` ${videoId}: re-fetching source container to persist…`);
try {
- await downloadOneManaged({
+ const record = await downloadOneManaged({
channelSlug,
channelConfig,
paths,
@@ -136,7 +139,16 @@ export async function persistKept({
// captions, which is every kept one (release 10 slice N).
forceMedia: true,
});
- result.persisted += 1;
+ // A download that RETURNED is not a persisted source (release 11 slice
+ // O3, review low 2): a failed forced media pass leaves the download `ok`
+ // (the transcript is fine) with only its n: 3 attempt failed.
+ const failure = sourceFetchFailure(record);
+ if (failure) {
+ result.failed += 1;
+ log(` ${videoId}: persist failed — ${failure}`);
+ } else {
+ result.persisted += 1;
+ }
} catch (e) {
result.failed += 1;
log(` ${videoId}: persist failed — ${(e as Error).message}`);
diff --git a/common/controller/persistKeptForceMedia.test.ts b/common/controller/persistKeptForceMedia.test.ts
@@ -47,6 +47,11 @@ if (has("--skip-download") && has("--write-auto-subs")) {
process.exit(0);
}
if ((arg("-o") || "").includes("source-media")) {
+ if (fs.existsSync(path.join(root, "media-fail"))) {
+ fs.writeFileSync(path.join(dir, "source-media.f137.mp4.part"), "half a container");
+ console.error("ERROR: [download] Got error: HTTP Error 403: Forbidden");
+ process.exit(1);
+ }
fs.writeFileSync(path.join(dir, "source-media.mp4"), "container bytes for " + id);
console.log("DLOM_ARCHIVE youtube " + id);
process.exit(0);
@@ -55,83 +60,123 @@ console.error("fake: unknown invocation " + argv.join(" "));
process.exit(2);
`;
-test("persistKept forces the media pass: a kept youtube-handling video with a transcript is persisted, with no audio", async () => {
- const root = await mkdtemp(path.join(tmpdir(), "persist-kept-force-"));
- try {
- // Disk floor off, so the per-item gate cannot turn this into a skip on a
- // full disk. Set before persistKept's first getSettings() → getPaths().
- const settingsFile = path.join(root, "settings.json");
- await writeFile(settingsFile, JSON.stringify({ minFreeDiskGB: 0 }));
- process.env.SETTINGS_FILE = settingsFile;
- process.env.TRANSCRIPTS_DIR = root;
- process.env.FAKE_ROOT = root;
+// ONE settings file for the whole process: persistKept reads settings through
+// getPaths(), which caches the first SETTINGS_FILE it sees, so a per-test file
+// would be gone by the second test. Disk floor off, so the per-item gate cannot
+// turn a run into a skip on a full disk.
+const SETTINGS_ROOT = await mkdtemp(path.join(tmpdir(), "persist-kept-settings-"));
+const SETTINGS_FILE = path.join(SETTINGS_ROOT, "settings.json");
+await writeFile(SETTINGS_FILE, JSON.stringify({ minFreeDiskGB: 0 }));
+process.env.SETTINGS_FILE = SETTINGS_FILE;
+process.env.TRANSCRIPTS_DIR = SETTINGS_ROOT;
- const ytdlp = path.join(root, "fake-ytdlp.cjs");
- await writeFile(ytdlp, FAKE_YTDLP);
- await chmod(ytdlp, 0o755);
- const meta = {
- id: ID,
- title: "A kept video",
- upload_date: "20260101",
- duration: 60,
- webpage_url: VIDEO,
- subtitles: {},
- automatic_captions: { en: [{ url: "https://example/cap" }] },
- };
- await writeFile(path.join(root, "meta.json"), JSON.stringify(meta));
+const TRANSCRIPT = '{"transcription":[{"text":"kept"}]}';
- const paths = {
- transcriptsDir: root,
- channelsDir: path.join(root, "channels"),
- savedVideosDir: path.join(root, "saved-videos"),
- ytdlpBin: ytdlp,
- ffmpegBin: path.join(root, "no-ffmpeg"),
- ffprobeBin: path.join(root, "no-ffprobe"),
- } as Paths;
- const videoDir = path.join(paths.channelsDir, "c", "data", ID);
- await mkdir(videoDir, { recursive: true });
- await writeFile(path.join(videoDir, "metadata.info.json"), JSON.stringify(meta));
- const transcript = '{"transcription":[{"text":"kept"}]}';
- await writeFile(path.join(videoDir, "transcript.json"), transcript);
+// A youtube-handling channel keeping its latest video, which has a transcript;
+// persistKept over it. `mediaFail`: the media pass dies part-way with a 403.
+async function runPersistKept(opts: { mediaFail?: boolean } = {}) {
+ const root = await mkdtemp(path.join(tmpdir(), "persist-kept-force-"));
+ process.env.FAKE_ROOT = root;
+ const ytdlp = path.join(root, "fake-ytdlp.cjs");
+ await writeFile(ytdlp, FAKE_YTDLP);
+ await chmod(ytdlp, 0o755);
+ if (opts.mediaFail) await writeFile(path.join(root, "media-fail"), "");
+ const meta = {
+ id: ID,
+ title: "A kept video",
+ upload_date: "20260101",
+ duration: 60,
+ webpage_url: VIDEO,
+ subtitles: {},
+ automatic_captions: { en: [{ url: "https://example/cap" }] },
+ };
+ await writeFile(path.join(root, "meta.json"), JSON.stringify(meta));
+
+ const paths = {
+ transcriptsDir: root,
+ channelsDir: path.join(root, "channels"),
+ savedVideosDir: path.join(root, "saved-videos"),
+ ytdlpBin: ytdlp,
+ ffmpegBin: path.join(root, "no-ffmpeg"),
+ ffprobeBin: path.join(root, "no-ffprobe"),
+ } as Paths;
+ const videoDir = path.join(paths.channelsDir, "c", "data", ID);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "metadata.info.json"), JSON.stringify(meta));
+ await writeFile(path.join(videoDir, "transcript.json"), TRANSCRIPT);
- let log = "";
- const result = await persistKept({
- paths,
- channelSlug: "c",
- channelConfig: {
- handling: "youtube",
- url: "https://www.youtube.com/@c/videos",
- keepLatest: 1,
- } as ChannelConfig,
- onLog: (s) => {
- log += s;
- },
- });
+ let log = "";
+ const result = await persistKept({
+ paths,
+ channelSlug: "c",
+ channelConfig: {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ keepLatest: 1,
+ } as ChannelConfig,
+ onLog: (s) => {
+ log += s;
+ },
+ });
+ const argvs = (await readFile(path.join(root, "argv.log"), "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => JSON.parse(l) as string[]);
+ return {
+ result,
+ log,
+ argvs,
+ videoDir,
+ cleanup: () => rm(root, { recursive: true, force: true }),
+ };
+}
- assert.equal(result.kept, 1);
- assert.equal(result.persisted, 1);
- assert.match(log, /forceMedia: downloading the source although a transcript is on disk/);
+test("persistKept forces the media pass: a kept youtube-handling video with a transcript is persisted, with no audio", async () => {
+ const r = await runPersistKept();
+ try {
+ assert.equal(r.result.kept, 1);
+ assert.equal(r.result.persisted, 1);
+ assert.match(r.log, /forceMedia: downloading the source although a transcript is on disk/);
// The media pass ran: three spawns, the last without --skip-download.
- const argvs = (await readFile(path.join(root, "argv.log"), "utf8"))
- .split("\n")
- .filter(Boolean)
- .map((l) => JSON.parse(l) as string[]);
- assert.equal(argvs.length, 3);
- assert.ok(!argvs[2].includes("--skip-download"));
+ assert.equal(r.argvs.length, 3);
+ assert.ok(!r.argvs[2].includes("--skip-download"));
// Persisted, and nothing else: the pointer is there, the container is in
// the store, no audio beside the transcript, the transcript untouched.
const pointer = JSON.parse(
- await readFile(path.join(videoDir, "saved-video.json"), "utf8"),
+ await readFile(path.join(r.videoDir, "saved-video.json"), "utf8"),
) as { dir: string; file: string; keepReason?: string };
assert.equal(pointer.keepReason, "override");
assert.equal(
await readFile(path.join(pointer.dir, pointer.file), "utf8"),
`container bytes for ${ID}`,
);
- const files = await readdir(videoDir);
+ const files = await readdir(r.videoDir);
assert.deepEqual(files.filter((f) => /^(audio|source-media)\./.test(f)), []);
- assert.equal(await readFile(path.join(videoDir, "transcript.json"), "utf8"), transcript);
+ assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT);
+ } finally {
+ await r.cleanup();
+ }
+});
+
+// A RETURNED DOWNLOAD IS NOT A PERSISTED SOURCE (release 11 slice O3, review
+// low 2). A forced media pass that fails over a transcript leaves the download
+// `ok`, so counting every return as `persisted` claimed a container that is
+// not there.
+test("persistKept counts a failed media pass as failed, not persisted", async () => {
+ const r = await runPersistKept({ mediaFail: true });
+ try {
+ assert.equal(r.result.kept, 1);
+ assert.equal(r.result.persisted, 0);
+ assert.equal(r.result.failed, 1);
+ assert.match(
+ r.log,
+ new RegExp(`${ID}: persist failed — ERROR: \\[download\\] Got error: HTTP Error 403: Forbidden`),
+ );
+ assert.match(r.log, /Persist kept: 0 persisted, 0 already saved, 1 failed/);
+ const files = await readdir(r.videoDir);
+ assert.ok(!files.includes("saved-video.json"));
+ assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT);
} finally {
- await rm(root, { recursive: true, force: true });
+ await r.cleanup();
}
});
