commit 54cb4f2c228c680e023c141aa4b4e972c3ffc089
parent eb677ac7cafc1d44165d44c41a26cc197d68484f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 02:26:22 -0400
Merge r11/runner-lows — release 11 slice O3: a maybe-missing video whose recheck failed stays maybe-missing on the published badge; a failed forced media pass keeps the subtitle pass's status and fails its fetch job; a bucket retry keeps its card and run log (the cookies-mode flake); a cut whose commit fails after the write says so and revalidates, and a half-way all names the earlier cut
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
23 files changed, 1183 insertions(+), 106 deletions(-)
diff --git a/common/controller/cutRelease.test.ts b/common/controller/cutRelease.test.ts
@@ -7,6 +7,7 @@ import { execa } from "execa";
import {
cutReleaseForWorkspace,
cutReleases,
+ describeCutFailure,
describeRelease,
resolveVersion,
todayISO,
@@ -363,12 +364,24 @@ test(
root,
}).finally(() => chmod(exportDir, 0o755));
assert.equal(outcome.ok, false);
- assert.equal(outcome.untouched, undefined);
+ // Something WAS written (release 11 slice O3; it used to be absent).
+ assert.equal(outcome.untouched, false);
const [first, second] = outcome.results;
assert.equal(first.ok && first.committed, true);
assert.equal(second.workspace, "export");
assert.match(!second.ok ? second.error : "", /^Could not write .*export\/CHANGELOG\.md: EACCES/);
+ assert.equal(!second.ok && second.written, undefined);
assert.deepEqual(outcome.notAttempted, []);
+ // The one sentence names the failure AND the cut before it (review L4:
+ // the route's `error` used to name only the export).
+ const editorSha = first.ok ? (first.commitSha ?? "").slice(0, 8) : "";
+ assert.match(
+ describeCutFailure(outcome, "all"),
+ new RegExp(
+ `^export: Could not write .*export/CHANGELOG\\.md: EACCES[^\\n]*\\. ` +
+ `Before it, editor was already cut \\(## \\[0\\.9\\.1\\] - ${DATE}, committed ${editorSha}\\)\\.$`,
+ ),
+ );
assert.deepEqual(await subjects(root), ["Release editor 0.9.1", "init"]);
assert.equal(await read(root, "export"), exportSource);
},
@@ -505,3 +518,146 @@ test("resolveVersion maps the keywords and keeps a literal", () => {
assert.equal(resolveVersion("next-minor", null), "0.1.0");
assert.equal(resolveVersion(" 2.0.0 ", "0.9.0"), "2.0.0");
});
+
+// A COMMIT THAT FAILS AFTER THE WRITE (release 11 slice O3, release 10 slice P
+// review L3). The changelog on disk has its new heading; the result used to be
+// a bare { ok: false } that nothing could tell from a refusal, so neither the
+// form nor the route revalidated the pages that render it. A pre-commit hook
+// that refuses stands in for any git failure past the guard.
+async function refuseCommits(root: string): Promise<void> {
+ const hooks = path.join(root, ".git", "hooks");
+ await mkdir(hooks, { recursive: true });
+ await writeFile(
+ path.join(hooks, "pre-commit"),
+ "#!/bin/sh\necho 'hook says no' >&2\nexit 1\n",
+ );
+ await chmod(path.join(hooks, "pre-commit"), 0o755);
+ // Local config beats a global core.hooksPath, which would skip .git/hooks.
+ await git(root, "config", "core.hooksPath", hooks);
+}
+
+test("a commit that fails after the write says the file was written (written: true, not untouched)", async () => {
+ const source = changelog({ pending: ["- x"], latest: "0.9.0" });
+ await withRepo({ editor: source }, async (root) => {
+ await refuseCommits(root);
+ const result = await cutReleaseForWorkspace({
+ workspace: "editor",
+ version: "next",
+ commit: true,
+ date: DATE,
+ root,
+ });
+ assert.equal(result.ok, false);
+ assert.equal(!result.ok && result.written, true);
+ assert.match(
+ !result.ok ? result.error : "",
+ /^Cut release 0\.9\.1, but the commit failed: [\s\S]*hook says no/,
+ );
+ assert.match(await read(root, "editor"), new RegExp(`## \\[0\\.9\\.1\\] - ${DATE}\\n- x`));
+ assert.deepEqual(await subjects(root), ["init"]);
+ });
+ // Through cutReleases, a single workspace: not untouched, and the sentence
+ // is the writer's.
+ await withRepo({ editor: source }, async (root) => {
+ await refuseCommits(root);
+ const outcome = await cutReleases({
+ workspace: "editor",
+ version: "next",
+ commit: true,
+ date: DATE,
+ root,
+ });
+ assert.equal(outcome.ok, false);
+ assert.equal(outcome.untouched, false);
+ assert.match(
+ describeCutFailure(outcome, "editor"),
+ /^Cut release 0\.9\.1, but the commit failed: [\s\S]*hook says no/,
+ );
+ });
+});
+
+test("all whose first commit fails after its write: stopped, not untouched, the export not tried", async () => {
+ const exportSource = changelog({ pending: ["- x"], latest: "0.9.0" });
+ await withRepo(
+ { editor: changelog({ pending: ["- e"], latest: "0.9.0" }), export: exportSource },
+ async (root) => {
+ await refuseCommits(root);
+ const outcome = await cutReleases({
+ workspace: "all",
+ version: "next",
+ commit: true,
+ date: DATE,
+ root,
+ });
+ assert.equal(outcome.ok, false);
+ assert.equal(outcome.untouched, false);
+ assert.deepEqual(outcome.notAttempted, ["export"]);
+ assert.equal(outcome.results.length, 1);
+ const [editor] = outcome.results;
+ assert.equal(editor.workspace, "editor");
+ assert.equal(!editor.ok && editor.written, true);
+ assert.match(
+ describeCutFailure(outcome, "all"),
+ /^editor: Cut release 0\.9\.1, but the commit failed: [\s\S]*hook says no/,
+ );
+ assert.match(await read(root, "editor"), new RegExp(`## \\[0\\.9\\.1\\] - ${DATE}`));
+ assert.equal(await read(root, "export"), exportSource);
+ assert.deepEqual(await subjects(root), ["init"]);
+ },
+ );
+});
+
+test("untouched is true for every refusal and false for a clean cut; a refusal's sentence is the writer's", async () => {
+ await withRepo({ editor: changelog({ pending: [], latest: "0.9.0" }) }, async (root) => {
+ const refused = await cutReleases({
+ workspace: "editor",
+ version: "next",
+ commit: false,
+ date: DATE,
+ root,
+ });
+ assert.equal(refused.untouched, true);
+ assert.equal(
+ describeCutFailure(refused, "editor"),
+ "Nothing pending to release under [Unreleased].",
+ );
+ });
+ await withRepo({ editor: changelog({ pending: ["- x"], latest: "0.9.0" }) }, async (root) => {
+ const cut = await cutReleases({
+ workspace: "editor",
+ version: "next",
+ commit: false,
+ date: DATE,
+ root,
+ });
+ assert.equal(cut.ok, true);
+ assert.equal(cut.untouched, false);
+ });
+});
+
+test(
+ "all whose FIRST write fails wrote nothing: untouched, the export not tried",
+ { skip: isRoot ? "root ignores directory permissions" : false },
+ async () => {
+ const editorSource = changelog({ pending: ["- e"], latest: "0.9.0" });
+ const exportSource = changelog({ pending: ["- x"], latest: "0.9.0" });
+ await withRepo({ editor: editorSource, export: exportSource }, async (root) => {
+ const editorDir = path.join(root, "editor");
+ await chmod(editorDir, 0o555);
+ const outcome = await cutReleases({
+ workspace: "all",
+ version: "next",
+ commit: false,
+ date: DATE,
+ root,
+ }).finally(() => chmod(editorDir, 0o755));
+ assert.equal(outcome.ok, false);
+ assert.equal(outcome.untouched, true);
+ assert.deepEqual(outcome.notAttempted, ["export"]);
+ assert.match(describeCutFailure(outcome, "all"), /^editor: Could not write .*: EACCES/);
+ assert.doesNotMatch(describeCutFailure(outcome, "all"), /Before it/);
+ assert.equal(await read(root, "editor"), editorSource);
+ assert.equal(await read(root, "export"), exportSource);
+ });
+ },
+);
diff --git a/common/controller/cutRelease.ts b/common/controller/cutRelease.ts
@@ -128,7 +128,16 @@ export type CutReleaseResult =
// HEAD after the release commit; only when `committed`.
commitSha?: string;
}
- | { ok: false; workspace: ReleaseWorkspace; error: string };
+ | {
+ ok: false;
+ workspace: ReleaseWorkspace;
+ error: string;
+ // THE FILE WAS WRITTEN; only the commit after it failed (release 11
+ // slice O3, release 10 review L3). The changelog on disk now carries the
+ // new heading, so a caller that revalidates pages on a cut must do so
+ // here too. Absent on every other failure: nothing was written.
+ written?: true;
+ };
export type CutReleaseOptions = {
workspace: ReleaseWorkspace;
@@ -237,7 +246,12 @@ async function applyCut(
`Release ${workspace} ${version}`,
);
if (!result.ok) {
- return fail(`Cut release ${version}, but the commit failed: ${result.error}`);
+ return {
+ ok: false,
+ workspace,
+ error: `Cut release ${version}, but the commit failed: ${result.error}`,
+ written: true,
+ };
}
const sha = await headSha(repo.root);
return {
@@ -284,10 +298,20 @@ export type CutReleasesOutcome = {
results: CutReleaseResult[];
// The workspaces a failure stopped before they were tried.
notAttempted: ReleaseWorkspace[];
- // `all` refused before writing anything: neither changelog was touched.
- untouched?: true;
+ // No changelog was written. True for every refusal, `all`'s preflight
+ // included; false once any file was written, a cut whose commit then failed
+ // (`written`) included. cutReleases ALWAYS sets it (release 11 slice O3, so
+ // a caller can tell a refusal from a cut stopped half-way — review L4); it
+ // is optional only so a hand-built outcome (the CLI's display tests) need
+ // not carry it, and there absent reads as false.
+ untouched?: boolean;
};
+// Whether any result of a run wrote its changelog.
+function nothingWritten(results: readonly CutReleaseResult[]): boolean {
+ return !results.some((r) => r.ok || r.written === true);
+}
+
/**
* Cut `editor`, `export`, or `all` (editor then export, with the SAME version:
* a keyword resolves against the HIGHER of the two latest headings, so neither
@@ -312,6 +336,7 @@ export async function cutReleases(
version: result.ok ? result.version : null,
results: [result],
notAttempted: [],
+ untouched: nothingWritten([result]),
};
}
const workspaces: ReleaseWorkspace[] = [...RELEASE_WORKSPACES];
@@ -365,16 +390,48 @@ export async function cutReleases(
for (const cut of plans) {
const result = await applyCut(repo, cut, opts.commit);
if (!result.ok) {
+ const results = [...done, result];
return {
ok: false,
version,
- results: [...done, result],
+ results,
notAttempted: workspaces.slice(done.length + 1),
+ untouched: nothingWritten(results),
};
}
done.push(result);
}
- return { ok: true, version, results: done, notAttempted: [] };
+ return { ok: true, version, results: done, notAttempted: [], untouched: false };
+}
+
+/**
+ * The one sentence for a cut that did not fully happen — the ops route's
+ * top-level `error` (release 11 slice O3, release 10 review L4). The failing
+ * workspace's refusal (prefixed with its name for `all`), then whatever was
+ * already cut before it: an `all` stopped half-way by I/O or git has cut and
+ * possibly committed the editor, and a caller that reads only `error` must not
+ * be told only about the export.
+ */
+export function describeCutFailure(
+ outcome: CutReleasesOutcome,
+ target: ReleaseTarget,
+): string {
+ const failure = outcome.results.find((r) => !r.ok);
+ if (!failure || failure.ok) return "cut failed";
+ const head = target === "all" ? `${failure.workspace}: ${failure.error}` : failure.error;
+ const done = outcome.results.flatMap((r) =>
+ r.ok
+ ? [
+ `${r.workspace} was already cut (${r.heading}, ${
+ r.committed
+ ? `committed${r.commitSha ? ` ${r.commitSha.slice(0, 8)}` : ""}`
+ : "not committed"
+ })`,
+ ]
+ : [],
+ );
+ if (done.length === 0) return head;
+ return `${head.replace(/\.$/, "")}. Before it, ${done.join("; ")}.`;
}
export type ReleaseSummary =
diff --git a/common/controller/maybeMissingBuild.test.ts b/common/controller/maybeMissingBuild.test.ts
@@ -0,0 +1,149 @@
+// Integration: the "Missing?" presence badge, through the REAL buildIndex, over
+// a temp corpus — the published half of resolveMaybeMissingState.
+//
+// The export draws the badge from a summary's `state` (export e2e
+// availability-state.spec.ts pins state → badge with mocked summaries). What
+// decides that `state` for a video that fell out of its channel's listing is
+// buildIndex's maybe-missing overlay, and that is what this file pins: the
+// site's summaries page, which is where the search results and the Availability
+// filter read `state` (the shared transcript pages carry cues, not state).
+//
+// Release 11 slice O3: a confirm probe made after the scan that FAILED (an
+// `error` availability) used to publish the video as available — no badge —
+// because stateFromAvailability folds `error` into `available`. It now stays
+// maybe_missing, like a video never probed.
+//
+// Run with: node_modules/.bin/tsx --test common/controller/maybeMissingBuild.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync, mkdirSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+// getPaths() is lazy and cached, and nothing above calls it at import time, so
+// pointing the whole path graph at a temp root here isolates this file's
+// process from the real corpus (the curatedTagsBuild.test.ts pattern).
+const ROOT = mkdtempSync(path.join(tmpdir(), "maybe-missing-build-"));
+process.env.TRANSCRIPTS_DIR = path.join(ROOT, "transcripts");
+process.env.EXPORT_PUBLIC_DIR = path.join(ROOT, "public");
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+
+const { getPaths } = await import("../lib/paths");
+const { buildIndex } = await import("./buildIndex");
+
+const paths = getPaths();
+const CHANNEL = "test-channel";
+const SITE = "testsite";
+
+const SCAN_AT = "2026-08-01T12:00:00.000Z";
+const BEFORE_SCAN = "2026-07-20T00:00:00.000Z";
+const AFTER_SCAN = "2026-08-02T00:00:00.000Z";
+
+// One video per outcome of the overlay. Only LISTED is in the channel's
+// listing; every other id is in maybe-missing.json.
+const LISTED = "vid-listed";
+const NEVER_PROBED = "vid-never-probed";
+const ERROR_AFTER = "vid-error-after";
+const PUBLIC_AFTER = "vid-public-after";
+const PUBLIC_BEFORE = "vid-public-before";
+const DELETED = "vid-deleted";
+
+function writeJson(file: string, value: unknown): void {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value, null, 2));
+}
+
+function videoDir(id: string): string {
+ return path.join(paths.channelsDir, CHANNEL, "data", id);
+}
+
+function seedCorpus(): void {
+ writeFileSync(paths.settingsFile, JSON.stringify({}));
+ writeJson(path.join(paths.channelsDir, CHANNEL, "config.json"), {
+ handling: "youtube",
+ name: "Test Channel",
+ url: "https://www.youtube.com/@example/videos",
+ });
+ const ids = [LISTED, NEVER_PROBED, ERROR_AFTER, PUBLIC_AFTER, PUBLIC_BEFORE, DELETED];
+ ids.forEach((id, i) => {
+ writeJson(path.join(videoDir(id), "metadata.info.json"), {
+ id,
+ title: `Video ${id}`,
+ channel: "Test Channel",
+ upload_date: `2026010${i + 1}`,
+ duration: 120,
+ description: "fixture",
+ webpage_url: `https://www.youtube.com/watch?v=${id}`,
+ extractor_key: "Youtube",
+ });
+ writeFileSync(
+ path.join(videoDir(id), "transcript.en.vtt"),
+ "WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nA line of transcript.\n",
+ );
+ });
+ const availability = (id: string, availability: string, checkedAt: string) =>
+ writeJson(path.join(videoDir(id), "availability.json"), { checkedAt, availability });
+ availability(ERROR_AFTER, "error", AFTER_SCAN);
+ availability(PUBLIC_AFTER, "public", AFTER_SCAN);
+ availability(PUBLIC_BEFORE, "public", BEFORE_SCAN);
+ availability(DELETED, "deleted", BEFORE_SCAN);
+ writeJson(path.join(paths.channelsDir, CHANNEL, "maybe-missing.json"), {
+ checkedAt: SCAN_AT,
+ freshPlaylistCount: 1,
+ ids: [NEVER_PROBED, ERROR_AFTER, PUBLIC_AFTER, PUBLIC_BEFORE, DELETED],
+ });
+ writeJson(path.join(paths.sitesDir, SITE, "site.json"), {
+ siteId: SITE,
+ siteTitle: "Test Site",
+ siteDescription: "fixture",
+ headerTitle: "Test Site",
+ homeTagline: "",
+ socialLinks: [],
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [{ slug: CHANNEL, groupId: "default" }],
+ });
+}
+
+type Published = { id: string; state?: string; isDeleted?: boolean };
+
+function readPage(file: string): Published[] {
+ return JSON.parse(readFileSync(file, "utf8")) as Published[];
+}
+
+const summariesPage = () =>
+ readPage(path.join(paths.exportSitesIndexDir, SITE, "summaries", "page-0000.json"));
+
+function stateOf(page: Published[], id: string): string {
+ const record = page.find((r) => r.id === id);
+ assert.ok(record, `${id} is published`);
+ // The build omits `state` for an available video (the badge's absence).
+ return record.state ?? "available";
+}
+
+test("the published state of each video that left the listing, as the badge reads it", async () => {
+ seedCorpus();
+ const log: string[] = [];
+ const res = await buildIndex({ paths, onLog: (s) => log.push(s) });
+ assert.equal(res.totalCount, 6);
+
+ const expected: Record<string, string> = {
+ [LISTED]: "available",
+ [NEVER_PROBED]: "maybe_missing",
+ // THE CASE: the confirm probe failed after the scan. No evidence either
+ // way, so the badge stays.
+ [ERROR_AFTER]: "maybe_missing",
+ [PUBLIC_AFTER]: "available",
+ [PUBLIC_BEFORE]: "maybe_missing",
+ [DELETED]: "deleted",
+ };
+ const page = summariesPage();
+ const got = Object.fromEntries(Object.keys(expected).map((id) => [id, stateOf(page, id)]));
+ assert.deepEqual(got, expected);
+ // The build's own count line names the three unconfirmed videos.
+ assert.ok(
+ log.some((l) => l.includes("Availability: 3 video(s) unconfirmed-missing from a channel listing.")),
+ log.filter((l) => l.startsWith("Availability")).join("\n") || "no Availability line",
+ );
+});
diff --git a/common/controller/persistKept.ts b/common/controller/persistKept.ts
@@ -7,7 +7,10 @@ import { resolveCookiePolicy } from "../lib/cookiePolicy";
import { isSavedVideo } from "../lib/savedVideo-server";
import { computeKeptVideoIds } from "./keptVideos";
import { findVideoSourceUrl } from "./undownloadedVideos";
-import { downloadOneManaged } from "../ytdlp/downloadOneManaged";
+import {
+ downloadOneManaged,
+ sourceFetchFailure,
+} from "../ytdlp/downloadOneManaged";
// Bulk "persist kept now" pass (Phase 5). Ensures every video currently in the
// channel's keep-latest window has its source container saved to the store, for
@@ -119,7 +122,7 @@ export async function persistKept({
}
log(` ${videoId}: re-fetching source container to persist…`);
try {
- await downloadOneManaged({
+ const record = await downloadOneManaged({
channelSlug,
channelConfig,
paths,
@@ -136,7 +139,16 @@ export async function persistKept({
// captions, which is every kept one (release 10 slice N).
forceMedia: true,
});
- result.persisted += 1;
+ // A download that RETURNED is not a persisted source (release 11 slice
+ // O3, review low 2): a failed forced media pass leaves the download `ok`
+ // (the transcript is fine) with only its n: 3 attempt failed.
+ const failure = sourceFetchFailure(record);
+ if (failure) {
+ result.failed += 1;
+ log(` ${videoId}: persist failed — ${failure}`);
+ } else {
+ result.persisted += 1;
+ }
} catch (e) {
result.failed += 1;
log(` ${videoId}: persist failed — ${(e as Error).message}`);
diff --git a/common/controller/persistKeptForceMedia.test.ts b/common/controller/persistKeptForceMedia.test.ts
@@ -47,6 +47,11 @@ if (has("--skip-download") && has("--write-auto-subs")) {
process.exit(0);
}
if ((arg("-o") || "").includes("source-media")) {
+ if (fs.existsSync(path.join(root, "media-fail"))) {
+ fs.writeFileSync(path.join(dir, "source-media.f137.mp4.part"), "half a container");
+ console.error("ERROR: [download] Got error: HTTP Error 403: Forbidden");
+ process.exit(1);
+ }
fs.writeFileSync(path.join(dir, "source-media.mp4"), "container bytes for " + id);
console.log("DLOM_ARCHIVE youtube " + id);
process.exit(0);
@@ -55,83 +60,123 @@ console.error("fake: unknown invocation " + argv.join(" "));
process.exit(2);
`;
-test("persistKept forces the media pass: a kept youtube-handling video with a transcript is persisted, with no audio", async () => {
- const root = await mkdtemp(path.join(tmpdir(), "persist-kept-force-"));
- try {
- // Disk floor off, so the per-item gate cannot turn this into a skip on a
- // full disk. Set before persistKept's first getSettings() → getPaths().
- const settingsFile = path.join(root, "settings.json");
- await writeFile(settingsFile, JSON.stringify({ minFreeDiskGB: 0 }));
- process.env.SETTINGS_FILE = settingsFile;
- process.env.TRANSCRIPTS_DIR = root;
- process.env.FAKE_ROOT = root;
+// ONE settings file for the whole process: persistKept reads settings through
+// getPaths(), which caches the first SETTINGS_FILE it sees, so a per-test file
+// would be gone by the second test. Disk floor off, so the per-item gate cannot
+// turn a run into a skip on a full disk.
+const SETTINGS_ROOT = await mkdtemp(path.join(tmpdir(), "persist-kept-settings-"));
+const SETTINGS_FILE = path.join(SETTINGS_ROOT, "settings.json");
+await writeFile(SETTINGS_FILE, JSON.stringify({ minFreeDiskGB: 0 }));
+process.env.SETTINGS_FILE = SETTINGS_FILE;
+process.env.TRANSCRIPTS_DIR = SETTINGS_ROOT;
- const ytdlp = path.join(root, "fake-ytdlp.cjs");
- await writeFile(ytdlp, FAKE_YTDLP);
- await chmod(ytdlp, 0o755);
- const meta = {
- id: ID,
- title: "A kept video",
- upload_date: "20260101",
- duration: 60,
- webpage_url: VIDEO,
- subtitles: {},
- automatic_captions: { en: [{ url: "https://example/cap" }] },
- };
- await writeFile(path.join(root, "meta.json"), JSON.stringify(meta));
+const TRANSCRIPT = '{"transcription":[{"text":"kept"}]}';
- const paths = {
- transcriptsDir: root,
- channelsDir: path.join(root, "channels"),
- savedVideosDir: path.join(root, "saved-videos"),
- ytdlpBin: ytdlp,
- ffmpegBin: path.join(root, "no-ffmpeg"),
- ffprobeBin: path.join(root, "no-ffprobe"),
- } as Paths;
- const videoDir = path.join(paths.channelsDir, "c", "data", ID);
- await mkdir(videoDir, { recursive: true });
- await writeFile(path.join(videoDir, "metadata.info.json"), JSON.stringify(meta));
- const transcript = '{"transcription":[{"text":"kept"}]}';
- await writeFile(path.join(videoDir, "transcript.json"), transcript);
+// A youtube-handling channel keeping its latest video, which has a transcript;
+// persistKept over it. `mediaFail`: the media pass dies part-way with a 403.
+async function runPersistKept(opts: { mediaFail?: boolean } = {}) {
+ const root = await mkdtemp(path.join(tmpdir(), "persist-kept-force-"));
+ process.env.FAKE_ROOT = root;
+ const ytdlp = path.join(root, "fake-ytdlp.cjs");
+ await writeFile(ytdlp, FAKE_YTDLP);
+ await chmod(ytdlp, 0o755);
+ if (opts.mediaFail) await writeFile(path.join(root, "media-fail"), "");
+ const meta = {
+ id: ID,
+ title: "A kept video",
+ upload_date: "20260101",
+ duration: 60,
+ webpage_url: VIDEO,
+ subtitles: {},
+ automatic_captions: { en: [{ url: "https://example/cap" }] },
+ };
+ await writeFile(path.join(root, "meta.json"), JSON.stringify(meta));
+
+ const paths = {
+ transcriptsDir: root,
+ channelsDir: path.join(root, "channels"),
+ savedVideosDir: path.join(root, "saved-videos"),
+ ytdlpBin: ytdlp,
+ ffmpegBin: path.join(root, "no-ffmpeg"),
+ ffprobeBin: path.join(root, "no-ffprobe"),
+ } as Paths;
+ const videoDir = path.join(paths.channelsDir, "c", "data", ID);
+ await mkdir(videoDir, { recursive: true });
+ await writeFile(path.join(videoDir, "metadata.info.json"), JSON.stringify(meta));
+ await writeFile(path.join(videoDir, "transcript.json"), TRANSCRIPT);
- let log = "";
- const result = await persistKept({
- paths,
- channelSlug: "c",
- channelConfig: {
- handling: "youtube",
- url: "https://www.youtube.com/@c/videos",
- keepLatest: 1,
- } as ChannelConfig,
- onLog: (s) => {
- log += s;
- },
- });
+ let log = "";
+ const result = await persistKept({
+ paths,
+ channelSlug: "c",
+ channelConfig: {
+ handling: "youtube",
+ url: "https://www.youtube.com/@c/videos",
+ keepLatest: 1,
+ } as ChannelConfig,
+ onLog: (s) => {
+ log += s;
+ },
+ });
+ const argvs = (await readFile(path.join(root, "argv.log"), "utf8"))
+ .split("\n")
+ .filter(Boolean)
+ .map((l) => JSON.parse(l) as string[]);
+ return {
+ result,
+ log,
+ argvs,
+ videoDir,
+ cleanup: () => rm(root, { recursive: true, force: true }),
+ };
+}
- assert.equal(result.kept, 1);
- assert.equal(result.persisted, 1);
- assert.match(log, /forceMedia: downloading the source although a transcript is on disk/);
+test("persistKept forces the media pass: a kept youtube-handling video with a transcript is persisted, with no audio", async () => {
+ const r = await runPersistKept();
+ try {
+ assert.equal(r.result.kept, 1);
+ assert.equal(r.result.persisted, 1);
+ assert.match(r.log, /forceMedia: downloading the source although a transcript is on disk/);
// The media pass ran: three spawns, the last without --skip-download.
- const argvs = (await readFile(path.join(root, "argv.log"), "utf8"))
- .split("\n")
- .filter(Boolean)
- .map((l) => JSON.parse(l) as string[]);
- assert.equal(argvs.length, 3);
- assert.ok(!argvs[2].includes("--skip-download"));
+ assert.equal(r.argvs.length, 3);
+ assert.ok(!r.argvs[2].includes("--skip-download"));
// Persisted, and nothing else: the pointer is there, the container is in
// the store, no audio beside the transcript, the transcript untouched.
const pointer = JSON.parse(
- await readFile(path.join(videoDir, "saved-video.json"), "utf8"),
+ await readFile(path.join(r.videoDir, "saved-video.json"), "utf8"),
) as { dir: string; file: string; keepReason?: string };
assert.equal(pointer.keepReason, "override");
assert.equal(
await readFile(path.join(pointer.dir, pointer.file), "utf8"),
`container bytes for ${ID}`,
);
- const files = await readdir(videoDir);
+ const files = await readdir(r.videoDir);
assert.deepEqual(files.filter((f) => /^(audio|source-media)\./.test(f)), []);
- assert.equal(await readFile(path.join(videoDir, "transcript.json"), "utf8"), transcript);
+ assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT);
+ } finally {
+ await r.cleanup();
+ }
+});
+
+// A RETURNED DOWNLOAD IS NOT A PERSISTED SOURCE (release 11 slice O3, review
+// low 2). A forced media pass that fails over a transcript leaves the download
+// `ok`, so counting every return as `persisted` claimed a container that is
+// not there.
+test("persistKept counts a failed media pass as failed, not persisted", async () => {
+ const r = await runPersistKept({ mediaFail: true });
+ try {
+ assert.equal(r.result.kept, 1);
+ assert.equal(r.result.persisted, 0);
+ assert.equal(r.result.failed, 1);
+ assert.match(
+ r.log,
+ new RegExp(`${ID}: persist failed — ERROR: \\[download\\] Got error: HTTP Error 403: Forbidden`),
+ );
+ assert.match(r.log, /Persist kept: 0 persisted, 0 already saved, 1 failed/);
+ const files = await readdir(r.videoDir);
+ assert.ok(!files.includes("saved-video.json"));
+ assert.equal(await readFile(path.join(r.videoDir, "transcript.json"), "utf8"), TRANSCRIPT);
} finally {
- await rm(root, { recursive: true, force: true });
+ await r.cleanup();
}
});
diff --git a/common/lib/availability-server.ts b/common/lib/availability-server.ts
@@ -140,8 +140,19 @@ export async function resolveEffectiveAvailability(
// anyway (a truncated or oddly paginated playlist fetch) is resolved, and
// must stop being flagged.
//
-// Anything else — never probed, or last probed before the scan — is
-// `maybe_missing`: we know it left the listing, not yet why.
+// Anything else — never probed, last probed before the scan, or probed after
+// it with an `error` — is `maybe_missing`: we know it left the listing, not
+// yet why.
+//
+// AN `error` PROBE ANSWERS NOTHING, however recent (release 11 slice O3). It is
+// a 403, a network failure, YouTube's bot check or the one rate-limited probe of
+// a blocked run: the probe never saw the video. `stateFromAvailability` folds
+// it into `available` because it is no evidence the video is GONE — and by the
+// same token it is no evidence the video is THERE. Read as a confirmation, it
+// cleared the "Missing?" badge of exactly the videos whose confirm probe failed.
+// (`needs_auth` is different: an age gate is the video answering.) Display
+// only — this feeds the published presence state via buildIndex; the clean gate
+// reads resolveEffectiveAvailability, which this does not touch.
export async function resolveMaybeMissingState(
videoDir: string,
scannedAtMs: number,
@@ -150,6 +161,7 @@ export async function resolveMaybeMissingState(
if (!record) return "maybe_missing";
const confirmed = stateFromAvailability(record.availability);
if (confirmed !== "available") return confirmed;
+ if (record.availability === "error") return "maybe_missing";
const checkedAtMs = Date.parse(record.checkedAt);
if (Number.isFinite(checkedAtMs) && checkedAtMs >= scannedAtMs) {
return "available";
diff --git a/common/lib/videoState.test.ts b/common/lib/videoState.test.ts
@@ -99,8 +99,34 @@ test("in the set + public, last probed BEFORE the scan → maybe_missing", async
});
});
-// needs_auth/error are not evidence the video left its channel, so they behave
-// exactly like public: still absent from the listing, still unconfirmed.
+// A probe made AFTER the scan that FAILED saw nothing: a 403, a network error,
+// the bot check, the one rate-limited probe of a blocked run. It must not read
+// as a confirmation the way a public re-probe does (release 11 slice O3; it
+// used to, through stateFromAvailability's `error` → available fold).
+test("in the set + error, probed AFTER the scan → still maybe_missing", async () => {
+ await withVideoDir(async (videoDir) => {
+ await seedAvailability(videoDir, "error", AFTER_SCAN);
+ assert.equal(
+ await resolveMaybeMissingState(videoDir, SCAN_AT_MS),
+ "maybe_missing",
+ );
+ });
+});
+
+// An age gate is the video answering: a needs_auth re-probe after the scan
+// confirms it is there, exactly like public.
+test("in the set + needs_auth, probed AFTER the scan → available", async () => {
+ await withVideoDir(async (videoDir) => {
+ await seedAvailability(videoDir, "needs_auth", AFTER_SCAN);
+ assert.equal(
+ await resolveMaybeMissingState(videoDir, SCAN_AT_MS),
+ "available",
+ );
+ });
+});
+
+// needs_auth/error are not evidence the video left its channel, so before the
+// scan they behave exactly like public: still absent, still unconfirmed.
for (const availability of ["needs_auth", "error"] as const) {
test(`in the set + ${availability} before the scan → maybe_missing`, async () => {
await withVideoDir(async (videoDir) => {
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -1,5 +1,5 @@
import path from "node:path";
-import { appendFile, mkdir, readdir, readFile, rm } from "node:fs/promises";
+import { appendFile, mkdir, readdir, readFile, rm, stat } from "node:fs/promises";
import { createWriteStream, type Dirent, type WriteStream } from "node:fs";
import { execa } from "execa";
import {
@@ -38,6 +38,7 @@ import {
type DownloadOutcomeStatus,
} from "../lib/downloadOutcome";
import { writeDownloadOutcome } from "../lib/downloadOutcome-server";
+import { formatBytes } from "../lib/format";
import { recordAvailability } from "../lib/availability-server";
import { withMetadataHistory } from "../lib/metadataHistory-server";
import {
@@ -517,6 +518,46 @@ function attemptSucceeded(exitCode: number | null): boolean {
return exitCode === 0 || exitCode === 101;
}
+// DID A DOWNLOAD THAT WAS ASKED FOR THE SOURCE GET IT? (release 11 slice O3.)
+// For "Persist source video" and the whole-recording fetch: null when nothing
+// failed, else yt-dlp's reason. Two ways to fail: the download failed outright
+// (the status says so), or the forced media pass over a transcript failed —
+// which leaves the status as the subtitle pass made it, so only the last
+// attempt (the n: 3 media pass) says so. Reading the status alone ended those
+// jobs `done` with no file.
+//
+// WHEN THE LAST ATTEMPT IS THE MEDIA PASS, IT ALONE ANSWERS (review low 1). That
+// pass IS the source fetch; a later step's status is not about the source. A
+// fallback whose container came down and was persisted, and whose inline
+// whisper then failed, ends the download `failed` — and reading the status
+// first called that source "not downloaded".
+export function sourceFetchFailure(record: DownloadOutcomeRecord): string | null {
+ const last = record.attempts.at(-1);
+ const reason = (fallback: string) => last?.error?.trim() || fallback;
+ if (last?.kind === "no-subs-fallback") {
+ return attemptSucceeded(last.ytdlpExitCode)
+ ? null
+ : reason(`yt-dlp exited ${last.ytdlpExitCode ?? "without a code"}`);
+ }
+ if (record.status === "failed" || record.status === "failed-corrupt-source") {
+ return reason(`the download ended ${record.status}`);
+ }
+ return null;
+}
+
+// yt-dlp's partial source container(s) in a video dir — `source-media.<fmt>.part`
+// and its fragments — with their sizes, for the failed-media log line.
+async function sourceMediaPartials(videoDir: string): Promise<string[]> {
+ const entries = await readdir(videoDir).catch(() => [] as string[]);
+ const out: string[] = [];
+ for (const name of entries.sort()) {
+ if (!name.startsWith("source-media.") || !/\.part(?:-Frag\d+)?$/.test(name)) continue;
+ const st = await stat(path.join(videoDir, name)).catch(() => null);
+ out.push(st ? `${name} (${formatBytes(st.size)})` : name);
+ }
+ return out;
+}
+
async function hasAnyTranscriptOnDisk(videoDir: string): Promise<boolean> {
const entries = await readdir(videoDir).catch(() => [] as string[]);
return entries.some((e) => {
@@ -1467,6 +1508,26 @@ async function runManagedDownload(
// fellBackToTranscribe flag distinguishes from a normal "ok"
// so the UI and downstream jobs can tell what happened.
}
+ } else if (keepTranscript) {
+ // THE MEDIA FAILED, THE DOWNLOAD DID NOT (release 11 slice O3). A
+ // persist-only pass over a transcript is an extra the caller asked for,
+ // not the download: the subtitle pass succeeded and the transcript is
+ // on disk, so the status stays the subtitle pass's and only the failed
+ // media attempt is recorded (n: 3 above, with its error). Marking the
+ // whole download failed put "Download failed" on a video whose
+ // transcript is fine. A caller that asked for the source reads the
+ // attempt through sourceFetchFailure. yt-dlp's partial container is
+ // left where it is — a retry resumes it — and named here, because
+ // nothing else surfaces a source-media partial.
+ const partials = await sourceMediaPartials(videoDir);
+ opts.onLog(
+ `forceMedia: the source download failed (yt-dlp exit ${fallbackRes.exitCode}); ` +
+ `the transcript on disk is untouched and the download stays ${status}.` +
+ (partials.length
+ ? ` Left for a retry to resume: ${partials.join(", ")}.`
+ : "") +
+ "\n",
+ );
} else {
status = "failed";
lastSucceeded = false;
diff --git a/common/ytdlp/forceMedia.test.ts b/common/ytdlp/forceMedia.test.ts
@@ -13,7 +13,12 @@ import { tmpdir } from "node:os";
import path from "node:path";
import type { Paths } from "../lib/paths";
import type { ChannelConfig } from "../lib/channelConfig";
-import { downloadOneManaged, type ManagedDownloadOpts } from "./downloadOneManaged";
+import {
+ downloadOneManaged,
+ sourceFetchFailure,
+ type ManagedDownloadOpts,
+} from "./downloadOneManaged";
+import type { DownloadOutcomeRecord } from "../lib/downloadOutcome";
// Run with:
// pnpm --filter yt-dlp-transcript-common exec tsx --test ytdlp/forceMedia.test.ts
@@ -56,6 +61,12 @@ if (has("--skip-download") && has("--write-auto-subs")) {
process.exit(0);
}
if ((arg("-o") || "").includes("source-media")) {
+ if (fs.existsSync(path.join(root, "media-fail"))) {
+ // A media pass that dies mid-download: the partial yt-dlp leaves, then a 403.
+ fs.writeFileSync(path.join(dir, "source-media.f137.mp4.part"), "half a container");
+ console.error("ERROR: [download] Got error: HTTP Error 403: Forbidden");
+ process.exit(1);
+ }
fs.writeFileSync(path.join(dir, "source-media.mp4"), "container bytes for " + id);
console.log("DLOM_ARCHIVE youtube " + id);
process.exit(0);
@@ -77,6 +88,8 @@ type Run = {
videoDir: string;
storeDir: string;
files: string[];
+ // The channel's download archive after the run.
+ archive: string;
};
async function runWith(opts: {
@@ -88,6 +101,9 @@ async function runWith(opts: {
priorMeta?: Record<string, unknown>;
config?: Partial<ChannelConfig>;
managed?: Partial<ManagedDownloadOpts>;
+ // The media pass (the source-media output) fails with a 403 after writing a
+ // partial.
+ mediaFail?: boolean;
}): Promise<Run & { cleanup: () => Promise<void> }> {
const root = await mkdtemp(path.join(tmpdir(), "force-media-"));
const ytdlp = path.join(root, "fake-ytdlp.cjs");
@@ -97,6 +113,7 @@ async function runWith(opts: {
await chmod(ytdlp, 0o755);
await chmod(ffmpeg, 0o755);
await writeFile(path.join(root, "meta.json"), JSON.stringify(opts.meta));
+ if (opts.mediaFail) await writeFile(path.join(root, "media-fail"), "");
process.env.FAKE_ROOT = root;
const paths = {
transcriptsDir: root,
@@ -144,6 +161,7 @@ async function runWith(opts: {
videoDir,
storeDir: path.join(paths.savedVideosDir, "c", ID),
files: (await readdir(videoDir)).sort(),
+ archive: await readFile(path.join(paths.channelsDir, "c", "archive"), "utf8").catch(() => ""),
cleanup: () => rm(root, { recursive: true, force: true }),
};
}
@@ -300,3 +318,96 @@ test("the prefetch's rewrite of metadata.info.json lands in metadata.history.jso
await r.cleanup();
}
});
+
+// A FAILED MEDIA PASS OVER A TRANSCRIPT IS NOT A FAILED DOWNLOAD (release 11
+// slice O3; release 10 slice N's review L2). The subtitle pass succeeded and the
+// transcript is on disk; the extra the caller asked for — the source — failed.
+// The status stays the subtitle pass's (the video page said "Download failed"
+// on a video whose transcript is fine), the failed attempt is recorded, and the
+// caller that asked for the source learns it through sourceFetchFailure.
+test("forceMedia over a transcript whose media pass fails: the download stays ok, only the media attempt failed", async () => {
+ const r = await runWith({ meta: META, transcript: TRANSCRIPT, managed: FORCED, mediaFail: true });
+ try {
+ assert.equal(r.argvs.length, 3);
+ assert.equal(r.record.status, "ok");
+ assert.equal(r.record.failureClass, undefined);
+ assert.deepEqual(
+ r.record.attempts.map((a) => [a.kind, a.ytdlpExitCode]),
+ [
+ ["metadata-prefetch", 0],
+ ["primary", 0],
+ ["no-subs-fallback", 1],
+ ],
+ );
+ assert.match(r.record.attempts[2].error ?? "", /HTTP Error 403: Forbidden/);
+ assert.match(sourceFetchFailure(r.record) ?? "", /HTTP Error 403: Forbidden/);
+ assert.match(
+ r.log,
+ /forceMedia: the source download failed \(yt-dlp exit 1\); the transcript on disk is untouched and the download stays ok\. Left for a retry to resume: source-media\.f137\.mp4\.part \(16 B\)\.\n/,
+ );
+ // The transcript is untouched and still counts: the video stays in the
+ // archive as the subtitle pass left it.
+ assert.equal(await readFile(path.join(r.videoDir, TRANSCRIPT.name), "utf8"), TRANSCRIPT.body);
+ assert.match(r.archive, new RegExp(`youtube ${ID}`));
+ // Nothing persisted; the partial is left for a retry to resume.
+ assert.ok(!r.files.includes("saved-video.json"));
+ assert.ok(r.files.includes("source-media.f137.mp4.part"), r.files.join(","));
+ // The sidecar on disk says the same as the returned record.
+ const onDisk = JSON.parse(
+ await readFile(path.join(r.videoDir, "download-outcome.json"), "utf8"),
+ ) as DownloadOutcomeRecord;
+ assert.equal(onDisk.status, "ok");
+ } finally {
+ await r.cleanup();
+ }
+});
+
+test("sourceFetchFailure: a failed download, a failed forced pass, and the clean cases", () => {
+ const base = { videoId: ID, startedAt: "t0", finishedAt: "t1" };
+ const attempt = (kind: DownloadOutcomeRecord["attempts"][number]["kind"], code: number | null, error?: string) => ({
+ n: 1,
+ kind,
+ handling: "youtube" as const,
+ usedCookies: false,
+ ytdlpExitCode: code,
+ ...(error ? { error } : {}),
+ });
+ assert.equal(
+ sourceFetchFailure({ ...base, status: "failed", attempts: [attempt("primary", 1, "ERROR: gone")] }),
+ "ERROR: gone",
+ );
+ assert.equal(
+ sourceFetchFailure({ ...base, status: "failed-corrupt-source", attempts: [] }),
+ "the download ended failed-corrupt-source",
+ );
+ assert.equal(
+ sourceFetchFailure({
+ ...base,
+ status: "ok",
+ attempts: [attempt("primary", 0), attempt("no-subs-fallback", 1)],
+ }),
+ "yt-dlp exited 1",
+ );
+ assert.equal(
+ sourceFetchFailure({
+ ...base,
+ status: "ok",
+ attempts: [attempt("primary", 0), attempt("no-subs-fallback", 0)],
+ }),
+ null,
+ );
+ assert.equal(
+ sourceFetchFailure({ ...base, status: "ok-with-cookies", attempts: [attempt("auth-retry", 0)] }),
+ null,
+ );
+ // The fallback fetched and persisted the source, then inline whisper failed
+ // and ended the download `failed`: the SOURCE is there (review low 1).
+ assert.equal(
+ sourceFetchFailure({
+ ...base,
+ status: "failed",
+ attempts: [attempt("primary", 0), attempt("no-subs-fallback", 0)],
+ }),
+ null,
+ );
+});
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -3,6 +3,9 @@
## [Unreleased]
- **The homepage can be built and deployed from `/sites`.** Under a new **Homepage** section, after Hub, there is **Build homepage** (tick **Deploy after build** to ship it in the same job, only if the build succeeds) and **Deploy homepage**, which ships the build already in `homepage/out`. A **Preview branch** box beside them sends either deploy to a Cloudflare Pages preview of the `archilyzer` project instead of production, and shows the preview's address as you type; a name Cloudflare would refuse or rewrite, or `main`, greys the deploy buttons out and says why. A line under the buttons says what a deploy would ship: when `homepage/out` was built (or that it holds no build yet), and where it goes, with the live URL. Deploy homepage with nothing built is refused before any job starts. The homepage reads the search index as it stands, so run **Build index** first when its numbers should move. The jobs run the same code as `archilyzer build homepage` / `deploy homepage`, and show on `/jobs` as `build-homepage`, `deploy-homepage` and `build-deploy-homepage`. The Hub section no longer describes the homepage.
- **`pnpm ops build-homepage` and `pnpm ops deploy-homepage`.** The same two jobs over HTTP: `build-homepage` takes `{"deploy": true}` to deploy after a successful build, and both take `{"preview": "<branch>"}` for a preview (`build-homepage` only with `deploy`). `deploy-homepage` answers with the preview's address, and refuses a bad preview name or a missing build before any job starts.
+- **"Persist source video" or a whole-recording fetch that cannot get the source no longer marks the video's download failed.** When YouTube's subtitles came down but the source video did not, the video page said "Download failed" over a transcript that is fine. The download now keeps the subtitle pass's result and records only the failed media attempt, with yt-dlp's reason. The run itself now ends failed with that reason; it used to end done with no file, so `fetch_clip` could only say the job "finished but named no file". A partial source file is left for a retry to resume, and the run's log names it.
+- **A bucket's retry keeps its log when it empties the bucket.** On a channel's Download stage, "Download with cookies", the partial-download resume and the missing-transcript retry could lose their run log part-way: the video they fetched left the bucket, the page refreshed, and the card disappeared with the log in it. The card now stays, with its log and its button disabled, until the page is reloaded. The Transcribe stage's "Fetch audio" button does the same. (A Diagnostics card still disappears, log and all, when its retry empties it.)
+- **A release cut whose commit fails still refreshes the pages.** When the changelog's new heading was written but the commit after it failed, the Cut release form and `pnpm ops cut-release` answered as if nothing had happened and no page showed the new heading until a reload. Both now refresh the changelog pages, and `pnpm ops cut-release` says the file was written. Every refused cut's answer says whether anything was written (`untouched`), and a cut of both changelogs that stopped half-way names the one already cut as well as the failure.
## [0.9.4] - 2026-09-28
- **On the Dark ground the sidebar's Archilyzer mark has a thin outline.** Its slate tile now has a 1-pixel ring just outside it, following its rounded corners, in the colour of the mark's unlit lines, so the tile's edge shows against the dark page. Light and Sepia are unchanged, and so is the favicon.
diff --git a/editor/app/api/ops/cut-release/route.ts b/editor/app/api/ops/cut-release/route.ts
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
import {
cutReleases,
dateProblem,
+ describeCutFailure,
RELEASE_TARGETS,
versionSpecProblem,
} from "yt-dlp-transcript-common/controller/cutRelease";
@@ -30,9 +31,12 @@ export const dynamic = "force-dynamic";
// commit each — or neither: every check runs before either file is written.
//
// 200 { ok: true, version, results } when every changelog was cut. A refusal
-// is a 400 { ok: false, error, version, results, notAttempted, untouched? }.
-// `untouched: true` means `all` was refused before writing anything; without
-// it, `results` says what WAS done before a write or a commit failed.
+// is a 400 { ok: false, error, version, results, notAttempted, untouched }.
+// `untouched` is always there (release 11 slice O3): true means no changelog
+// was written — every refusal, `all`'s preflight included; false means a cut
+// stopped half-way (a write or a commit failed after something was written),
+// and then `error` names what was already cut as well as what failed, and a
+// result with `written: true` is a changelog cut whose commit failed.
export async function POST(request: Request) {
return ops(request, ["workspace", "version", "commit", "date"], async (body) => {
const workspace = oneOf(body, "workspace", RELEASE_TARGETS);
@@ -45,7 +49,9 @@ export async function POST(request: Request) {
const commit = optBool(body, "commit") ?? false;
const outcome = await cutReleases({ workspace, version, commit, date });
- if (outcome.results.some((r) => r.ok)) revalidateAfterReleaseCut();
+ // Whenever a changelog changed on disk — a cut whose commit then failed
+ // included (review L3: it used to revalidate nothing).
+ if (outcome.untouched !== true) revalidateAfterReleaseCut();
if (outcome.ok) {
return NextResponse.json({
ok: true,
@@ -53,17 +59,11 @@ export async function POST(request: Request) {
results: outcome.results,
});
}
- const failure = outcome.results.find((r) => !r.ok);
- const reason = failure && !failure.ok ? failure.error : "cut failed";
- return opsFail(
- workspace === "all" && failure ? `${failure.workspace}: ${reason}` : reason,
- 400,
- {
- version: outcome.version,
- results: outcome.results,
- notAttempted: outcome.notAttempted,
- ...(outcome.untouched ? { untouched: true } : {}),
- },
- );
+ return opsFail(describeCutFailure(outcome, workspace), 400, {
+ version: outcome.version,
+ results: outcome.results,
+ notAttempted: outcome.notAttempted,
+ untouched: outcome.untouched === true,
+ });
});
}
diff --git a/editor/app/channels/[slug]/components/RetryBucketControl.tsx b/editor/app/channels/[slug]/components/RetryBucketControl.tsx
@@ -27,6 +27,9 @@ type Props = {
replaceAutoSubs?: boolean;
// Overrides the default "Retry (n)" button text.
buttonLabel?: string;
+ // Told when a run STARTS here, so a parent card that hides itself on an
+ // empty bucket can stay mounted for this run's log (see below).
+ onRun?: () => void;
};
export function RetryBucketControl({
@@ -39,14 +42,24 @@ export function RetryBucketControl({
forceCookies,
replaceAutoSubs,
buttonLabel,
+ onRun,
}: Props) {
const [queue, setQueue] = useState(defaultQueueKey);
const [handlingOverride, setHandlingOverride] = useState(
replaceAutoSubs ? "transcribe" : "",
);
const [abortOnError, setAbortOnError] = useState(false);
+ // A RETRY EMPTIES ITS OWN BUCKET, and the log must survive that (release 11
+ // slice O3; plans/FACTS.md, "A run log lives in the panel's React state").
+ // Every video the run fetches refreshes the snapshot, and the page's refresh
+ // then hands this control an empty `ids` while the run is still streaming —
+ // returning null there unmounted the StreamActionLog and took the log with
+ // it (cookies-mode.spec's intermittent "Retry needs cookies output" not
+ // found). Once a run has started here the control stays, with its button
+ // disabled while the bucket is empty; a reload drops it as before.
+ const [ranHere, setRanHere] = useState(false);
- if (ids.length === 0) return null;
+ if (ids.length === 0 && !ranHere) return null;
return (
<div
@@ -54,8 +67,10 @@ export function RetryBucketControl({
aria-label={`retry ${actionLabel} bucket`}
>
<StreamActionLog
- trigger={() =>
- retryBucketAction(
+ trigger={() => {
+ setRanHere(true);
+ onRun?.();
+ return retryBucketAction(
slug,
ids,
queue,
@@ -64,8 +79,9 @@ export function RetryBucketControl({
bucketKey,
forceCookies,
replaceAutoSubs,
- )
- }
+ );
+ }}
+ disabled={ids.length === 0}
cancelAction={cancelJobAction}
buttonLabel={
buttonLabel
diff --git a/editor/app/channels/[slug]/components/stages/DownloadStage.tsx b/editor/app/channels/[slug]/components/stages/DownloadStage.tsx
@@ -352,7 +352,10 @@ function NoTranscriptList({
defaultQueueKey: string;
existingQueues: string[];
}) {
- if (ids.length === 0) return null;
+ // Stays mounted once its retry has run: the run empties this bucket, and
+ // the log must outlive that (RetryBucketControl says why).
+ const [ranHere, setRanHere] = useState(false);
+ if (ids.length === 0 && !ranHere) return null;
return (
<div className="flex flex-col gap-2 rounded border border-border p-3">
<div>
@@ -378,6 +381,7 @@ function NoTranscriptList({
defaultQueueKey={defaultQueueKey}
existingQueues={existingQueues}
bucketKey="noTranscript"
+ onRun={() => setRanHere(true)}
/>
</div>
);
@@ -394,7 +398,10 @@ function PartialDownloadsList({
defaultQueueKey: string;
existingQueues: string[];
}) {
- if (ids.length === 0) return null;
+ // Stays mounted once its retry has run: the run empties this bucket, and
+ // the log must outlive that (RetryBucketControl says why).
+ const [ranHere, setRanHere] = useState(false);
+ if (ids.length === 0 && !ranHere) return null;
return (
<div className="flex flex-col gap-2 rounded border border-border p-3">
<div>
@@ -423,6 +430,7 @@ function PartialDownloadsList({
defaultQueueKey={defaultQueueKey}
existingQueues={existingQueues}
bucketKey="partialDownloads"
+ onRun={() => setRanHere(true)}
/>
</div>
);
@@ -439,7 +447,10 @@ function NeedsCookiesList({
defaultQueueKey: string;
existingQueues: string[];
}) {
- if (ids.length === 0) return null;
+ // Stays mounted once its retry has run: the run empties this bucket, and
+ // the log must outlive that (RetryBucketControl says why).
+ const [ranHere, setRanHere] = useState(false);
+ if (ids.length === 0 && !ranHere) return null;
return (
<div className="flex flex-col gap-2 rounded border border-border p-3">
<div>
@@ -470,6 +481,7 @@ function NeedsCookiesList({
defaultQueueKey={defaultQueueKey}
existingQueues={existingQueues}
bucketKey="needsCookies"
+ onRun={() => setRanHere(true)}
forceCookies
/>
</div>
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -70,7 +70,10 @@ import {
} from "yt-dlp-transcript-common/jobs/downloadBackoff";
import { getSettings } from "yt-dlp-transcript-common/lib/settings";
import { resolveCookiePolicy } from "yt-dlp-transcript-common/lib/cookiePolicy";
-import { downloadOneManaged } from "yt-dlp-transcript-common/ytdlp/downloadOneManaged";
+import {
+ downloadOneManaged,
+ sourceFetchFailure,
+} from "yt-dlp-transcript-common/ytdlp/downloadOneManaged";
import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp";
import {
runManagedFunction,
@@ -291,7 +294,7 @@ async function archiveSourceVideo(
onLog(`${requesterLine(persistOrigin)}\n`);
}
onLog(`Re-downloading ${videoId} to archive its source video…\n`);
- await downloadOneManaged({
+ const record = await downloadOneManaged({
channelSlug: slug,
channelConfig: r.config,
paths,
@@ -314,6 +317,16 @@ async function archiveSourceVideo(
`/channels/${slug}/videos/${videoId}`,
`/channels/${slug}`,
]);
+ // THE JOB IS THE SOURCE, SO ITS STATUS IS THE SOURCE'S (release 11
+ // slice O3). A forced media pass that fails over a transcript leaves
+ // the download `ok` — the transcript is fine — so without this the job
+ // ended `done` with no file, and the whole-recording fetch's caller was
+ // told only that it "finished but named no file". Failing the job puts
+ // yt-dlp's line in the log tail the poll returns.
+ const failure = sourceFetchFailure(record);
+ if (failure) {
+ throw new Error(`The source video was not downloaded: ${failure}`);
+ }
} finally {
task.end();
}
diff --git a/editor/app/sites/lib/cutReleaseAction.ts b/editor/app/sites/lib/cutReleaseAction.ts
@@ -38,7 +38,13 @@ export async function cutReleaseAction(
version,
commit: shouldCommit,
});
- if (!result.ok) return { ok: false, error: result.error };
+ if (!result.ok) {
+ // The heading was written and only the commit after it failed (release 11
+ // slice O3, release 10 review L3): the pages that render this changelog
+ // are stale all the same. The error already says the file changed.
+ if (result.written) revalidateAfterReleaseCut();
+ return { ok: false, error: result.error };
+ }
revalidateAfterReleaseCut();
return { ok: true, version: result.version, committed: result.committed };
}
diff --git a/editor/e2e/cookies-mode.spec.ts b/editor/e2e/cookies-mode.spec.ts
@@ -301,6 +301,23 @@ test("defer: excluded from batches, surfaced in Needs cookies, downloads via the
// The bucket empties once the snapshot regenerates.
await waitForNeedsCookiesBucket(0);
+ // …and the page refreshes onto the EMPTY bucket with the card still up and
+ // its run's log in it (release 11 slice O3). The card used to return null on
+ // an empty bucket, so whichever refresh landed first — the per-video
+ // snapshot's, mid-run — unmounted the log: the intermittent "Retry needs
+ // cookies output: element(s) not found" above. Waiting for the empty list
+ // makes that race certain rather than likely: it renders only after the
+ // refresh, inside the card that must have survived it.
+ await expect(page.getByLabel("needs cookies empty")).toBeVisible({
+ timeout: 20_000,
+ });
+ await expect(retryLog).toContainText(
+ "Managed download complete: 1 succeeded, 0 failed",
+ );
+ await expect(
+ bucket.getByRole("button", { name: /^Download with cookies \(0\)$/ }),
+ ).toBeDisabled();
+ // A fresh page has nothing to show for an empty bucket.
await page.reload();
await expect(page.getByLabel("retry needs cookies bucket")).toHaveCount(0);
});
diff --git a/editor/e2e/fetch-window.spec.ts b/editor/e2e/fetch-window.spec.ts
@@ -9,7 +9,7 @@
// Same token as /api/worker/* (the test server runs with
// WORKER_TOKEN=test-worker-token; see package.json dev:test).
-import { readdir, readFile, stat } from "node:fs/promises";
+import { readdir, readFile, stat, writeFile } from "node:fs/promises";
import { test, expect, type APIRequestContext } from "@playwright/test";
import {
generateReport,
@@ -455,3 +455,57 @@ test("a full-source fetch on a youtube-handling video that already has a transcr
to: `Synthetic ${HAS_TRANSCRIPT}`,
});
});
+
+// …AND WHEN THAT DOWNLOAD FAILS, THE JOB SAYS SO (release 11 slice O3). The
+// forced media pass failing over a transcript is not a failed download — the
+// subtitle pass succeeded, the transcript is fine, so the outcome stays `ok` —
+// but the job exists for the source, so the job fails, with yt-dlp's line in
+// the log tail the poll returns. Before, it ended `done` with no file, and the
+// MCP could only say the job "finished but named no file".
+test("a full-source fetch whose media download fails ends failed with yt-dlp's line; the download stays ok", async ({
+ request,
+}) => {
+ test.setTimeout(90_000);
+ const HAS_TRANSCRIPT = "fake00000001";
+ await writeFile(resolvePath(rel(".fake-ytdlp-media-fail")), "");
+ const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
+ headers: AUTH,
+ data: {
+ channelSlug: SLUG,
+ videoId: HAS_TRANSCRIPT,
+ full: true,
+ requestedBy: "mcp",
+ },
+ });
+ expect(post.status(), await post.text()).toBe(202);
+ const { jobId } = (await post.json()) as { jobId: string };
+
+ const finished = await pollJob(request, jobId);
+ expect(finished.status, JSON.stringify(finished)).toBe("failed");
+ expect(finished.file).toBeUndefined();
+ expect(String(finished.error)).toContain(
+ "The source video was not downloaded: ERROR: [download] Got error: HTTP Error 403: Forbidden",
+ );
+ expect(String(finished.error)).toContain(
+ "forceMedia: the source download failed (yt-dlp exit 1); the transcript on disk is untouched and the download stays ok.",
+ );
+
+ const outcome = await readJson<{
+ status: string;
+ attempts: Array<{ kind: string; ytdlpExitCode: number | null }>;
+ }>(rel(`data/${HAS_TRANSCRIPT}/download-outcome.json`));
+ expect(outcome.status).toBe("ok");
+ expect(outcome.attempts.map((a) => [a.kind, a.ytdlpExitCode])).toEqual([
+ ["metadata-prefetch", 0],
+ ["primary", 0],
+ ["no-subs-fallback", 1],
+ ]);
+ const files = await readdir(resolvePath(rel(`data/${HAS_TRANSCRIPT}`)));
+ expect(files).toContain("transcript.en.vtt");
+ expect(files).not.toContain("saved-video.json");
+ // The partial is left for a retry to resume, and the log named it.
+ expect(files).toContain("source-media.f137.mp4.part");
+ expect(String(finished.error)).toContain(
+ "Left for a retry to resume: source-media.f137.mp4.part (",
+ );
+});
diff --git a/editor/e2e/fixtures/bin/fake-ytdlp.mjs b/editor/e2e/fixtures/bin/fake-ytdlp.mjs
@@ -939,6 +939,19 @@ async function main() {
await writeMetadata(videoDir, id, urlSentinels(url));
}
process.stdout.write(`[download] Fetching ${id}\n`);
+ // `.fake-ytdlp-media-fail` in the channel root (the fake's cwd): the media
+ // download dies part-way — the partial yt-dlp leaves behind, then a 403 —
+ // for the forced-media failure specs (release 11 slice O3).
+ if (existsSync(".fake-ytdlp-media-fail")) {
+ await writeFile(
+ path.join(videoDir, "source-media.f137.mp4.part"),
+ `fake-ytdlp partial source container for ${id}\n`,
+ );
+ process.stderr.write(
+ `ERROR: [download] Got error: HTTP Error 403: Forbidden\n`,
+ );
+ process.exit(1);
+ }
await writeFile(
path.join(videoDir, "source-media.mp4"),
`fake-ytdlp synthesised source container for ${id}\n`,
diff --git a/editor/e2e/ops-cut-release.spec.ts b/editor/e2e/ops-cut-release.spec.ts
@@ -116,6 +116,12 @@ test("cut-release cuts the editor changelog and answers with the heading it wrot
error: "Could not find a `## [Unreleased]` heading to cut from.",
},
]);
+ // Every 400 says whether anything was written (release 11 slice O3): a
+ // refusal wrote nothing. A cut stopped half-way — a write or a commit that
+ // failed after a file changed — answers `untouched: false` with the earlier
+ // cut named in `error`; the controller's tests drive those (a git failure is
+ // not something this server's fixture changelogs can produce).
+ expect(again.body.untouched).toBe(true);
});
test("all cuts both changelogs with ONE version, resolved against the higher latest", async ({
diff --git a/editor/e2e/persist-youtube-handling.spec.ts b/editor/e2e/persist-youtube-handling.spec.ts
@@ -173,3 +173,48 @@ test("a download whose metadata changed upstream appends to metadata.history.jso
await expect(entries).toContainText("view_count");
await expect(entries).toContainText("100 → 150");
});
+
+// A FAILED MEDIA PASS IS NOT A FAILED DOWNLOAD (release 11 slice O3). With the
+// fake's media download dying part-way, the run fails — the operator asked for
+// the source and did not get it — but the download outcome stays the subtitle
+// pass's `ok`, so the video page no longer says "Download failed" over a
+// transcript that is fine.
+test("Persist source video whose media download fails: the run fails with yt-dlp's line, the video is not marked Download failed", async ({
+ page,
+}) => {
+ test.setTimeout(120_000);
+ await resetData("youtube-with-playlist");
+ await writeFile(resolvePath(rel(".fake-ytdlp-media-fail")), "");
+
+ await page.goto(`/channels/${SLUG}/videos/${HAS_TRANSCRIPT}`);
+ await page.getByLabel("Source video stage summary").click();
+ await page.getByRole("button", { name: "Persist source video", exact: true }).click();
+ const log = page.getByLabel(`Persist source video for ${HAS_TRANSCRIPT} output`);
+ await expect(log).toContainText(
+ "forceMedia: the source download failed (yt-dlp exit 1); the transcript on disk is untouched and the download stays ok.",
+ { timeout: 30_000 },
+ );
+ await expect(log).toContainText(
+ "[error] The source video was not downloaded: ERROR: [download] Got error: HTTP Error 403: Forbidden",
+ { timeout: 30_000 },
+ );
+
+ const outcome = await outcomeOf(HAS_TRANSCRIPT);
+ expect(outcome?.status).toBe("ok");
+ expect(outcome?.attempts.map((a) => [a.kind, a.ytdlpExitCode])).toEqual([
+ ["metadata-prefetch", 0],
+ ["primary", 0],
+ ["no-subs-fallback", 1],
+ ]);
+ // The transcript is there (the subtitle pass re-fetched it, as on any
+ // re-download); the failed media pass wrote none.
+ const files = await readdir(resolvePath(rel(`data/${HAS_TRANSCRIPT}`)));
+ expect(files).toContain("transcript.en.vtt");
+ expect(files).not.toContain("saved-video.json");
+
+ // The page, fresh: no failed-download badge, and nothing persisted.
+ await page.reload();
+ await expect(page.getByLabel("download outcome")).toHaveCount(0);
+ await page.getByLabel("Source video stage summary").click();
+ await expect(page.getByLabel(`unpersist source video ${HAS_TRANSCRIPT}`)).toHaveCount(0);
+});
diff --git a/export/CHANGELOG.md b/export/CHANGELOG.md
@@ -1,5 +1,8 @@
# Changelog
+## [Unreleased]
+- **A video whose recheck failed shows as possibly missing rather than available.** When a video drops out of its channel's listing it is marked "Missing?" until a recheck says why. A recheck that could not reach the video — a blocked request or a network error — used to clear the mark as if the video had been found. It now leaves "Missing?" in place until a recheck actually reaches the video. Needs a rebuild and deploy of every export site.
+
## [0.9.4] - 2026-09-28
- **On the Dark ground the mark's tile has a thin outline.** The header mark's ink tile is the colour of the Dark page, so only its lines showed. On Dark the tile now has a 1-pixel ring just outside it, following its rounded corners, in the colour of the mark's unlit lines. The small Archilyzer mark by the footer credit and the hub's mark get the same ring in their own slate's unlit colour. Light and Sepia are unchanged, and so are the icons. Needs a rebuild and deploy of every export site and the hub.
diff --git a/plans/FACTS.md b/plans/FACTS.md
@@ -3696,6 +3696,21 @@ box that outlives the warning stops answering `getByLabel("incomplete transcript
(`cards/PerFileTranscodeRow.tsx:39` since 3b) is NOT an instance: its `return null` depends on the file's extension against a static format
list, which a transcode cannot change, and the rows are keyed by file name.
+**On the channel page, the bucket cards** (release 11 slice O3, `86449b88`): `RetryBucketControl`
+and the Download stage's three bucket cards (`NoTranscriptList`, `PartialDownloadsList`,
+`NeedsCookiesList`) returned null on an empty bucket, and a retry EMPTIES its own bucket. What hands
+the card its emptied bucket is not only the end-of-run refresh: `recordTaskDone` →
+`requestSnapshotOnFinish` (`common/jobs/streamCommand.ts`) regenerates the snapshot after EACH video
+a batch finishes, the pulse moves, and `AutoRefresh` refreshes the page MID-RUN. Now the shape
+above: `ranHere` set in the trigger (and `onRun` to tell the card), null only while
+`ids.length === 0 && !ranHere`, `disabled={ids.length === 0}` (`RetryBucketControl.tsx:60-84`,
+`DownloadStage.tsx:357,404,454`). `cookies-mode.spec.ts:241` waits for the refreshed EMPTY card
+(`needs cookies empty`), which makes the old race certain rather than likely. The Transcribe
+stage's Fetch audio control is covered too (its section stays while `autoSubsCount > 0`). **NOT
+Diagnostics:** its two grids drop an emptied bucket's card before the control renders
+(`DiagnosticsStage.tsx:156` `populated`, `:694` `listed`), so a Diagnostics retry that empties its
+bucket still loses its log — open.
+
The persist case needed a fixture: `editor/e2e/fixtures/bin/fake-ytdlp.mjs:680-725` (at
`12d1778`) grows an app-extraction branch — the LAST branch checked, matched on the media
output template naming `source-media` (only `plan.extractionMode === "app"` emits that), which
@@ -6780,8 +6795,46 @@ S4, as shipped"; `release-10.md` "Slice L2 / L1, as shipped". Every anchor below
`transcript.json` is untouched. It rewrites `metadata.info.json` (every managed download's
prefetch does); since slice N each rewrite that changes the bytes appends to
`metadata.history.json` (`common/lib/metadataHistory.ts`). The window path never writes metadata.
+ **A forced pass that FAILS over a transcript** (release 11 slice O3) keeps the download's status
+ (the subtitle pass's `ok`) and records only the failed n: 3 attempt; `archiveSourceVideo` reads
+ it through `sourceFetchFailure` and throws, so the job ends `failed` with yt-dlp's line in the
+ poll's `error` tail — it used to end `done` with no file ("finished but named no file").
Neither path dedupes a running job: a repeated request while one runs queues a second fetch, which
is why every text says "resume with job".
- The editor must already HAVE the channel (404 `Channel "<slug>" not found` otherwise); a
public-only setup gets the "no editor configured" error naming both env vars, and the README's
`yt-dlp --download-sections` is the no-editor fallback only.
+
+### O3 — runner lows (release 11, 2026-09-28)
+
+- **An `error` probe never confirms a maybe-missing video** (`resolveMaybeMissingState`,
+ `common/lib/availability-server.ts:156-170`, `:164`). `stateFromAvailability` folds `error` (a
+ 403, a network failure, the bot check, a blocked run's one rate-limited probe) into `available`
+ because it is no evidence the video is gone; it is no evidence it is there either, so a probe
+ after the scan that came back `error` now leaves it `maybe_missing`. `needs_auth` (an age gate)
+ after the scan still confirms. The ONE caller is `buildIndex.ts:1180` (the published `state` on a
+ site's summaries pages, and the `videoState` sub-DB behind the status chart); the clean gate reads
+ `resolveEffectiveAvailability`, which the fix does not touch. `maybeMissingBuild.test.ts` pins it
+ through the real `buildIndex`. The shared transcript pages carry no `state`; the summaries (and
+ subs) pages do.
+- **The five real videos it moved (read-only count, 2026-09-28)** were all Rumble probes that got
+ `HTTP Error 410: Gone` on 2026-09-25 and were stored as `error`, just before `6d5cdbc3` taught
+ `parseUnavailableFromStderr` that 410 is `deleted`: `rekietalaw-rumble/v7e07us` and
+ `the-quartering-rumble/v4vriou v4x5o1l v4yqask v501kfc`. A "Full-check unexpected" on those two
+ channels would settle them `deleted`.
+- **`sourceFetchFailure(record)`** (`common/ytdlp/downloadOneManaged.ts`) is how a caller that
+ asked for the SOURCE tells whether it got it. When the last attempt is the `no-subs-fallback`
+ media pass, ITS exit code alone answers (a fallback that persisted the container and whose inline
+ whisper then failed ends `failed`, and the source is there — review low 1); otherwise the status
+ (`failed`, `failed-corrupt-source`). Read by `archiveSourceVideo` (throws) and `persistKept`
+ (counts it `failed`, not `persisted`). A failed
+ `keepTranscript` pass (`:1503`) leaves the status alone and logs `forceMedia: the source download
+ failed (yt-dlp exit N); the transcript on disk is untouched and the download stays <status>.`,
+ naming any `source-media.*.part` it leaves (left on purpose: a retry resumes it).
+- **`cutReleases` always sets `untouched`** (`common/controller/cutRelease.ts`, `nothingWritten`
+ `:311`): true = no changelog was written (every refusal); false once any was, including a result
+ with **`written: true`** — a cut whose file was written and whose commit then failed (`:253`).
+ The type keeps it optional only for hand-built outcomes (`common/bin/_cli.test.ts`).
+ `describeCutFailure(outcome, target)` (`:415`) is the ops route's 400 `error`: the failure,
+ `<workspace>: `-prefixed for `all`, then `Before it, editor was already cut (## [x] - date,
+ committed <sha8>).` The route revalidates whenever `untouched !== true`.
diff --git a/plans/release-11.md b/plans/release-11.md
@@ -187,6 +187,213 @@ waits on the :3001 restart. After it:
2. **Deploy homepage**, or `pnpm ops deploy-homepage --wait`. The job's log ends with
`[deployed] https://<hash>.archilyzer.pages.dev`.
+### Slice O3, as shipped — runner lows (2026-09-28)
+
+Branch `r11/runner-lows` off `main` `2162db92`, worktree `/home/user/Projects/r11-runner-lows`, one
+Opus implementer, beside O4 and O6. Four lows left by release 10: one from L2 ("New low"), one from
+slice N's review (L2) and its knock-ons, the `cookies-mode.spec:241` intermittent N's full suite
+found, and slice P's review lows L3 + L4. No settings, site or channel key; nothing on disk moves.
+`CutReleaseResult.written` and an always-set `CutReleasesOutcome.untouched` are additive.
+
+**1 — a failed confirm probe no longer clears "Missing?"** (`common/lib/availability-server.ts`).
+- `resolveMaybeMissingState` decides the published state of a video that fell out of its
+ channel's listing. A probe made after the scan that came back `public` clears the flag; so did
+ one that came back `error`, through `stateFromAvailability`'s `error` → `available` fold. An
+ `error` probe (a 403, a network failure, the bot check, a blocked run's one rate-limited probe)
+ never saw the video, so it is no evidence it is there. It now leaves the video `maybe_missing`,
+ like one never probed. `needs_auth` after the scan still confirms: an age gate is the video
+ answering.
+- **Display only, and the clean gate is untouched — proof.** `grep -rn resolveMaybeMissingState`
+ over the whole repo (not `node_modules` or `.next`) finds one caller, `buildIndex.ts:1180` (the maybe-missing overlay →
+ `stateByIndexKey` → a site's summaries pages and the `videoState` sub-DB the status chart
+ reads), plus `videoState.test.ts`. The diff to `availability-server.ts` is the one line and its
+ comment; `resolveEffectiveAvailability`, which `verifyBeforeClean` and the MtimeRecord read, is
+ not in it.
+- **The export check** is `common/controller/maybeMissingBuild.test.ts` (new), which runs the REAL
+ `buildIndex` over a temp corpus: one video per outcome of the overlay, and the `state` each gets
+ on the site's `summaries/page-0000.json` — the field the export's badge and Availability filter
+ read. `export/e2e/availability-state.spec.ts` was NOT extended: it fulfils the summaries with
+ mocked `state`s, so a new case there could not see the fix; it was re-run (9/9) as the contract
+ check that `state: "maybe_missing"` still draws "Missing?". The shared transcript pages carry no
+ `state` (found while writing the test).
+- **The real count, read-only** (`o3-flip-count.ts` in the job scratch dir, `o3-flip-count.log`: reads
+ `maybe-missing.json`, `availability.json`, `download-outcome.json`, `metadata.info.json`'s
+ existence and `sites/*/site.json` through the build's own loaders; opens no LMDB, writes nothing):
+ 1,827 maybe-missing ids in 31 channels; **5 videos across 2 channels move from available to
+ maybe-missing at the next index build** — `rekietalaw-rumble/v7e07us` and
+ `the-quartering-rumble/v4vriou v4x5o1l v4yqask v501kfc`. Per site: **jeralyzer 4** ("Missing?"
+ 212 → 216), **rekietalyzer 1** (0 → 1), anilyzer, bonnellyzer, hasanalyzer and jasolyzer 0. All
+ five are Rumble probes that got `HTTP Error 410: Gone` on 2026-09-25 and were stored as `error`,
+ hours before `6d5cdbc3` taught the classifier that 410 is `deleted`; they are really deleted, and
+ "Missing?" is closer than "available". A "Full-check unexpected" on those two channels would
+ settle them as Deleted (the operator's call; nothing was run).
+
+**2 — a failed forced download keeps the subtitle pass's status** (`common/ytdlp/downloadOneManaged.ts`).
+- The attempt-3 `else` set `status = "failed"` and `lastSucceeded = false` for a `keepTranscript`
+ pass too, so a "Persist source video", `fetch_clip` `full: true` or Persist kept now whose MEDIA
+ failed put "Download failed" on a video whose transcript is fine. A `keepTranscript` failure now
+ keeps the status (the subtitle pass's `ok` / `ok-with-cookies`) and `lastSucceeded`, so the
+ archive line the subtitle pass printed is appended as on the success path; the failed n: 3
+ attempt is recorded with its error (no `failureClass`: the download did not fail). One log line:
+ `forceMedia: the source download failed (yt-dlp exit N); the transcript on disk is untouched and
+ the download stays ok. Left for a retry to resume: source-media.f137.mp4.part (<size>).`
+- **Knock-on 1, fixed: the job reports it.** `sourceFetchFailure(record)` (exported) says whether a
+ download asked for the source got it: the status for an outright failure, the last attempt for a
+ failed forced pass. `archiveSourceVideo` (`videoActions.ts`, both "Persist source video" and
+ `fetchFullSourceAction`) throws `The source video was not downloaded: <yt-dlp's line>` after its
+ revalidate, so the job ends `failed` and the poll's `error` tail carries the line; the MCP renders
+ "Editor job … failed" with it instead of "finished but named no file". A download that failed
+ outright ends the job `failed` too (before: `done`).
+- **Knock-on 2, left on purpose: the `.part`.** yt-dlp resumes a partial container on a retry, and
+ deleting one that is several GB makes a retry after a network blip start over — more requests
+ against the source, which the pacing rules want fewer of. Nothing else surfaces a
+ `source-media.*.part` (the Partial downloads bucket matches `audio.<ext>.part` only), so the log
+ line names it and its size instead.
+
+**3 — the `cookies-mode.spec:241` intermittent** (`DownloadStage.tsx`, `RetryBucketControl.tsx`).
+- The mechanism is wider than the end-of-run refresh: `recordTaskDone` → `requestSnapshotOnFinish`
+ regenerates the snapshot after EACH video a batch finishes, the pulse moves and `AutoRefresh`
+ refreshes the page while the run still streams. The refreshed `NeedsCookiesList` got an empty
+ bucket and returned null, and so did the `RetryBucketControl` in it — two unmounts, either
+ enough to lose the log.
+- The FACTS shape, in both: `ranHere` set inside the trigger (`RetryBucketControl` also calls a new
+ optional `onRun` so its card knows), null only while `ids.length === 0 && !ranHere`, and
+ `disabled={ids.length === 0}` so the surviving panel is not a second Run button. Applied to all
+ three bucket cards on the Download stage (`NoTranscriptList`, `PartialDownloadsList`,
+ `NeedsCookiesList`, same hazard) and to `RetryBucketControl` itself. Beyond the Download stage
+ it reaches one more control: the Transcribe stage's **Fetch audio** (its section stays mounted
+ while `autoSubsCount > 0`, and a fetch moves videos from "Needs audio" to "Ready to transcribe").
+ It does NOT reach Diagnostics: both of its grids drop an emptied bucket's whole card
+ (`DiagnosticsStage.tsx:156` `populated`, `:694` `listed`), control and log with it (corrected in
+ review; see Review fixes). A reload drops an empty card as before (the reload check at the end of
+ the same test still passes); labels and test ids unchanged.
+- **Made deterministic.** The spec now waits for the refreshed, EMPTY card (`needs cookies empty`,
+ which renders only inside a card that survived the refresh) and asserts the log's last line and a
+ disabled `Download with cookies (0)` in it. On the old code this fails every time, at that line
+ (below).
+
+**4 — cut-release review lows L3 + L4** (`common/controller/cutRelease.ts`, `api/ops/cut-release`).
+- **L3:** `applyCut`'s commit failure returns `{ok: false, workspace, error, written: true}` — the
+ file on disk has its new heading. The route revalidates whenever anything was written.
+- **L4:** `cutReleases` sets `untouched` on EVERY outcome (true = no changelog written, every refusal
+ included; false once one was, a `written` failure included); the type keeps it optional only so
+ the CLI's hand-built display tests compile, and absent reads as false. `describeCutFailure`
+ builds the route's 400 `error`: the failure (prefixed `<workspace>: ` for `all`), then `Before
+ it, editor was already cut (## [0.9.5] - 2026-09-28, committed 1a2b3c4d).` Every 400 from the
+ writer carries `untouched`. An `all` whose FIRST write fails wrote nothing and is `untouched: true`, so the CLI
+ prints "not cut — all cuts both or neither, and nothing was written" for the export — true.
+ The input refusals before the writer (a bad workspace, version, date or key) are the ops door's
+ 400s and carry no `untouched`, as before.
+- The CLI (`common/bin/release.ts`) is unchanged: its per-line output already named the earlier cut
+ and says "failed — Cut release X, but the commit failed" for L3.
+- **The `/sites` form** (`editor/app/sites/lib/cutReleaseAction.ts`) revalidates on `written: true`
+ too. It is under `editor/app/sites/**`, which slice O4 owned tonight, so it landed after O4's
+ merge (Review fixes).
+
+| sha | what |
+|---|---|
+| `877403db` | 1: `resolveMaybeMissingState` — `error` after the scan stays `maybe_missing`; `videoState.test.ts` +2, `maybeMissingBuild.test.ts` (new, 1) |
+| `8c72265c` | 2: the `keepTranscript` failure keeps the status; `sourceFetchFailure`; `archiveSourceVideo` fails the job; `forceMedia.test.ts` +2; the fake's `.fake-ytdlp-media-fail` knob; `fetch-window.spec.ts` +1, `persist-youtube-handling.spec.ts` +1 |
+| `86449b88` | 3: `ranHere` / `onRun` / disabled in `RetryBucketControl` and the three Download-stage bucket cards; `cookies-mode.spec.ts:241` made deterministic |
+| `edec55ca` | 4: `written`, always-set `untouched`, `describeCutFailure`, the route; `cutRelease.test.ts` +4 (1 changed); `ops-cut-release.spec.ts` pins `untouched` |
+| _this_ | `plans:` this record, FACTS (three amendments, one new section), the `[Unreleased]` bullets |
+
+**Gates**, from the worktree root; logs `o3-*.log` in `$T`.
+- **tsc** (`pnpm -r --no-bail --workspace-concurrency=1 exec tsc --noEmit`) clean before each code
+ commit: `o3-tsc1` (item 1), `o3-tsc3` (items 2 + 3, item 4 stashed), `o3-tsc4` (item 4).
+- **common 2,047/2,047** (release 10's 2,038 + 9: videoState +2, maybeMissingBuild +1, forceMedia
+ +2, cutRelease +4); **editor unit 85/85**; **`test:scripts` 174 + 1 skip of 175**; **mcp
+ 269/269** (`o3-units1.log`, on `edec55ca`).
+- **Editor build** `pnpm --filter editor exec next build` ok, 55 s (`o3-build1.log`; the route table
+ lists `ƒ /api/ops/cut-release`). **Export build not run**: nothing under `export/` changed but
+ its CHANGELOG.
+- **e2e** (queued, detached, `export/public` linked per path from the primary with `sw.js` a plain
+ copy, no dangling links; the lock was free each time):
+
+ | run | specs | passed | failed | time |
+ |---|---|---|---|---|
+ | A | `cookies-mode maybe-missing persist-youtube-handling fetch-window saved-videos cut-release ops-cut-release retry-bucket partial-downloads-bucket auto-subs-replace bulk-actions no-subs-fallback storage-locations` (`o3-e2e-specs.txt`: the prompt's seven, the four that drive a retry-bucket control, and the two that drive the source-media branch) | **61** | **0** | 3.9 min |
+ | B | `cookies-mode.spec.ts --repeat-each=3` | **18** | **0** | 1.8 min |
+ | C | EXPORT `availability-state.spec.ts` | **9** | **0** | 30 s |
+
+ `archives-off.spec.ts` draws no state badge, so it was not added. The primary's
+ `export/public/sw.js` was not written (md5 `355d0d21…`, mtime 00:51:32, before and after).
+- **Numbers tool:** `o3-flip-count.ts` (item 1), above.
+
+**They bite.**
+- Unit, item 1 (`o3-bite1.log`): with the `error` line reverted, `maybeMissingBuild.test.ts` fails
+ and `videoState.test.ts`'s "error, probed AFTER the scan" fails; 15 others pass.
+- Unit, items 2 + 4 (`o3-bite2.log`): with the `keepTranscript` branch removed, forceMedia's new
+ behaviour test fails (the `sourceFetchFailure` table test is of a new function); with `written`
+ and the always-set `untouched` reverted, 5 of 20 cutRelease tests fail (the four new and the
+ changed half-way test).
+- **e2e** (`o3-e2e-bite.log`): the six source files at their `2162db92` versions, the new specs and
+ fake kept; `fetch-window persist-youtube-handling cookies-mode ops-cut-release`: **16 passed, 4
+ failed, 2.5 min**, and the four are exactly the new and changed tests —
+ `cookies-mode.spec.ts:241` at `getByLabel("needs cookies empty")` (the card is gone),
+ `fetch-window.spec.ts:465` (`done`, not `failed`), `ops-cut-release.spec.ts:73`
+ (`untouched` undefined), `persist-youtube-handling.spec.ts:182` (no failure line; the old run
+ marked the download failed). Restored with `git checkout HEAD --`.
+
+**Found and left.**
+- ~~`persistKept` still counts a returned download as persisted~~: done in the review fixes.
+- **L3/L4 have no HTTP e2e for the half-way cases.** The fixture server's changelogs are gitignored
+ copies, so a commit there fails only after the dirty-tree guard has run git in the WORKTREE, and a
+ failed write needs a read-only directory the server's path cannot be pointed at. The controller's
+ tests drive both in temp repos (a refusing pre-commit hook; a `0555` directory); the spec pins
+ `untouched: true` on a refusal.
+- **A Diagnostics card after its run.** A Diagnostics retry that empties its bucket still loses
+ its card and its log: `DiagnosticsStage.tsx:156` / `:694` filter emptied buckets out before the
+ control renders. The follow-up is to lift a ran-bucket set into both sections and keep those
+ cards in `populated` / `listed`.
+- **The five 410 videos** are published as available today and will read "Missing?" after the next
+ build; a Full-check unexpected would make them Deleted.
+
+**Review fixes** (review SHIP AFTER FIXES, `o3-review.md` in the job scratch dir: one should-fix, wording only; three
+lows; one nit; the three questions ruled — keep `archiveSourceVideo`'s failure broad, staying
+mounted is fine as a rule, keep `untouched` optional tonight).
+- **Merge.** `export/.next/dev` removed first (run C's export dev server had left stale types there,
+ so `tsc` failed in `export/` — low 3, environment only), then `git merge main` at `baaa4b47` (O4
+ merged). Conflicts only in `editor/CHANGELOG.md` (one `[Unreleased]`: O4's two bullets, then
+ O3's three) and this file's Record (O4's section, then O3's).
+- **The deferred `/sites` form change.** `cutReleaseAction` revalidates when the result is
+ `written: true`, then returns the error as before. The reviewer read it as correct and complete;
+ no e2e can make the fixture server's commit fail after the write, so tsc and the controller's
+ tests cover it, and `cut-release.spec.ts` re-ran green.
+- **should-fix — the Diagnostics claim was false.** The record and the changelog said the Diagnostics
+ buckets' retry controls keep their log; both Diagnostics grids drop an emptied bucket's card, so
+ they do not. Item 3 above, the editor bullet ("The Transcribe stage's "Fetch audio" button does the
+ same. (A Diagnostics card still disappears…)"), FACTS and found-and-left now say exactly what is
+ covered: the Download stage's three cards and the Transcribe stage's Fetch audio.
+- **low 1 — `sourceFetchFailure`'s order.** A no-subs fallback that fetched and persisted the source
+ and whose inline whisper then failed ends the download `failed`; reading the status first called
+ that source "not downloaded". When the last attempt is the media pass, its exit code alone
+ answers now; otherwise the status. Latent here (`inlineTranscribeOnFallback` is off on the
+ primary).
+- **low 2 — `persistKept`.** It counted every returned download as `persisted`; it now reads
+ `sourceFetchFailure` and counts a failed forced pass as `failed`, with the reason in its log.
+- **nit.** The inert `key="retry"` on the Download stage's three retry controls is gone.
+
+| sha | what |
+|---|---|
+| `0c770285` | merge `main` (`baaa4b47`, O4): the changelog and Record conflicts |
+| `f62d00f1` | the Cut release form revalidates on `written: true` |
+| `e9947421` | low 1: `sourceFetchFailure` reads a media-pass last attempt first; `forceMedia.test.ts`'s table +1 row |
+| `315c76e1` | low 2: `persistKept` counts a failed forced pass as failed; `persistKeptForceMedia.test.ts` +1 (setup made one helper with a process-wide settings file) |
+| `5c9987d2` | nit: the inert keys dropped |
+| _this_ | `plans:` these fixes, the corrected wording (record, changelog, FACTS) |
+
+**Gates on `5c9987d2`:**
+- tsc clean after the merge (`o3-tsc5.log`) and on the fixes (`o3-tsc6.log`); every fix commit is a
+ disjoint set of files from that tree.
+- common **2,051/2,051** (2,041 on `main` after O4, + O3's 10), editor unit **85/85**,
+ `test:scripts` **175 + 1 skip of 176**, mcp **269/269** (`o3-round2.log`).
+- e2e `cut-release ops-cut-release persist-youtube-handling fetch-window saved-videos cookies-mode`
+ (`o3-e2e-specs2.txt`, `o3-e2e2-full.log`): **28 passed, 0 failed, 1.3 min**, no queue wait.
+- **They bite** (`o3-bite3.log`): with the old order in `sourceFetchFailure`, the table test fails on
+ the new row; with `persistKept` counting every return, the new persistKept test fails; the other
+ 8 in the two files pass.
+
## Rollout
Nothing is rolled out tonight. The morning runbook lists what is owed: the :3001 editor restart,
