commit 781493ed9b18aef9cd766ef040a9a8b5d834c871
parent b568edb817b2a4c2fa658dd624cb0786cbb5dda8
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Mon, 28 Sep 2026 01:37:39 -0400
common: a maybe-missing video whose confirm probe failed stays maybe_missing
resolveMaybeMissingState read an `error` availability probed after the
listing scan as a confirmation (stateFromAvailability folds error into
available), so a video whose confirm probe hit a 403, a network error, the
bot check or a blocked run's rate limit published as available with no
"Missing?" badge. An error probe is no evidence either way; the video now
stays maybe_missing, like one never probed. needs_auth (an age gate: the
video answering) still confirms.
Display only: the one caller is buildIndex's maybe-missing overlay (the
published presence state and the status chart); the clean gate reads
resolveEffectiveAvailability, untouched.
videoState.test.ts +2 (error after the scan, needs_auth after the scan);
maybeMissingBuild.test.ts (new) runs the real buildIndex over a temp corpus
and pins the state each video gets on the site's summaries page.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
3 files changed, 191 insertions(+), 4 deletions(-)
diff --git a/common/controller/maybeMissingBuild.test.ts b/common/controller/maybeMissingBuild.test.ts
@@ -0,0 +1,149 @@
+// Integration: the "Missing?" presence badge, through the REAL buildIndex, over
+// a temp corpus — the published half of resolveMaybeMissingState.
+//
+// The export draws the badge from a summary's `state` (export e2e
+// availability-state.spec.ts pins state → badge with mocked summaries). What
+// decides that `state` for a video that fell out of its channel's listing is
+// buildIndex's maybe-missing overlay, and that is what this file pins: the
+// site's summaries page, which is where the search results and the Availability
+// filter read `state` (the shared transcript pages carry cues, not state).
+//
+// Release 11 slice O3: a confirm probe made after the scan that FAILED (an
+// `error` availability) used to publish the video as available — no badge —
+// because stateFromAvailability folds `error` into `available`. It now stays
+// maybe_missing, like a video never probed.
+//
+// Run with: node_modules/.bin/tsx --test common/controller/maybeMissingBuild.test.ts
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtempSync, writeFileSync, mkdirSync, readFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
+
+// getPaths() is lazy and cached, and nothing above calls it at import time, so
+// pointing the whole path graph at a temp root here isolates this file's
+// process from the real corpus (the curatedTagsBuild.test.ts pattern).
+const ROOT = mkdtempSync(path.join(tmpdir(), "maybe-missing-build-"));
+process.env.TRANSCRIPTS_DIR = path.join(ROOT, "transcripts");
+process.env.EXPORT_PUBLIC_DIR = path.join(ROOT, "public");
+process.env.SETTINGS_FILE = path.join(ROOT, "settings.json");
+
+const { getPaths } = await import("../lib/paths");
+const { buildIndex } = await import("./buildIndex");
+
+const paths = getPaths();
+const CHANNEL = "test-channel";
+const SITE = "testsite";
+
+const SCAN_AT = "2026-08-01T12:00:00.000Z";
+const BEFORE_SCAN = "2026-07-20T00:00:00.000Z";
+const AFTER_SCAN = "2026-08-02T00:00:00.000Z";
+
+// One video per outcome of the overlay. Only LISTED is in the channel's
+// listing; every other id is in maybe-missing.json.
+const LISTED = "vid-listed";
+const NEVER_PROBED = "vid-never-probed";
+const ERROR_AFTER = "vid-error-after";
+const PUBLIC_AFTER = "vid-public-after";
+const PUBLIC_BEFORE = "vid-public-before";
+const DELETED = "vid-deleted";
+
+function writeJson(file: string, value: unknown): void {
+ mkdirSync(path.dirname(file), { recursive: true });
+ writeFileSync(file, JSON.stringify(value, null, 2));
+}
+
+function videoDir(id: string): string {
+ return path.join(paths.channelsDir, CHANNEL, "data", id);
+}
+
+function seedCorpus(): void {
+ writeFileSync(paths.settingsFile, JSON.stringify({}));
+ writeJson(path.join(paths.channelsDir, CHANNEL, "config.json"), {
+ handling: "youtube",
+ name: "Test Channel",
+ url: "https://www.youtube.com/@example/videos",
+ });
+ const ids = [LISTED, NEVER_PROBED, ERROR_AFTER, PUBLIC_AFTER, PUBLIC_BEFORE, DELETED];
+ ids.forEach((id, i) => {
+ writeJson(path.join(videoDir(id), "metadata.info.json"), {
+ id,
+ title: `Video ${id}`,
+ channel: "Test Channel",
+ upload_date: `2026010${i + 1}`,
+ duration: 120,
+ description: "fixture",
+ webpage_url: `https://www.youtube.com/watch?v=${id}`,
+ extractor_key: "Youtube",
+ });
+ writeFileSync(
+ path.join(videoDir(id), "transcript.en.vtt"),
+ "WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nA line of transcript.\n",
+ );
+ });
+ const availability = (id: string, availability: string, checkedAt: string) =>
+ writeJson(path.join(videoDir(id), "availability.json"), { checkedAt, availability });
+ availability(ERROR_AFTER, "error", AFTER_SCAN);
+ availability(PUBLIC_AFTER, "public", AFTER_SCAN);
+ availability(PUBLIC_BEFORE, "public", BEFORE_SCAN);
+ availability(DELETED, "deleted", BEFORE_SCAN);
+ writeJson(path.join(paths.channelsDir, CHANNEL, "maybe-missing.json"), {
+ checkedAt: SCAN_AT,
+ freshPlaylistCount: 1,
+ ids: [NEVER_PROBED, ERROR_AFTER, PUBLIC_AFTER, PUBLIC_BEFORE, DELETED],
+ });
+ writeJson(path.join(paths.sitesDir, SITE, "site.json"), {
+ siteId: SITE,
+ siteTitle: "Test Site",
+ siteDescription: "fixture",
+ headerTitle: "Test Site",
+ homeTagline: "",
+ socialLinks: [],
+ groups: [{ id: "default", name: "All channels", selectedByDefault: true }],
+ defaultGroupId: "default",
+ channels: [{ slug: CHANNEL, groupId: "default" }],
+ });
+}
+
+type Published = { id: string; state?: string; isDeleted?: boolean };
+
+function readPage(file: string): Published[] {
+ return JSON.parse(readFileSync(file, "utf8")) as Published[];
+}
+
+const summariesPage = () =>
+ readPage(path.join(paths.exportSitesIndexDir, SITE, "summaries", "page-0000.json"));
+
+function stateOf(page: Published[], id: string): string {
+ const record = page.find((r) => r.id === id);
+ assert.ok(record, `${id} is published`);
+ // The build omits `state` for an available video (the badge's absence).
+ return record.state ?? "available";
+}
+
+test("the published state of each video that left the listing, as the badge reads it", async () => {
+ seedCorpus();
+ const log: string[] = [];
+ const res = await buildIndex({ paths, onLog: (s) => log.push(s) });
+ assert.equal(res.totalCount, 6);
+
+ const expected: Record<string, string> = {
+ [LISTED]: "available",
+ [NEVER_PROBED]: "maybe_missing",
+ // THE CASE: the confirm probe failed after the scan. No evidence either
+ // way, so the badge stays.
+ [ERROR_AFTER]: "maybe_missing",
+ [PUBLIC_AFTER]: "available",
+ [PUBLIC_BEFORE]: "maybe_missing",
+ [DELETED]: "deleted",
+ };
+ const page = summariesPage();
+ const got = Object.fromEntries(Object.keys(expected).map((id) => [id, stateOf(page, id)]));
+ assert.deepEqual(got, expected);
+ // The build's own count line names the three unconfirmed videos.
+ assert.ok(
+ log.some((l) => l.includes("Availability: 3 video(s) unconfirmed-missing from a channel listing.")),
+ log.filter((l) => l.startsWith("Availability")).join("\n") || "no Availability line",
+ );
+});
diff --git a/common/lib/availability-server.ts b/common/lib/availability-server.ts
@@ -140,8 +140,19 @@ export async function resolveEffectiveAvailability(
// anyway (a truncated or oddly paginated playlist fetch) is resolved, and
// must stop being flagged.
//
-// Anything else — never probed, or last probed before the scan — is
-// `maybe_missing`: we know it left the listing, not yet why.
+// Anything else — never probed, last probed before the scan, or probed after
+// it with an `error` — is `maybe_missing`: we know it left the listing, not
+// yet why.
+//
+// AN `error` PROBE ANSWERS NOTHING, however recent (release 11 slice O3). It is
+// a 403, a network failure, YouTube's bot check or the one rate-limited probe of
+// a blocked run: the probe never saw the video. `stateFromAvailability` folds
+// it into `available` because it is no evidence the video is GONE — and by the
+// same token it is no evidence the video is THERE. Read as a confirmation, it
+// cleared the "Missing?" badge of exactly the videos whose confirm probe failed.
+// (`needs_auth` is different: an age gate is the video answering.) Display
+// only — this feeds the published presence state via buildIndex; the clean gate
+// reads resolveEffectiveAvailability, which this does not touch.
export async function resolveMaybeMissingState(
videoDir: string,
scannedAtMs: number,
@@ -150,6 +161,7 @@ export async function resolveMaybeMissingState(
if (!record) return "maybe_missing";
const confirmed = stateFromAvailability(record.availability);
if (confirmed !== "available") return confirmed;
+ if (record.availability === "error") return "maybe_missing";
const checkedAtMs = Date.parse(record.checkedAt);
if (Number.isFinite(checkedAtMs) && checkedAtMs >= scannedAtMs) {
return "available";
diff --git a/common/lib/videoState.test.ts b/common/lib/videoState.test.ts
@@ -99,8 +99,34 @@ test("in the set + public, last probed BEFORE the scan → maybe_missing", async
});
});
-// needs_auth/error are not evidence the video left its channel, so they behave
-// exactly like public: still absent from the listing, still unconfirmed.
+// A probe made AFTER the scan that FAILED saw nothing: a 403, a network error,
+// the bot check, the one rate-limited probe of a blocked run. It must not read
+// as a confirmation the way a public re-probe does (release 11 slice O3; it
+// used to, through stateFromAvailability's `error` → available fold).
+test("in the set + error, probed AFTER the scan → still maybe_missing", async () => {
+ await withVideoDir(async (videoDir) => {
+ await seedAvailability(videoDir, "error", AFTER_SCAN);
+ assert.equal(
+ await resolveMaybeMissingState(videoDir, SCAN_AT_MS),
+ "maybe_missing",
+ );
+ });
+});
+
+// An age gate is the video answering: a needs_auth re-probe after the scan
+// confirms it is there, exactly like public.
+test("in the set + needs_auth, probed AFTER the scan → available", async () => {
+ await withVideoDir(async (videoDir) => {
+ await seedAvailability(videoDir, "needs_auth", AFTER_SCAN);
+ assert.equal(
+ await resolveMaybeMissingState(videoDir, SCAN_AT_MS),
+ "available",
+ );
+ });
+});
+
+// needs_auth/error are not evidence the video left its channel, so before the
+// scan they behave exactly like public: still absent, still unconfirmed.
for (const availability of ["needs_auth", "error"] as const) {
test(`in the set + ${availability} before the scan → maybe_missing`, async () => {
await withVideoDir(async (videoDir) => {
