commit a495e1a2fcd423ae1bdbeb8fc1940031a442f1c1
parent 7f7ac6a868e50250ebaa90271b4ed8613a652289
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 19 Sep 2026 23:57:56 -0400
download filter: a channel can decline a video by title, and the decline settles
`ChannelConfig.downloadFilter` is two case-insensitive regexes over
`title + "\n" + description` — the only text a download filter ever sees,
since it runs on the per-video metadata prefetch. It is the first entry in
the FILTERS registry, ahead of skipLive, so an unwanted video is settled
rather than classified as a live-stream retry.
A non-match is settled by a TERMINAL OUTCOME keyed on the filter's signature,
not by an archive line. An archive line means "downloaded" to
verifyTranscripts and the missingFromArchive bucket, and undownloadedIds is
artifact-based, so the id would come straight back. `"v1\0"+include+"\0"+exclude`
re-evaluates itself instead: edit either pattern and every video the old one
decided stops being settled, with no migration and no sweep.
isSettledByFilter is the ONE definition of settled; isVideoSettledByFilter is
its per-video-dir form, shaped like isDeferredAuthExcluded so the exclusion
passes can treat the two the same way. A configured filter with no metadata
fails CLOSED as a non-permanent skip — the existing retryable bucket — and an
unparseable regex makes the filter inert plus one log line.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
6 files changed, 519 insertions(+), 5 deletions(-)
diff --git a/common/lib/channelConfig.ts b/common/lib/channelConfig.ts
@@ -56,6 +56,14 @@ export type AudioCheckConfig = {
resumeDuringProbe?: boolean;
};
+// Per-channel download filter patterns. Regex SOURCES, always compiled with
+// the "i" flag; an unparseable pattern makes the filter inert rather than
+// failing a download (the editor form refuses to save one).
+export type DownloadFilterConfig = {
+ include?: string;
+ exclude?: string;
+};
+
export type ChannelConfig = {
handling: ChannelHandling;
// Omitted = "video" (every channel that predates the posts corpus).
@@ -159,6 +167,19 @@ export type ChannelConfig = {
// omitted, the global SiteSettings value is used. Set false to allow this
// channel to download currently-live/upcoming videos.
skipLiveDownloads?: boolean;
+ // Per-channel title/description download filter. Both patterns are
+ // case-insensitive regex SOURCES (no delimiters, no flags) matched against
+ // `title + "\n" + description` from the per-video metadata prefetch — the
+ // only text a download filter ever gets to see. `exclude` wins over
+ // `include`. A video the filter declines is SETTLED by a terminal
+ // download-outcome keyed on the filter's signature (see
+ // downloadFilterSignature), NOT by an archive line: an archive line would
+ // mean "downloaded" to verifyTranscripts and the missingFromArchive bucket,
+ // and would still leave the id in the artifact-based undownloadedIds. The
+ // signature is what makes the settlement self-expiring — change either
+ // pattern and every settled video is re-evaluated on the next run.
+ // Omitted/empty = no filter (the registry entry is inert).
+ downloadFilter?: DownloadFilterConfig;
// Per-channel override of the global cookies-from-browser browser spec
// (SiteSettings.cookiesFromBrowser). Omitted/empty = inherit the global
// value. See common/lib/cookiePolicy.ts.
@@ -331,6 +352,20 @@ export function parseChannelConfig(raw: unknown): ChannelConfig | null {
if (typeof r.skipLiveDownloads === "boolean") {
config.skipLiveDownloads = r.skipLiveDownloads;
}
+ // Download filter. Blank/whitespace patterns are dropped rather than stored,
+ // so an all-blank object leaves the key absent and the filter inert — the
+ // same "empty means inherit-off" rule the cookie overrides use.
+ if (r.downloadFilter && typeof r.downloadFilter === "object") {
+ const df = r.downloadFilter as Record<string, unknown>;
+ const include = typeof df.include === "string" ? df.include.trim() : "";
+ const exclude = typeof df.exclude === "string" ? df.exclude.trim() : "";
+ if (include || exclude) {
+ config.downloadFilter = {
+ ...(include ? { include } : {}),
+ ...(exclude ? { exclude } : {}),
+ };
+ }
+ }
if (typeof r.cookiesFromBrowser === "string" && r.cookiesFromBrowser.trim()) {
config.cookiesFromBrowser = r.cookiesFromBrowser.trim();
}
diff --git a/common/lib/downloadFilters.test.ts b/common/lib/downloadFilters.test.ts
@@ -0,0 +1,278 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import type { ChannelConfig } from "./channelConfig";
+import type { DownloadOutcomeRecord } from "./downloadOutcome";
+import {
+ compileDownloadFilter,
+ downloadFilterSignature,
+ evaluateDownloadFilters,
+ isSettledByFilter,
+ type DownloadFilterContext,
+} from "./downloadFilters";
+import type { RawMetadata } from "./transcripts-server";
+
+// Run with: node_modules/.bin/tsx --test common/lib/downloadFilters.test.ts
+
+const BASE_CONFIG: ChannelConfig = { handling: "youtube", name: "Test" };
+
+function meta(over: Partial<RawMetadata> = {}): RawMetadata {
+ return {
+ id: "vid",
+ title: "Synthetic vid",
+ description: "Synthetic video vid",
+ is_live: false,
+ was_live: false,
+ live_status: "not_live",
+ ...over,
+ };
+}
+
+function ctx(
+ over: Partial<DownloadFilterContext> = {},
+): DownloadFilterContext {
+ return {
+ metadata: meta(),
+ channelConfig: BASE_CONFIG,
+ // Global skip-live default is ON, matching downloadOneManaged's fallback.
+ settings: { skipLiveDownloads: true },
+ ...over,
+ };
+}
+
+function withFilter(
+ include?: string,
+ exclude?: string,
+): ChannelConfig {
+ return {
+ ...BASE_CONFIG,
+ downloadFilter: {
+ ...(include ? { include } : {}),
+ ...(exclude ? { exclude } : {}),
+ },
+ };
+}
+
+test("no filter configured -> no decision, and no signature", () => {
+ assert.equal(downloadFilterSignature(undefined), null);
+ assert.equal(downloadFilterSignature({}), null);
+ assert.equal(downloadFilterSignature({ include: " " }), null);
+ assert.equal(compileDownloadFilter(undefined), null);
+ assert.equal(evaluateDownloadFilters(ctx()), null);
+});
+
+test("include matches the title -> downloads", () => {
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("guest"),
+ metadata: meta({ title: "Synthetic guestvid0001", description: "x" }),
+ }),
+ );
+ assert.equal(d, null);
+});
+
+test("include misses -> permanent skip carrying the signature", () => {
+ const config = withFilter("guest");
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: config,
+ metadata: meta({ title: "Synthetic plainvid0001", description: "x" }),
+ }),
+ );
+ assert.ok(d);
+ assert.equal(d.skip, true);
+ assert.equal(d.filter, "titleFilter");
+ assert.equal(d.permanent, true);
+ assert.equal(d.signature, downloadFilterSignature(config.downloadFilter));
+ assert.match(d.reason, /does not match include/);
+});
+
+test("the description alone can satisfy include", () => {
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("special guest"),
+ metadata: meta({
+ title: "Episode 12",
+ description: "With a special guest this week",
+ }),
+ }),
+ );
+ assert.equal(d, null);
+});
+
+test("exclude wins over include", () => {
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("guest", "rerun"),
+ metadata: meta({ title: "guest episode (rerun)", description: "" }),
+ }),
+ );
+ assert.ok(d);
+ assert.equal(d.permanent, true);
+ assert.match(d.reason, /matches exclude/);
+});
+
+test("matching is case-insensitive in both directions", () => {
+ assert.equal(
+ evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("GUEST"),
+ metadata: meta({ title: "a guest appears", description: "" }),
+ }),
+ ),
+ null,
+ );
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter(undefined, "rerun"),
+ metadata: meta({ title: "A RERUN", description: "" }),
+ }),
+ );
+ assert.ok(d);
+ assert.equal(d.permanent, true);
+});
+
+test("invalid regex -> inert filter, one log line, nothing skipped", () => {
+ const lines: string[] = [];
+ const d = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("elf("),
+ metadata: meta({ title: "anything at all", description: "" }),
+ onLog: (l) => lines.push(l),
+ }),
+ );
+ assert.equal(d, null);
+ assert.equal(lines.length, 1);
+ assert.match(lines[0], /INERT/);
+ assert.equal(compileDownloadFilter({ include: "elf(" }), null);
+ // An inert filter still HAS a signature — the config is what it is; only the
+ // compile fails. Nothing gets settled because nothing is skipped.
+ assert.notEqual(downloadFilterSignature({ include: "elf(" }), null);
+});
+
+test("null metadata with a filter configured -> NON-permanent skip", () => {
+ const d = evaluateDownloadFilters(
+ ctx({ channelConfig: withFilter("guest"), metadata: null }),
+ );
+ assert.ok(d);
+ assert.equal(d.skip, true);
+ assert.equal(d.filter, "titleFilter");
+ assert.equal(d.permanent, false);
+ assert.match(d.reason, /failing closed/);
+});
+
+test("null metadata with NO filter configured -> no skip (fail-open)", () => {
+ assert.equal(evaluateDownloadFilters(ctx({ metadata: null })), null);
+});
+
+function settledOutcome(signature: string): DownloadOutcomeRecord {
+ return {
+ videoId: "plainvid0001",
+ status: "skipped-filtered",
+ startedAt: "2026-01-01T00:00:00.000Z",
+ finishedAt: "2026-01-01T00:00:01.000Z",
+ attempts: [],
+ filter: {
+ name: "titleFilter",
+ reason: "does not match include",
+ permanent: true,
+ signature,
+ },
+ };
+}
+
+test("a signature change flips isSettledByFilter back to false", () => {
+ const config = withFilter("guest");
+ const sig = downloadFilterSignature(config.downloadFilter);
+ assert.ok(sig);
+ const outcome = settledOutcome(sig);
+
+ assert.equal(isSettledByFilter(outcome, config), true);
+ // Same include, different exclude -> different signature -> unsettled.
+ assert.equal(isSettledByFilter(outcome, withFilter("guest", "rerun")), false);
+ // A different include -> unsettled.
+ assert.equal(isSettledByFilter(outcome, withFilter("plain")), false);
+ // Filter removed entirely -> unsettled.
+ assert.equal(isSettledByFilter(outcome, BASE_CONFIG), false);
+ assert.equal(isSettledByFilter(outcome, null), false);
+});
+
+test("only a permanent, signed filter skip is settled", () => {
+ const config = withFilter("guest");
+ const sig = downloadFilterSignature(config.downloadFilter)!;
+ assert.equal(isSettledByFilter(null, config), false);
+ assert.equal(
+ isSettledByFilter(
+ { status: "ok", filter: { name: "titleFilter", reason: "", permanent: true, signature: sig } },
+ config,
+ ),
+ false,
+ );
+ // skip-live's skip carries neither flag: never settled, always retried.
+ assert.equal(
+ isSettledByFilter(
+ { status: "skipped-filtered", filter: { name: "skipLive", reason: "live" } },
+ config,
+ ),
+ false,
+ );
+ // Permanent but unsigned can never match a signature.
+ assert.equal(
+ isSettledByFilter(
+ {
+ status: "skipped-filtered",
+ filter: { name: "titleFilter", reason: "", permanent: true },
+ },
+ config,
+ ),
+ false,
+ );
+});
+
+test("skipLive still fires, and titleFilter runs first", () => {
+ // A live video with no title filter: skip-live decides, as before.
+ const live = evaluateDownloadFilters(
+ ctx({ metadata: meta({ is_live: true, live_status: "is_live" }) }),
+ );
+ assert.ok(live);
+ assert.equal(live.filter, "skipLive");
+ assert.equal(live.permanent, undefined);
+
+ // A live video that ALSO misses the include: the title filter answers,
+ // because it is first in the registry and its verdict is the terminal one.
+ const both = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("guest"),
+ metadata: meta({
+ title: "Synthetic plainvid0001",
+ description: "",
+ is_live: true,
+ live_status: "is_live",
+ }),
+ }),
+ );
+ assert.ok(both);
+ assert.equal(both.filter, "titleFilter");
+
+ // A live video that PASSES the include still falls through to skip-live.
+ const passThrough = evaluateDownloadFilters(
+ ctx({
+ channelConfig: withFilter("guest"),
+ metadata: meta({
+ title: "Synthetic guestvid0001",
+ description: "",
+ is_live: true,
+ live_status: "is_live",
+ }),
+ }),
+ );
+ assert.ok(passThrough);
+ assert.equal(passThrough.filter, "skipLive");
+
+ // Finished VOD is untouched by either (regression guard).
+ assert.equal(
+ evaluateDownloadFilters(
+ ctx({ metadata: meta({ was_live: true, live_status: "was_live" }) }),
+ ),
+ null,
+ );
+});
diff --git a/common/lib/downloadFilters.ts b/common/lib/downloadFilters.ts
@@ -10,7 +10,8 @@
// Adding a filter = append one entry to FILTERS. Each filter sees the same
// context (metadata + channel config + resolved settings).
-import type { ChannelConfig } from "./channelConfig";
+import type { ChannelConfig, DownloadFilterConfig } from "./channelConfig";
+import type { DownloadOutcomeRecord } from "./downloadOutcome";
import type { RawMetadata } from "./transcripts-server";
export type DownloadFilterSettings = {
@@ -22,12 +23,26 @@ export type DownloadFilterContext = {
metadata: RawMetadata | null;
channelConfig: ChannelConfig;
settings: DownloadFilterSettings;
+ // Optional job log. A filter uses it only to report that it made itself
+ // INERT (an unparseable regex on disk) — a silent no-op filter is the one
+ // failure mode nothing downstream can explain.
+ onLog?: (line: string) => void;
};
export type DownloadFilterDecision = {
skip: boolean;
filter: string;
reason: string;
+ // TERMINAL, not "tried and failed". A permanent skip settles the video: the
+ // snapshot keeps it out of undownloadedIds and out of totals.videos, and sync
+ // counts it as a hit so a filtered channel's paged walk still stops on its
+ // first settled page instead of re-walking every non-match every day.
+ // Undefined/false = the historical retryable skip (skip-live).
+ permanent?: boolean;
+ // The filter identity a permanent skip is settled AGAINST. Re-evaluated on
+ // every read: a video is settled only while this equals the channel's current
+ // signature, so editing a pattern un-settles everything it decided.
+ signature?: string;
};
type DownloadFilter = {
@@ -68,7 +83,125 @@ const skipLive: DownloadFilter = {
},
};
-const FILTERS: ReadonlyArray<DownloadFilter> = [skipLive];
+// ---------------------------------------------------------------------------
+// The per-channel title/description filter.
+// ---------------------------------------------------------------------------
+
+// The filter's identity, and the whole settlement mechanism in one string.
+// Returns null when the channel has no filter configured (both patterns blank),
+// which is also the "nothing is settled here" answer — an outcome recorded under
+// a signature can never match null.
+//
+// Versioned ("v1") so a future change to what the patterns are matched against
+// invalidates every stored settlement rather than silently keeping decisions
+// that were made against different text. NUL-separated because a regex may
+// contain any other character.
+export function downloadFilterSignature(
+ filter: DownloadFilterConfig | undefined | null,
+): string | null {
+ const include = filter?.include?.trim() ?? "";
+ const exclude = filter?.exclude?.trim() ?? "";
+ if (!include && !exclude) return null;
+ return `v1\0${include}\0${exclude}`;
+}
+
+export type CompiledDownloadFilter = {
+ include: RegExp | null;
+ exclude: RegExp | null;
+ signature: string;
+};
+
+// Compile a channel's patterns. Returns null when there is no filter AND when a
+// pattern doesn't parse — an invalid regex makes the filter inert rather than
+// failing every download on the channel. The editor form refuses to save one, so
+// this only fires for a hand-edited config.json.
+export function compileDownloadFilter(
+ filter: DownloadFilterConfig | undefined | null,
+): CompiledDownloadFilter | null {
+ const signature = downloadFilterSignature(filter);
+ if (!signature) return null;
+ const include = filter?.include?.trim() ?? "";
+ const exclude = filter?.exclude?.trim() ?? "";
+ try {
+ return {
+ include: include ? new RegExp(include, "i") : null,
+ exclude: exclude ? new RegExp(exclude, "i") : null,
+ signature,
+ };
+ } catch {
+ return null;
+ }
+}
+
+// The only text a download filter ever sees. Kept here so the preview job and
+// the filter itself can never match against different strings.
+export function downloadFilterText(
+ meta: Pick<RawMetadata, "title" | "description"> | null | undefined,
+): string {
+ return `${meta?.title ?? ""}\n${meta?.description ?? ""}`;
+}
+
+// Per-channel include/exclude over title + description. Runs BEFORE skipLive so
+// a video the operator doesn't want is settled without also being classified as
+// a live-stream retry.
+//
+// FAILS CLOSED on missing metadata: a prefetch that produced nothing cannot be
+// matched, and downloading it anyway would defeat the filter on exactly the
+// videos a flaky extractor hides. The skip is recorded NON-permanent, so it
+// lands in the existing retryable skippedByFilter bucket and the next run tries
+// again — the opposite of a settled video.
+const titleFilter: DownloadFilter = {
+ name: "titleFilter",
+ evaluate(ctx) {
+ const raw = ctx.channelConfig.downloadFilter;
+ const signature = downloadFilterSignature(raw);
+ if (!signature) return null;
+ const compiled = compileDownloadFilter(raw);
+ if (!compiled) {
+ ctx.onLog?.(
+ `Download filter is INERT: include=${JSON.stringify(
+ raw?.include ?? "",
+ )} exclude=${JSON.stringify(
+ raw?.exclude ?? "",
+ )} is not a valid regex — nothing will be filtered.\n`,
+ );
+ return null;
+ }
+ const m = ctx.metadata;
+ if (!m) {
+ return {
+ skip: true,
+ filter: "titleFilter",
+ reason:
+ "no metadata to match the download filter against (failing closed)",
+ permanent: false,
+ signature,
+ };
+ }
+ const text = downloadFilterText(m);
+ if (compiled.exclude && compiled.exclude.test(text)) {
+ return {
+ skip: true,
+ filter: "titleFilter",
+ reason: `title/description matches exclude /${compiled.exclude.source}/i`,
+ permanent: true,
+ signature,
+ };
+ }
+ if (compiled.include && !compiled.include.test(text)) {
+ return {
+ skip: true,
+ filter: "titleFilter",
+ reason: `title/description does not match include /${compiled.include.source}/i`,
+ permanent: true,
+ signature,
+ };
+ }
+ return null;
+ },
+};
+
+const FILTERS: ReadonlyArray<DownloadFilter> = [titleFilter, skipLive];
// Evaluate every filter; the first one that says "skip" wins. Returns null when
// the video passes all filters.
@@ -81,3 +214,26 @@ export function evaluateDownloadFilters(
}
return null;
}
+
+// THE ONE DEFINITION OF "SETTLED". Every reader — the snapshot's bucket, the
+// sync page walk, the batch exclusion pass, the video page's badge — asks this
+// and nothing else, because "is this video done with?" answered two ways is how
+// a filtered channel ends up both settled and re-queued.
+//
+// Three things must hold: the outcome is a filter skip, it was recorded as
+// permanent, and its signature is still the channel's. The third is what makes
+// an edited pattern un-settle every video it decided, with no migration and no
+// sweep.
+export function isSettledByFilter(
+ outcome:
+ | Pick<DownloadOutcomeRecord, "status" | "filter">
+ | null
+ | undefined,
+ config: Pick<ChannelConfig, "downloadFilter"> | null | undefined,
+): boolean {
+ if (!outcome || outcome.status !== "skipped-filtered") return false;
+ const recorded = outcome.filter;
+ if (!recorded?.permanent || !recorded.signature) return false;
+ const current = downloadFilterSignature(config?.downloadFilter);
+ return current !== null && current === recorded.signature;
+}
diff --git a/common/lib/downloadOutcome-server.ts b/common/lib/downloadOutcome-server.ts
@@ -1,5 +1,10 @@
import path from "node:path";
import { readFile, rename, writeFile } from "node:fs/promises";
+import type { ChannelConfig } from "./channelConfig";
+import {
+ downloadFilterSignature,
+ isSettledByFilter,
+} from "./downloadFilters";
import {
DOWNLOAD_OUTCOME_FILENAME,
DOWNLOAD_OUTCOME_STATUS_VALUES,
@@ -44,5 +49,18 @@ export async function writeDownloadOutcome(
await rename(tmp, file);
}
+// Per-video-dir predicate for the download filter's settlement, shaped exactly
+// like isDeferredAuthExcluded (runYtdlp.ts) so the batch/sync exclusion passes
+// treat the two the same way. The signature check runs FIRST: a channel with no
+// filter configured pays no file read at all, which matters because this is
+// called once per URL on every page of every sync.
+export async function isVideoSettledByFilter(
+ videoDir: string,
+ config: Pick<ChannelConfig, "downloadFilter"> | null | undefined,
+): Promise<boolean> {
+ if (downloadFilterSignature(config?.downloadFilter) === null) return false;
+ return isSettledByFilter(await loadDownloadOutcome(videoDir), config);
+}
+
// Narrow re-export so callers don't have to import from both modules.
export type { DownloadOutcomeStatus };
diff --git a/common/lib/downloadOutcome.ts b/common/lib/downloadOutcome.ts
@@ -99,7 +99,22 @@ export type DownloadOutcomeRecord = {
};
// Set when status is "skipped-filtered": which app-level filter declined the
// download and why. Recorded so the UI/log can explain the skip.
- filter?: { name: string; reason: string };
+ //
+ // `permanent` + `signature` are what make a skip SETTLE. skip-live's skip is
+ // transient (the stream ends and the video becomes downloadable), so it
+ // carries neither and every sync retries it. The per-channel download filter's
+ // skip is terminal FOR AS LONG AS THE FILTER SAYS SO: `signature` is the
+ // filter's identity (see downloadFilterSignature) and a settled video is one
+ // whose recorded signature still equals the channel's current one. Change
+ // either pattern and the signature changes, the video stops being settled, and
+ // it is re-evaluated on the next run — which is the whole reason this is a
+ // signature rather than an archive line.
+ filter?: {
+ name: string;
+ reason: string;
+ permanent?: boolean;
+ signature?: string;
+ };
};
export const DOWNLOAD_OUTCOME_FILENAME = "download-outcome.json";
diff --git a/common/ytdlp/downloadOneManaged.ts b/common/ytdlp/downloadOneManaged.ts
@@ -604,10 +604,13 @@ async function runManagedDownload(
metadata,
channelConfig: opts.channelConfig,
settings: { skipLiveDownloads: opts.globalSkipLiveDownloads ?? true },
+ onLog: opts.onLog,
});
if (decision?.skip) {
opts.onLog(
- `Skipping ${canonicalId}: ${decision.reason} [filter=${decision.filter}]\n`,
+ `Skipping ${canonicalId}: ${decision.reason} [filter=${decision.filter}${
+ decision.permanent ? ", settled" : ""
+ }]\n`,
);
const finishedAt = new Date().toISOString();
const record: DownloadOutcomeRecord = {
@@ -617,7 +620,16 @@ async function runManagedDownload(
startedAt,
finishedAt,
attempts,
- filter: { name: decision.filter, reason: decision.reason },
+ // permanent/signature ride along verbatim: they are the filter's own
+ // verdict, and isSettledByFilter re-checks the signature on every read.
+ filter: {
+ name: decision.filter,
+ reason: decision.reason,
+ ...(decision.permanent === undefined
+ ? {}
+ : { permanent: decision.permanent }),
+ ...(decision.signature ? { signature: decision.signature } : {}),
+ },
};
try {
await mkdir(videoDir, { recursive: true });
