import type { Roster } from "./rosterStore"; // The single place "missing" is defined. Pure, no fs, so every set can be // unit-tested against a hand-built triple instead of a seeded temp corpus. // // Three inputs, and each answers a different question: // roster — every id the channel has EVER been seen to contain (durable) // listedIds — what the latest ACCEPTED enumeration contained (current truth) // onDiskIds — data// dir names (what we actually fetched) // // Before the roster existed only the last two were available, so "missing" // could only ever mean "on disk but no longer listed" — a video that was listed // and never fetched had no dir, could not be flagged, and vanished along with // its URL when the sweep overwrote `playlist`. missingNeverFetched is the // category that hole made inexpressible. export type ChannelSets = { // The current listing, sorted. listed: string[]; // Listed but nothing on disk — the download work-list. Replaces walking the // `playlist` file directly, so it can't evaporate with a bad enumeration. undownloaded: string[]; // Everything we know of that the current listing does not contain. missing: string[]; // missing ∩ on disk. EXACTLY today's maybe-missing set (see the note below). missingDownloaded: string[]; // missing, never fetched. We knew about this video, never downloaded it, and // it is gone — for an archival tool, the highest-value signal in the system. // The roster holds its URL precisely so a recovery attempt is still possible. missingNeverFetched: string[]; // On disk but not in the roster at all: a one-off import that predates the // roster, or a dir dropped in by hand. Diagnostic only. orphaned: string[]; }; export function deriveChannelSets(input: { roster: Roster; listedIds: ReadonlySet; onDiskIds: ReadonlySet; // Ids this channel's download filter has SETTLED — the operator's own "not // this one". Optional, and empty for every channel without a filter. // // WHY IT BELONGS HERE. Both sets below are defined by the ABSENCE of a // directory, and since a title-filter rejection stopped leaving its prefetch // dir behind, a settled video has none. Without this, a filtered-out video // that later left the listing read as `missingNeverFetched` — "we were told // about this, never got it, and now it is gone" — which is the highest-value // alarm this system raises, pointed at a video the operator asked us not to // fetch. `undownloaded` has the same shape of wrongness: it is a work list, // and settled work is not work. settledIds?: ReadonlySet; }): ChannelSets { const { roster, listedIds, onDiskIds } = input; const settledIds = input.settledIds ?? new Set(); const rosterIds = Object.keys(roster.entries); const listed: string[] = []; const undownloaded: string[] = []; for (const id of listedIds) { listed.push(id); if (!onDiskIds.has(id) && !settledIds.has(id)) undownloaded.push(id); } // missingDownloaded is derived from the DISK set, not from `roster ∩ disk`, // and that is deliberate: it is what maybe-missing.json has always meant // ("we have this and the listing no longer does"), and buildIndex, // channelSnapshot and verifyBeforeClean all read that file. Deriving it from // the roster instead would silently drop any on-disk video the roster hadn't // caught up with yet — a stale roster must never be able to un-flag a // deletion. With seeding + additive writers the two sets coincide anyway. const missingDownloaded: string[] = []; const orphaned: string[] = []; for (const id of onDiskIds) { if (!listedIds.has(id)) missingDownloaded.push(id); if (!roster.entries[id]) orphaned.push(id); } const missingNeverFetched: string[] = []; for (const id of rosterIds) { if (settledIds.has(id)) continue; if (!listedIds.has(id) && !onDiskIds.has(id)) missingNeverFetched.push(id); } const missing = [...missingDownloaded, ...missingNeverFetched]; return { listed: listed.sort(), undownloaded: undownloaded.sort(), missing: missing.sort(), missingDownloaded: missingDownloaded.sort(), missingNeverFetched: missingNeverFetched.sort(), orphaned: orphaned.sort(), }; }