import { AVAILABILITY_FILENAME, AVAILABILITY_VALUES, stateFromAvailability, type Availability, type VideoState, type AvailabilityHistoryEntry, type AvailabilityHistorySource, type AvailabilityRecord, } from "./availability"; import { loadDownloadOutcome } from "./downloadOutcome-server"; import { sidecar, sidecarField } from "./sidecar-server"; export function coerceAvailability(value: unknown): AvailabilityRecord | null { const parsed = value as Partial | null; if ( typeof parsed?.availability === "string" && (AVAILABILITY_VALUES as string[]).includes(parsed.availability) && typeof parsed.checkedAt === "string" ) { return parsed as AvailabilityRecord; } return null; } export const availabilitySidecar = sidecar( AVAILABILITY_FILENAME, sidecarField(coerceAvailability), ); export const { path: availabilityPath, load: loadAvailability, write: writeAvailability, } = availabilitySidecar; // Append-on-change writer that all availability persistence funnels through. // Records an observation in the video's history[] only when the availability // differs from the most recent known value, then writes the record atomically. // // `updateTopLevel` (default true): the explicit check/backfill sources own the // top-level checkedAt/availability/error/webpageUrl fields. The download source // passes `updateTopLevel: false` so it appends a history entry without // disturbing those fields (keeping resolveEffectiveAvailability's recency // comparison against download-outcome.json intact). export async function recordAvailability( videoDir: string, obs: { availability: Availability; observedAt: string; source: AvailabilityHistorySource; error?: string; webpageUrl?: string; }, opts?: { updateTopLevel?: boolean }, ): Promise { const updateTopLevel = opts?.updateTopLevel !== false; const existing = await loadAvailability(videoDir); const lastAvail = existing?.history?.at(-1)?.availability ?? existing?.availability ?? null; const changed = lastAvail !== obs.availability; const history: AvailabilityHistoryEntry[] = [...(existing?.history ?? [])]; if (changed) { history.push({ availability: obs.availability, observedAt: obs.observedAt, source: obs.source, }); } let record: AvailabilityRecord; if (updateTopLevel) { record = { checkedAt: obs.observedAt, availability: obs.availability, history, }; if (obs.error !== undefined) record.error = obs.error; if (obs.webpageUrl !== undefined) record.webpageUrl = obs.webpageUrl; } else if (existing) { record = { ...existing, history }; } else { record = { checkedAt: obs.observedAt, availability: obs.availability, history, }; if (obs.error !== undefined) record.error = obs.error; if (obs.webpageUrl !== undefined) record.webpageUrl = obs.webpageUrl; } await writeAvailability(videoDir, record); } // Resolve a video's availability class by reconciling two sources: // - availability.json (written by the explicit availability-check pass) // - download-outcome.json's last attempt's availabilityClass (recorded // whenever a download attempt finishes, including the failure cases // where yt-dlp reports the video is removed/private/members-only) // Both can be authoritative; the older one can also be stale. We use // whichever was written more recently. The outcome source matters not just // for videos that have never reached the availability check, but also for // videos whose status changed after the last check — e.g. a public video // that the uploader has since deleted will fail a download attempt with // `availabilityClass: "deleted"` long before the next availability sweep. export async function resolveEffectiveAvailability( videoDir: string, ): Promise { const [explicit, outcome] = await Promise.all([ loadAvailability(videoDir), loadDownloadOutcomeAvailability(videoDir), ]); if (explicit && outcome) { if ( outcome.finishedAt && new Date(outcome.finishedAt) > new Date(explicit.checkedAt) ) { return outcome.availability; } return explicit.availability; } if (explicit) return explicit.availability; return outcome?.availability ?? null; } // Decide the VideoState for one video that the last quick availability check // found absent from its channel's fresh playlist listing. `scannedAtMs` is that // check's `checkedAt`. Three outcomes, and both of the non-obvious ones earn // their keep: // // - a per-video probe that already *explains* the absence wins outright. // An unlisted video legitimately falls out of a listing forever, and // checkMaybeMissingAction passes `skipExpectedAbsent: true`, so a // known-gone video is never re-probed and would otherwise stay flagged // as unconfirmed permanently. // - the converse: a video re-probed *after* the scan that came back public // anyway (a truncated or oddly paginated playlist fetch) is resolved, and // must stop being flagged. // // Anything else — never probed, last probed before the scan, or probed after // it with an `error` — is `maybe_missing`: we know it left the listing, not // yet why. // // AN `error` PROBE ANSWERS NOTHING, however recent (release 11 slice O3). It is // a 403, a network failure, YouTube's bot check or the one rate-limited probe of // a blocked run: the probe never saw the video. `stateFromAvailability` folds // it into `available` because it is no evidence the video is GONE — and by the // same token it is no evidence the video is THERE. Read as a confirmation, it // cleared the "Missing?" badge of exactly the videos whose confirm probe failed. // (`needs_auth` is different: an age gate is the video answering.) Display // only — this feeds the published presence state via buildIndex; the clean gate // reads resolveEffectiveAvailability, which this does not touch. export async function resolveMaybeMissingState( videoDir: string, scannedAtMs: number, ): Promise { const record = await loadAvailability(videoDir); if (!record) return "maybe_missing"; const confirmed = stateFromAvailability(record.availability); if (confirmed !== "available") return confirmed; if (record.availability === "error") return "maybe_missing"; const checkedAtMs = Date.parse(record.checkedAt); if (Number.isFinite(checkedAtMs) && checkedAtMs >= scannedAtMs) { return "available"; } return "maybe_missing"; } // The download-outcome side of resolveEffectiveAvailability: the LAST // attempt's availabilityClass, when there is one. Reads through // loadDownloadOutcome, so a download-outcome.json that fails that shape check // no longer contributes an availability (slice 4b record, behaviour changes). async function loadDownloadOutcomeAvailability( videoDir: string, ): Promise<{ availability: Availability; finishedAt: string | null } | null> { const outcome = await loadDownloadOutcome(videoDir); const last = outcome?.attempts.at(-1); if (!outcome || !last?.availabilityClass) return null; return { availability: last.availabilityClass, finishedAt: typeof outcome.finishedAt === "string" ? outcome.finishedAt : null, }; }