// THE METADATA HISTORY SIDECAR, and the wrap every metadata.info.json writer // runs inside. Release 10 slice N; the rules (what is stored, what is only // fingerprinted, when an entry is written at all) are in metadataHistory.ts. // // ONE FILE, `metadata.history.json`, beside the file it describes — not a // directory. A new directory under data// would need the `clips/` treatment // in every video-dir enumerator (the snapshot, the storage view, the // reconciler); a bounded file needs none. // // IT IS NOT IN discardPrefetchDir's ALLOW-LIST, deliberately // (downloadOneManaged.ts, PREFETCH_OWN_FILES). A history exists only because a // metadata.info.json was already in the directory when some pass rewrote it — // the directory predates the pass now deciding whether to delete it, and the // history is the one record of what the source used to say. So a title-filter // rejection of such a directory keeps it and writes the outcome sidecar, which // is the cheap direction of being wrong (one metadata stub) that the allow-list // is built on. The case is narrow: a rejection discards its own prefetch dir, // so this needs a directory left by an earlier pass that was NOT rejected (a // failed download, say) and a filter that rejects it now. // // THE FILE yt-dlp WRITES IS STILL THE FILE. Nothing here changes what lands in // metadata.info.json; it reads the old bytes before the spawn and the new ones // after it, and appends what moved. // // SERVER-ONLY (node:fs). import path from "node:path"; import { readFile, stat } from "node:fs/promises"; import { withJsonFileLock, writeJsonAtomic } from "./jsonFile-server"; import { sidecar, sidecarField } from "./sidecar-server"; import { METADATA_HISTORY_FILENAME, appendMetadataHistoryEntry, buildMetadataHistoryEntry, coerceMetadataHistory, type MetadataHistoryEntry, type MetadataHistoryWriter, } from "./metadataHistory"; const INFO_JSON = "metadata.info.json"; export const metadataHistorySidecar = sidecar( METADATA_HISTORY_FILENAME, sidecarField(coerceMetadataHistory), ); export const { path: metadataHistoryPath, load: loadMetadataHistory } = metadataHistorySidecar; export type MetadataSnapshot = { text: Buffer; mtime: Date }; // The info json as it is on disk right now, or null when there is none (or it // cannot be read — the history is best-effort and never the reason a download // fails). export async function snapshotMetadata( videoDir: string, ): Promise { const file = path.join(videoDir, INFO_JSON); try { const [text, st] = await Promise.all([readFile(file), stat(file)]); return { text, mtime: st.mtime }; } catch { return null; } } export type MetadataRewriteContext = { by: MetadataHistoryWriter; requestedBy?: string; }; // Compare the file now against `before` and append an entry when it was // rewritten. Returns the entry, or null when there was nothing to record: no // old file (a first write is not a rewrite), no new file, or the same bytes. // // A READ-MODIFY-WRITE, so it holds the path's lock: two writers finishing at // once (a prefetch and an audio-check pass on one video are sequential, but a // second job on the same video is not) must not each append to the file they // read and drop the other's entry. export async function recordMetadataRewrite( videoDir: string, before: MetadataSnapshot | null, ctx: MetadataRewriteContext, ): Promise { if (!before) return null; const after = await snapshotMetadata(videoDir); if (!after) return null; const entry = buildMetadataHistoryEntry({ before, after, at: new Date().toISOString(), by: ctx.by, ...(ctx.requestedBy ? { requestedBy: ctx.requestedBy } : {}), }); if (!entry) return null; await withJsonFileLock(metadataHistorySidecar.path(videoDir), async () => { const existing = await loadMetadataHistory(videoDir); await metadataHistorySidecar.write( videoDir, appendMetadataHistoryEntry(existing, entry), ); }); return entry; } // Run one metadata writer with the history kept: snapshot before `run()`, // compare and append after it — whether it succeeded OR threw, because a // yt-dlp that fails late (a prefetch that wrote the file and then hit a // subtitle error) has still rewritten it. `run`'s own result or error is // returned unchanged; a failure to record is logged through `onLog` and // swallowed. `onEntry` hears the entry that was appended, when one was — the // refresh's closing summary names the keys that moved from it rather than // reading the sidecar back. export async function withMetadataHistory( videoDir: string, ctx: MetadataRewriteContext & { onLog?: (line: string) => void; onEntry?: (entry: MetadataHistoryEntry) => void; }, run: () => Promise, ): Promise { const before = await snapshotMetadata(videoDir); try { return await run(); } finally { if (before) { try { const entry = await recordMetadataRewrite(videoDir, before, ctx); if (entry) ctx.onEntry?.(entry); } catch (err) { ctx.onLog?.( `Could not record the metadata history: ${(err as Error).message}\n`, ); } } } } // THE ONE WRITER OF metadata.info.json THAT IS NOT yt-dlp: merge `patch` into // the file on disk, inside `withMetadataHistory`, so the rewrite is recorded // exactly as a yt-dlp one is (by `ctx.by`, with what moved). // // A MERGE, NEVER A REPLACEMENT. Every key the file has stays where it is and // keeps its value unless `patch` names it; a key the file lacks is appended in // `patch`'s order. Order is not cosmetic: two readers scan the file's BYTES // rather than parse it — the title from a 16 KB head (videoTitles.ts) and the // upload_date from an 8 KB tail (recencyIndex.ts) — so a caller adding a long // description and a date names the description first. // // Compact JSON, no trailing newline — yt-dlp's own shape, near enough for // every reader (they all allow whitespace after a colon). A patch that changes // nothing writes nothing. A missing file, or one that is not a JSON object, // throws: there is no record to complete, and a write would invent one. // // Under the file's lock, so two patches of one record never drop each other. // It does NOT serialise against a yt-dlp spawn rewriting the same file; the // caller's job runs on the channel's download queue for that reason. export async function patchMetadataInfo( videoDir: string, patch: Record, ctx: MetadataRewriteContext & { onLog?: (line: string) => void }, ): Promise<{ written: boolean }> { const file = path.join(videoDir, INFO_JSON); return withJsonFileLock(file, async () => { const text = await readFile(file, "utf8"); let current: unknown; try { current = JSON.parse(text); } catch { throw new Error(`${file} is not JSON`); } if (typeof current !== "object" || current === null || Array.isArray(current)) { throw new Error(`${file} is not a JSON object`); } const before = current as Record; const next: Record = { ...before }; let changed = false; for (const [key, value] of Object.entries(patch)) { if (value === undefined) continue; if (JSON.stringify(before[key]) === JSON.stringify(value)) continue; next[key] = value; changed = true; } if (!changed) return { written: false }; await withMetadataHistory(videoDir, ctx, () => writeJsonAtomic(file, next, { indent: 0, newline: false }), ); return { written: true }; }); }