// ARCHIVE.ORG FILE RECORDS, BROUGHT UP TO THEIR PROVENANCE'S RULES — offline. // // A record imported as ONE FILE of a multi-file item whose mirror has no // uploaded info.json was titled with the raw file name and dated with the // item's date. The rule is lib/archiveOrg.ts (withFileNameFields → // archiveOrgMetadataPatch): the title its name carries (where the item gives // the file none) and the day its name starts with. This brings every such // record of a channel up to it: // // archiveorg.json the mirror's `title` and `uploadDate`, when the rule // gives different ones (from a name only; an // info.json's are never touched); `file` keeps the raw // name // metadata.info.json `title`, `upload_date` and `timestamp` (dropped // beside a mirror's date, as at import), through // `patchMetadataInfo`, so the change is in // metadata.history.json as `archiveorg-provenance` // // Only those keys — every other key the provenance decides was written at // import. Only when they differ, so a second run writes nothing. No network. import path from "node:path"; import { getPaths, type Paths } from "../lib/paths"; import { readJsonFile } from "../lib/jsonFile-server"; import { assertChannelTextReadable } from "../lib/channelMedia"; import { archiveOrgMetadataPatch, withFileNameFields } from "../lib/archiveOrg"; import { loadArchiveOrgProvenance, writeArchiveOrgProvenance } from "../lib/archiveOrg-server"; import { patchMetadataInfo } from "../lib/metadataHistory-server"; import { readChannelConfig } from "./channels"; import { listChannelVideoIds } from "./keptVideos"; const REFRESHED_KEYS = ["title", "upload_date", "timestamp"] as const; export type ArchiveOrgRecordChange = { id: string; // Per key: the value before and after (null: absent / removed). changes: Partial>; }; export type ArchiveOrgRefreshResult = { // archive.org records of one file of an item. fileRecords: number; changed: ArchiveOrgRecordChange[]; // Sidecars whose mirror title or date was (or would be) rewritten. sidecars: number; failed: { id: string; error: string }[]; }; export async function refreshArchiveOrgRecords(opts: { slug: string; paths?: Paths; dryRun?: boolean; onLog?: (line: string) => void; }): Promise { const paths = opts.paths ?? getPaths(); const log = opts.onLog ?? (() => {}); const config = await readChannelConfig(paths, opts.slug); if (!config) throw new Error(`Channel "${opts.slug}" not found`); // An unreadable text tier is not an empty channel (AGENTS.md). await assertChannelTextReadable(paths, opts.slug, config); const dataDir = path.join(paths.channelsDir, opts.slug, "data"); const result: ArchiveOrgRefreshResult = { fileRecords: 0, changed: [], sidecars: 0, failed: [] }; for (const id of (await listChannelVideoIds(paths, opts.slug)).sort()) { const videoDir = path.join(dataDir, id); const prov = await loadArchiveOrgProvenance(videoDir); if (!prov?.file) continue; result.fileRecords++; try { const next = withFileNameFields(prov); if (next !== prov) { result.sidecars++; if (!opts.dryRun) await writeArchiveOrgProvenance(videoDir, next); } const read = await readJsonFile(path.join(videoDir, "metadata.info.json")); const info = read.ok && read.value && typeof read.value === "object" && !Array.isArray(read.value) ? (read.value as Record) : null; if (!info) continue; const full = archiveOrgMetadataPatch(next, info); const patch: Record = {}; const change: ArchiveOrgRecordChange = { id, changes: {} }; for (const key of REFRESHED_KEYS) { if (!(key in full)) continue; patch[key] = full[key]; change.changes[key] = { from: info[key] ?? null, to: full[key] }; } if (Object.keys(patch).length === 0) continue; result.changed.push(change); log( `${id}\n` + Object.entries(change.changes) .map(([k, c]) => ` ${k}: ${JSON.stringify(c!.from)}\n → ${JSON.stringify(c!.to)}`) .join("\n"), ); if (!opts.dryRun) { await patchMetadataInfo(videoDir, patch, { by: "archiveorg-provenance", requestedBy: "cli", onLog: log }); } } catch (err) { result.failed.push({ id, error: (err as Error).message }); log(`${id}: failed — ${(err as Error).message}`); } } return result; }