import path from "node:path"; import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { mkdir, readFile, readdir, stat } from "node:fs/promises"; import { writeJsonAtomic } from "../lib/jsonFile-server"; import { execa } from "execa"; import type { Paths } from "../lib/paths"; import { BACKUP_MANIFEST_FILENAME, backupEntryRelDir, backupEntryRelPath, parseBackupManifest, type BackupEntry, type BackupManifest, } from "../lib/savedVideoBackup"; import { listSavedVideos } from "./savedVideoInventory"; import { updateSavedVideoChecksum } from "../lib/savedVideo-server"; // Backup tooling for the saved-video store (Phase 4 of the video-persistence // feature). `dest` is treated as a local filesystem path (typically a mounted // backup disk): containers are mirrored into it with rsync — incremental and // resumable, so re-running only transfers changed/new files — and a manifest // recording each container's size + streamed sha256 is written at the dest root. // The verify step reads that manifest back and reports drift against the dest. // Stream a file through sha256 without buffering it whole (containers are large). async function sha256File(file: string): Promise { const hash = createHash("sha256"); await new Promise((resolve, reject) => { const stream = createReadStream(file); stream.on("error", reject); stream.on("data", (chunk) => hash.update(chunk)); stream.on("end", () => resolve()); }); return hash.digest("hex"); } export type BackupSavedVideosResult = { // Saved containers considered (one per data-dir pointer). entries: number; // Containers rsynced to the destination (entries minus any that errored). backedUp: number; // Total bytes across the backed-up containers. bytes: number; // Absolute path to the manifest written at the destination root. manifestPath: string; }; export async function backupSavedVideos({ paths, dest, onLog, signal, }: { paths: Paths; dest: string; onLog?: (msg: string) => void; signal?: AbortSignal; }): Promise { const log = onLog ?? ((m: string) => console.log(m)); const destRoot = dest.trim(); if (!destRoot) throw new Error("No backup destination configured"); const saved = await listSavedVideos({ paths }); log(`Backing up ${saved.length} saved video(s) to ${destRoot}`); await mkdir(destRoot, { recursive: true }); const manifestEntries: BackupEntry[] = []; let backedUp = 0; let bytes = 0; for (const entry of saved) { if (signal?.aborted) break; const destDir = path.join(destRoot, backupEntryRelDir(entry)); try { const st = await stat(entry.storedPath); await mkdir(destDir, { recursive: true }); // Trailing slash on destDir keeps rsync placing the file inside it. const args = ["-a", "--partial", entry.storedPath, `${destDir}/`]; log(`$ ${paths.rsyncBin} ${args.join(" ")}`); const child = execa(paths.rsyncBin, args, { cancelSignal: signal, all: true, buffer: false, reject: false, }); child.all?.on("data", (c: Buffer) => log(c.toString("utf8"))); const result = await child; if (result.exitCode !== 0) { if (signal?.aborted) break; log( `rsync failed for ${entry.slug}/${entry.videoId} (exit ${result.exitCode}); skipping`, ); continue; } const sha256 = await sha256File(entry.storedPath); // Cache the checksum back onto the pointer for the store/UI. await updateSavedVideoChecksum(entry.videoDir, sha256).catch(() => {}); manifestEntries.push({ slug: entry.slug, videoId: entry.videoId, file: entry.pointer.file, bytes: st.size, sha256, }); backedUp++; bytes += st.size; } catch (err) { log( `Failed to back up ${entry.slug}/${entry.videoId}: ${(err as Error).message}`, ); } } const manifest: BackupManifest = { backedUpAt: new Date().toISOString(), entries: manifestEntries, }; const manifestPath = path.join(destRoot, BACKUP_MANIFEST_FILENAME); await writeJsonAtomic(manifestPath, manifest); log( `Backup complete: ${backedUp}/${saved.length} container(s), ${bytes} bytes. Manifest at ${manifestPath}`, ); return { entries: saved.length, backedUp, bytes, manifestPath }; } export type BackupDrift = { // Manifest entries whose file is absent from the destination. missing: BackupEntry[]; // Present at the destination but a different size than the manifest records. sizeMismatch: BackupEntry[]; // Present and right-sized but whose content hash no longer matches (only // computed when `checksum` is true). checksumMismatch: BackupEntry[]; // Container files at the destination that the manifest doesn't know about // (relative // paths). extra: string[]; }; export type VerifyBackupResult = BackupDrift & { // True when every drift bucket is empty. ok: boolean; backedUpAt: string; checked: number; }; // Walk the destination two levels deep (/) collecting the // relative paths of stored container files, ignoring the manifest itself. async function listDestEntries(destRoot: string): Promise> { const out = new Set(); let slugs: string[]; try { slugs = (await readdir(destRoot, { withFileTypes: true })) .filter((e) => e.isDirectory()) .map((e) => e.name); } catch { return out; } for (const slug of slugs) { const slugDir = path.join(destRoot, slug); const ids = (await readdir(slugDir, { withFileTypes: true }).catch(() => [])) .filter((e) => e.isDirectory()) .map((e) => e.name); for (const id of ids) { const files = await readdir(path.join(slugDir, id)).catch( () => [] as string[], ); for (const f of files) out.add(path.join(slug, id, f)); } } return out; } export async function verifySavedVideoBackup({ paths: _paths, dest, onLog, signal, checksum = true, }: { paths: Paths; dest: string; onLog?: (msg: string) => void; signal?: AbortSignal; // Re-hash each present, right-sized file and compare to the manifest. Off // skips the (potentially expensive) hashing and only checks presence + size. checksum?: boolean; }): Promise { const log = onLog ?? ((m: string) => console.log(m)); const destRoot = dest.trim(); if (!destRoot) throw new Error("No backup destination configured"); let manifest: BackupManifest | null = null; try { manifest = parseBackupManifest( JSON.parse( await readFile(path.join(destRoot, BACKUP_MANIFEST_FILENAME), "utf8"), ), ); } catch { manifest = null; } if (!manifest) { throw new Error(`No readable backup manifest at ${destRoot}`); } const missing: BackupEntry[] = []; const sizeMismatch: BackupEntry[] = []; const checksumMismatch: BackupEntry[] = []; const known = new Set(); let checked = 0; for (const entry of manifest.entries) { if (signal?.aborted) break; const rel = backupEntryRelPath(entry); known.add(rel); const file = path.join(destRoot, rel); let st; try { st = await stat(file); } catch { missing.push(entry); continue; } checked++; if (st.size !== entry.bytes) { sizeMismatch.push(entry); continue; } if (checksum) { const got = await sha256File(file); if (got !== entry.sha256) checksumMismatch.push(entry); } } const destEntries = await listDestEntries(destRoot); const extra = [...destEntries].filter((rel) => !known.has(rel)).sort(); const ok = missing.length === 0 && sizeMismatch.length === 0 && checksumMismatch.length === 0 && extra.length === 0; log( `Verify: checked ${checked}/${manifest.entries.length}; ${missing.length} missing, ${sizeMismatch.length} size-mismatch, ${checksumMismatch.length} checksum-mismatch, ${extra.length} extra.`, ); return { ok, backedUpAt: manifest.backedUpAt, checked, missing, sizeMismatch, checksumMismatch, extra, }; }