import { test } from "node:test"; import assert from "node:assert/strict"; import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync, } from "node:fs"; import { cp, mkdir, rename, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import type { Paths } from "../lib/paths"; import { bundleCounts, bundleDir, corpusGeneratedAtIn, dockerSiteOutDir, exportOutPath, installBundle, pointExportOutAt, recoverInterruptedInstall, stageSiteArchives, unlinkExportOut, type BundleFs, } from "./build"; // Run with: // pnpm --filter yt-dlp-transcript-common test // // The per-target bundle layout (release 18): a build in export/out is // installed as //out by a swap through out.next, the // move falls back to a copy across filesystems (an injected EXDEV here), and // export/out becomes a link to the bundle built last. function tmp(): { root: string; paths: Paths } { const root = mkdtempSync(path.join(os.tmpdir(), "bundle-")); const exportDir = path.join(root, "export"); mkdirSync(exportDir, { recursive: true }); return { root, paths: { exportDir, exportPublicDir: path.join(exportDir, "public"), exportBuildsDir: path.join(exportDir, ".export-builds"), } as Paths, }; } function writeBuild(dir: string, marker: string): void { mkdirSync(path.join(dir, "_next"), { recursive: true }); writeFileSync(path.join(dir, "index.html"), marker); writeFileSync(path.join(dir, "_next", "app.js"), "x".repeat(10)); writeFileSync(path.join(dir, "corpus.json"), JSON.stringify({ generatedAt: `at-${marker}` })); } // A filesystem whose renames out of `crossFrom` fail EXDEV, as export/out's do // in the container (an image layer) when the builds dir is a volume. function exdevFs(crossFrom: string, calls: string[]): BundleFs { return { rename: (async (a: string, b: string) => { calls.push(`rename ${path.basename(String(a))} -> ${path.basename(String(b))}`); if (String(a).startsWith(crossFrom)) { throw Object.assign(new Error("EXDEV: cross-device link not permitted"), { code: "EXDEV" }); } return rename(a, b); }) as typeof rename, cp: (async (a: string, b: string, o?: object) => { calls.push(`cp ${path.basename(String(a))} -> ${path.basename(String(b))}`); return cp(a, b, o); }) as typeof cp, rm, mkdir, }; } test("bundleDir is //out — a site's is dockerSiteOutDir", () => { const p = { exportBuildsDir: "/e/.export-builds", exportDir: "/e" } as Paths; assert.equal(bundleDir(p, "jer"), "/e/.export-builds/jer/out"); assert.equal(bundleDir(p, "jer"), dockerSiteOutDir(p, "jer")); assert.equal(bundleDir(p, "_hub"), "/e/.export-builds/_hub/out"); assert.equal(exportOutPath(p), "/e/out"); }); test("installBundle moves the build in through out.next, replacing the last bundle; a leftover out.next goes first", async () => { const { root, paths } = tmp(); try { const dest = bundleDir(paths, "jer"); writeBuild(dest, "old"); writeBuild(`${dest}.next`, "leftover"); const out = exportOutPath(paths); writeBuild(out, "new"); assert.equal(await installBundle(out, dest), "rename"); assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "new"); assert.ok(!existsSync(out), "export/out moved away"); assert.ok(!existsSync(`${dest}.next`)); assert.ok(!existsSync(`${dest}.prev`)); } finally { rmSync(root, { recursive: true, force: true }); } }); test("installBundle copies across filesystems (EXDEV), then removes the source", async () => { const { root, paths } = tmp(); try { const dest = bundleDir(paths, "jer"); writeBuild(dest, "old"); const out = exportOutPath(paths); writeBuild(out, "new"); const calls: string[] = []; assert.equal(await installBundle(out, dest, exdevFs(out, calls)), "copy"); assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "new"); assert.equal(readFileSync(path.join(dest, "_next", "app.js"), "utf8"), "x".repeat(10)); assert.ok(!existsSync(out), "the source is removed after the copy"); assert.deepEqual(calls, [ "rename out -> out.next", // EXDEV "cp out -> out.next", "rename out -> out.prev", // siblings: same filesystem "rename out.next -> out", ]); } finally { rmSync(root, { recursive: true, force: true }); } }); test("export/out becomes a relative link to the bundle; unlinkExportOut removes a link, never a real build", async () => { const { root, paths } = tmp(); try { const out = exportOutPath(paths); writeBuild(out, "real"); await unlinkExportOut(paths); assert.ok(lstatSync(out).isDirectory(), "a real out/ is left for next build"); const dest = bundleDir(paths, "jer"); await installBundle(out, dest); await pointExportOutAt(paths, dest); assert.ok(lstatSync(out).isSymbolicLink()); assert.equal(readlinkSync(out), path.join(".export-builds", "jer", "out")); assert.equal(readFileSync(path.join(out, "index.html"), "utf8"), "real", "reads through the link"); // Pointing it at another bundle replaces the link, not the bundle. const hub = bundleDir(paths, "_hub"); writeBuild(hub, "hub"); await pointExportOutAt(paths, hub); assert.equal(readFileSync(path.join(out, "index.html"), "utf8"), "hub"); assert.ok(existsSync(path.join(dest, "index.html")), "the other bundle is untouched"); await unlinkExportOut(paths); assert.ok(!existsSync(out)); assert.ok(existsSync(path.join(hub, "index.html")), "the link's target survives"); // A real out/ left behind is replaced by the link. writeBuild(out, "stray"); await pointExportOutAt(paths, dest); assert.ok(lstatSync(out).isSymbolicLink()); } finally { rmSync(root, { recursive: true, force: true }); } }); test("stageSiteArchives moves the host staging into the bundle's (dockerSiteStagingDir) and counts it", async () => { const { root, paths } = tmp(); try { const host = path.join(paths.exportDir, ".r2-staging", "jer", "archives"); mkdirSync(host, { recursive: true }); writeFileSync(path.join(host, "a.zip"), "a"); writeFileSync(path.join(host, "b.zip"), "b"); writeFileSync(path.join(host, ".hidden"), ""); const bundleStaging = path.join(paths.exportBuildsDir, "jer", ".r2-staging", "jer", "archives"); mkdirSync(bundleStaging, { recursive: true }); writeFileSync(path.join(bundleStaging, "last-build.zip"), "old"); assert.equal(await stageSiteArchives(paths, "jer"), 2); assert.ok(existsSync(path.join(bundleStaging, "a.zip"))); assert.ok(!existsSync(path.join(bundleStaging, "last-build.zip")), "the last build's staging is replaced"); assert.ok(!existsSync(host)); // Nothing staged this build: the bundle has none either. assert.equal(await stageSiteArchives(paths, "jer"), 0); assert.ok(!existsSync(bundleStaging)); // Across filesystems. mkdirSync(host, { recursive: true }); writeFileSync(path.join(host, "c.zip"), "c"); const calls: string[] = []; assert.equal(await stageSiteArchives(paths, "jer", exdevFs(path.join(paths.exportDir, ".r2-staging"), calls)), 1); assert.ok(calls.some((c) => c.startsWith("cp "))); } finally { rmSync(root, { recursive: true, force: true }); } }); test("stageSiteArchives is a no-op where the two are one place (the build container's compose)", async () => { const root = mkdtempSync(path.join(os.tmpdir(), "bundle-same-")); try { // EXPORT_PUBLIC_DIR=/site/public and a builds dir whose IS /site. const site = path.join(root, "jer"); const paths = { exportPublicDir: path.join(site, "public"), exportBuildsDir: root } as Paths; const staged = path.join(site, ".r2-staging", "jer", "archives"); mkdirSync(staged, { recursive: true }); writeFileSync(path.join(staged, "a.zip"), "a"); assert.equal(await stageSiteArchives(paths, "jer"), 1); assert.ok(existsSync(path.join(staged, "a.zip"))); } finally { rmSync(root, { recursive: true, force: true }); } }); test("bundleCounts and corpusGeneratedAtIn read the bundle; links are not followed", async () => { const { root } = tmp(); try { const dir = path.join(root, "b"); writeBuild(dir, "m"); symlinkSync("/etc", path.join(dir, "link")); const c = await bundleCounts(dir); assert.equal(c.files, 3); assert.equal(c.bytes, 1 + 10 + JSON.stringify({ generatedAt: "at-m" }).length); assert.equal(await corpusGeneratedAtIn(dir), "at-m"); assert.equal(await corpusGeneratedAtIn(path.join(root, "none")), null); assert.deepEqual(await bundleCounts(path.join(root, "none")), { files: 0, bytes: 0 }); } finally { rmSync(root, { recursive: true, force: true }); } }); test("a crash between the two renames leaves out.prev and no out; the next build or install restores it first", async () => { const { root, paths } = tmp(); try { const dest = bundleDir(paths, "jer"); writeBuild(dest, "old"); const out = exportOutPath(paths); writeBuild(out, "new"); // The process dies on the second rename (out.next -> out). const dying: BundleFs = { rename: (async (a: string, b: string) => { if (String(a).endsWith("out.next") && String(b).endsWith(path.join("jer", "out"))) throw new Error("killed"); return rename(a, b); }) as typeof rename, cp, rm, mkdir, }; await assert.rejects(installBundle(out, dest, dying), /killed/); assert.ok(!existsSync(dest), "no out"); assert.ok(existsSync(`${dest}.prev`), "the last bundle, under the wrong name"); // Nothing to do when out is there; the old bundle back when it is not. assert.equal(await recoverInterruptedInstall(dest), true); assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "old"); assert.equal(await recoverInterruptedInstall(dest), false); // And an install over the same crash recovers before it swaps. rmSync(`${dest}.next`, { recursive: true, force: true }); await rename(dest, `${dest}.prev`); writeBuild(out, "newer"); await installBundle(out, dest); assert.equal(readFileSync(path.join(dest, "index.html"), "utf8"), "newer"); assert.ok(!existsSync(`${dest}.prev`)); } finally { rmSync(root, { recursive: true, force: true }); } });