import { test } from "node:test"; import assert from "node:assert/strict"; import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defaultPublish, type PublishSettings } from "../lib/settingsSchema"; import { buildPublishStatus, planPublishRun, publishPassDecision, stepKey, type PublishInputs, type PublishSiteInput, type PublishStatus, } from "./publishPlan"; import { STAGES } from "./stages"; import type { BuiltStamp, DeployedFile, IndexStamp } from "./stamps"; // Run with: pnpm --filter yt-dlp-transcript-common exec tsx --test publish/publishPlan.test.ts // // The publish status and plan (release 18) over hand-built inputs: every chip // state the plan's tables name, the policies, the preconditions a run carries, // and the lane's pass decision. Pure — no disk, no clock. const HERE = path.dirname(fileURLToPath(import.meta.url)); const MIN = 60_000; const NOW = 2_000_000_000_000; function stamp(over: Partial = {}): IndexStamp { return { v: 1, stampId: "stamp-1", generation: 7, scannedAt: NOW - 60 * MIN, builtAt: NOW - 55 * MIN, templatesAt: NOW - 55 * MIN, commit: "c1", index: { shortCircuited: false, added: 0, changed: 0, removed: 0, heldChannels: [] }, stats: { shortCircuited: false, notIndexedYet: 0, notIndexable: 0 }, sites: { alpha: { siteFp: null, statsFp: null, inputSig: "sig-alpha" }, beta: { siteFp: null, statsFp: null, inputSig: "sig-beta" }, }, hubSig: "hub-sig", ...over, }; } function built(target: string, over: Partial = {}): BuiltStamp { return { v: 1, stampId: `built-${target}`, target, kind: target === "_hub" ? "hub" : target === "_homepage" ? "homepage" : "site", indexStampId: "stamp-1", inputSig: target === "_hub" ? "hub-sig" : `sig-${target}`, builtAt: NOW - 50 * MIN, commit: "c1", branch: "main", runner: "local", audience: "public", corpusGeneratedAt: null, files: 1, bytes: 1, archivesStaged: 0, ...over, }; } function deployedTo(target: string, builtStampId: string, kind: "production" | "preview" = "production"): DeployedFile { const rec = { builtStampId, builtAt: NOW - 50 * MIN, kind, url: `https://${target}.example`, at: NOW - 40 * MIN, liveCheck: null, ...(kind === "preview" ? { branch: "preview", alias: `https://preview.${target}.pages.dev` } : {}), }; return kind === "production" ? { v: 1, target, production: rec, previews: {} } : { v: 1, target, previews: { preview: rec } }; } function site(siteId: string, over: Partial = {}): PublishSiteInput { return { siteId, title: siteId.toUpperCase(), private: false, listed: true, members: [`${siteId}-ch`, "shared-ch"], built: built(siteId), deployed: deployedTo(siteId, `built-${siteId}`), bundleProblem: null, deployProblem: null, pagesProblem: null, configChangedAt: null, policy: "off", cloudflareProject: siteId, url: `https://${siteId}.pages.dev`, ...over, }; } function inputs(over: Partial = {}, settings: Partial = {}): PublishInputs { return { index: { stamp: stamp(), lastIngestDoneAt: null, configChangedAt: null }, ingestByChannel: {}, commit: "c1", sites: [site("alpha"), site("beta")], hub: { built: built("_hub"), deployed: deployedTo("_hub", "built-_hub"), bundleProblem: null, deployProblem: null, pagesProblem: null, configChangedAt: null, policy: "off", cloudflareProject: "hub", url: null, }, homepage: { built: built("_homepage"), deployed: deployedTo("_homepage", "built-_homepage"), bundleProblem: null, deployProblem: null, pagesProblem: null, configChangedAt: null, policy: "off", cloudflareProject: "archilyzer", url: null, mainHead: null, }, settings: { ...defaultPublish(), ...settings }, jobs: [], ended: [], lane: { known: true, running: false, jobId: null, passRunning: false, lastCheckAt: null, nextCheckAt: null, lastPassAt: null, lastPassSummary: null, lastDecision: null, }, ...over, }; } const status = (i: PublishInputs): PublishStatus => buildPublishStatus(i, NOW); const siteOf = (s: PublishStatus, id: string) => s.sites.find((t) => t.target === id)!; const kinds = (s: PublishStatus["plan"]) => s.steps.map((x) => `${x.kind} ${x.target}`); // --- purity ------------------------------------------------------------------ test("the builder and the planner are pure: no fs, no clock, no singleton", async () => { const src = await readFile(path.join(HERE, "publishPlan.ts"), "utf8"); for (const banned of ["Date.now(", "getSettings(", "getPaths(", "getRegistry(", "getScheduler(", "readJobMeta("]) { assert.equal(src.includes(banned), false, `publishPlan.ts calls ${banned}`); } const specs = [...src.matchAll(/from\s+"([^"]+)"/g)].map((m) => m[1]); for (const spec of specs) { assert.equal(spec.startsWith("node:"), false, `publishPlan.ts imports ${spec}`); assert.ok(spec.startsWith("."), `publishPlan.ts imports a package (${spec})`); } // The same inputs give the same status. assert.deepEqual(status(inputs()), status(inputs())); }); // --- the index chip ------------------------------------------------------------ test("index: no stamp is stale and plans the update; every build is then after it", () => { const s = status(inputs({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } }, {})); assert.equal(s.index.fresh, false); assert.equal(s.index.chip.text, "no index yet"); assert.deepEqual(kinds(s.plan), ["update-index _index"]); // With a site on "build", it is built after the index's first update. const i = inputs({ index: { stamp: null, lastIngestDoneAt: null, configChangedAt: null } }); i.sites[0].policy = "build"; const p = status(i).plan; assert.deepEqual(kinds(p), ["update-index _index", "build-site alpha"]); assert.equal(p.steps[1].indexAfter, NOW); // The chip a build shows with no stamp is the stage's own "blocked". assert.equal(siteOf(status(i), "alpha").chips.built.text, "update the index first"); }); test("index: fresh, stale by new data (after scannedAt), stale by a config file", () => { assert.match(status(inputs()).index.chip.text, /^fresh · updated 55 min ago$/); const byData = status(inputs({ index: { stamp: stamp(), lastIngestDoneAt: NOW - 30 * MIN, configChangedAt: null } })); assert.equal(byData.index.fresh, false); assert.equal(byData.index.chip.text, "stale: new data since the last index"); // An ingest that ended BEFORE the scan began is in the index. assert.equal( status(inputs({ index: { stamp: stamp(), lastIngestDoneAt: NOW - 61 * MIN, configChangedAt: null } })).index.fresh, true, ); const byConfig = status(inputs({ index: { stamp: stamp(), lastIngestDoneAt: null, configChangedAt: NOW - MIN } })); assert.equal(byConfig.index.chip.text, "stale: a config file changed since the last index"); }); test("index: a running or queued update reads busy", () => { const job = { id: "J1", kind: "update-index" as const, target: "_index", runId: "r", queuedAt: NOW - MIN }; assert.equal(status(inputs({ jobs: [{ ...job, status: "running" }] })).index.chip.text, "updating…"); const q = status(inputs({ jobs: [{ ...job, status: "queued" }] })); assert.equal(q.index.chip.text, "update queued"); assert.equal(q.busy, true); }); // --- the built chip ------------------------------------------------------------ test("built: fresh; never built; stale by channels, by data, by config; a bundle problem", () => { assert.match(siteOf(status(inputs()), "alpha").chips.built.text, /^built 50 min ago$/); assert.equal(siteOf(status(inputs()), "alpha").buildFreshness.state, "fresh"); const never = inputs(); never.sites[0].built = null; assert.equal(siteOf(status(never), "alpha").chips.built.text, "never built"); // Channels: an ingest of a MEMBER after the build. const ch = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "elsewhere": NOW - MIN } }); const a = siteOf(status(ch), "alpha"); assert.deepEqual(a.buildStale, { reason: "channels", changedChannels: ["alpha-ch"] }); assert.equal(a.chips.built.text, "stale: 1 channel changed (alpha-ch)"); assert.equal(siteOf(status(ch), "beta").buildStale, undefined, "beta's members did not change"); // Data: the index says the site's inputs moved. const data = inputs({ index: { stamp: stamp({ sites: { ...stamp().sites, alpha: { siteFp: null, statsFp: null, inputSig: "moved" } } }), lastIngestDoneAt: null, configChangedAt: null, }, }); const d = siteOf(status(data), "alpha"); assert.deepEqual(d.buildStale, { reason: "data", changedChannels: [] }); assert.equal(d.chips.built.text, "stale: data changed"); assert.equal(d.builtFromCurrentIndex, false); assert.equal(siteOf(status(data), "beta").builtFromCurrentIndex, true); // Config: its site.json (or tags, aliases, duplicates) after the build. const cfg = inputs(); cfg.sites[0].configChangedAt = NOW - MIN; assert.equal(siteOf(status(cfg), "alpha").chips.built.text, "stale: config changed"); const bundle = inputs(); bundle.sites[0].bundleProblem = "no corpus.json"; assert.equal(siteOf(status(bundle), "alpha").chips.built.text, "bundle: no corpus.json"); }); test("built: channels changed is measured against builtCheckedAt — a no-op build clears it", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 30 * MIN } }); assert.equal(siteOf(status(i), "alpha").buildStale?.reason, "channels"); // A later no-op build found the bundle current at NOW - 20 min. i.sites[0].built = built("alpha", { checkedAt: NOW - 20 * MIN }); const a = siteOf(status(i), "alpha"); assert.equal(a.buildStale, undefined); assert.equal(a.buildFreshness.state, "fresh"); }); test("built: code newer is said, never stale", () => { const s = status(inputs({ commit: "c2" })); const a = siteOf(s, "alpha"); assert.equal(a.codeNewer, true); assert.equal(a.buildFreshness.state, "fresh"); assert.match(a.chips.built.text, /· code newer$/); assert.equal(a.chips.built.tone, "ok"); assert.equal(s.plan.steps.length, 0); }); // --- deployed / live --------------------------------------------------------------- test("deployed: the policy's record; a newer build not deployed; never; private", () => { const ok = siteOf(status(inputs()), "alpha"); assert.equal(ok.deployedIsBuilt, true); assert.match(ok.chips.deployed.text, /^production · 40 min ago$/); const newer = inputs(); newer.sites[0].built = built("alpha", { stampId: "built-alpha-2" }); const n = siteOf(status(newer), "alpha"); assert.equal(n.deployedIsBuilt, false); assert.equal(n.chips.deployed.text, "production: a newer build is not deployed"); const preview = inputs(); preview.sites[0].policy = "preview"; const p = siteOf(status(preview), "alpha"); assert.equal(p.deployKind, "preview"); assert.equal(p.chips.deployed.text, 'not on preview "preview"'); assert.equal(p.previewUrl, "https://preview.alpha.pages.dev"); const priv = inputs(); priv.sites[0] = site("alpha", { private: true, deployed: null, deployProblem: "private site", policy: "build" }); assert.equal(siteOf(status(priv), "alpha").chips.deployed.text, "never deployed (private)"); assert.equal(siteOf(status(inputs()), "alpha").chips.live.text, "not checked"); }); test("deployed: a deploy refused for want of its build reads 'waiting for its build'", () => { const i = inputs({ ended: [ { id: "D1", kind: "deploy-site", target: "alpha", status: "failed", exitCode: 3, endedAt: NOW - MIN, runId: "r", builtAfter: NOW - 2 * MIN, }, ], }); const a = siteOf(status(i), "alpha"); assert.deepEqual(a.chips.deployed, { tone: "blocked", text: "waiting for its build" }); i.ended[0] = { ...i.ended[0], builtAfter: undefined }; assert.equal(siteOf(status(i), "alpha").chips.deployed.text, "last deploy refused: precondition not met"); }); test("live: the record's live-check verdict", () => { const i = inputs(); const rec = i.sites[0].deployed!.production!; const check = { at: NOW, url: "https://alpha.pages.dev/corpus.json", plain: { status: 200 }, busted: { status: 200 }, expected: null, }; rec.liveCheck = { ...check, verdict: "ok" }; assert.equal(siteOf(status(i), "alpha").chips.live.text, "live ok"); rec.liveCheck = { ...check, verdict: "stale-edge" }; assert.equal(siteOf(status(i), "alpha").chips.live.tone, "warn"); }); // --- the plan: policies ------------------------------------------------------------ test("Publish now with every policy off is the index update alone", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN }, index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null } }); const s = status(i); assert.deepEqual(kinds(s.plan), ["update-index _index"]); assert.deepEqual( s.plan.skipped.map((x) => `${x.kind} ${x.target}: ${x.reason}`), ["build-site alpha: stale, and its policy is off"], ); assert.equal(siteOf(s, "alpha").next, "stale, and its policy is off"); }); test("policies: build builds; preview and production also deploy, after the build, with builtAfter", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "beta-ch": NOW - 10 * MIN }, index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null }, }); i.sites[0].policy = "preview"; i.sites[1].policy = "production"; const p = status(i).plan; assert.deepEqual(kinds(p), [ "update-index _index", "build-site alpha", "deploy-site alpha", "build-site beta", "deploy-site beta", ]); const [, ba, da, , db] = p.steps; assert.equal(ba.indexAfter, NOW, "a build in a run with the index carries indexAfter"); assert.equal(da.builtAfter, NOW, "a deploy whose build is in the run carries builtAfter"); assert.equal(da.preview, "preview"); assert.equal(da.previewUrl, "https://preview.alpha.pages.dev"); assert.equal(db.preview, undefined, "production"); assert.equal(db.builtAfter, NOW); // The words the targets show. assert.equal(siteOf(status(i), "alpha").next, 'build, then deploy to preview "preview"'); const buildOnly = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN } }); buildOnly.sites[0].policy = "build"; const b = status(buildOnly).plan; assert.deepEqual(kinds(b), ["build-site alpha"]); assert.equal(b.steps[0].indexAfter, undefined, "no index update in the run, no indexAfter"); }); test("plan: a site with neither changed channels nor a signature mismatch is skipped", () => { const i = inputs({ index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null } }); for (const s of i.sites) s.policy = "production"; assert.deepEqual(kinds(status(i).plan), ["update-index _index"]); }); test("plan: a deploy-only step when the build is current but not deployed", () => { const i = inputs(); i.sites[0].policy = "production"; i.sites[0].deployed = null; const p = status(i).plan; assert.deepEqual(kinds(p), ["deploy-site alpha"]); assert.equal(p.steps[0].builtAfter, undefined); }); test("private sites never deploy; a project-less site is skipped with the reason", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "beta-ch": NOW - 10 * MIN } }); // A private site's policy is clamped on read; even a stray production one // is never planned as a deploy. i.sites[0] = site("alpha", { private: true, policy: "production", deployProblem: "Site \"alpha\" is private" }); i.sites[1] = site("beta", { policy: "preview", pagesProblem: "no Pages project", cloudflareProject: null }); const p = status(i).plan; assert.deepEqual(kinds(p), ["build-site alpha", "build-site beta"]); assert.deepEqual( p.skipped.map((x) => `${x.kind} ${x.target}`).sort(), ["deploy-site alpha", "deploy-site beta"], ); }); test("plan: builds 'stale' builds every stale site whatever its policy, deploys only by policy", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "beta-ch": NOW - 10 * MIN } }); i.sites[1].policy = "production"; const p = planPublishRun(status(i), { builds: "stale" }); assert.deepEqual(kinds(p), ["build-site alpha", "build-site beta", "deploy-site beta"]); assert.deepEqual(kinds(planPublishRun(status(i), { builds: "stale", deploys: "none" })), [ "build-site alpha", "build-site beta", ]); }); test("plan: the docker runner builds every stale site in one _all stage", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "beta-ch": NOW - 10 * MIN } }, { runner: "docker" }); i.sites[0].policy = "production"; i.sites[1].policy = "build"; const p = status(i).plan; assert.deepEqual(kinds(p), ["build-site _all", "deploy-site alpha"]); assert.equal(p.steps[0].runner, "docker"); assert.equal(p.steps[1].builtAfter, NOW); }); test("plan: the hub and the homepage follow settings.publish's policies", () => { const i = inputs( { ingestByChannel: { "shared-ch": NOW - 10 * MIN }, index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null }, }, { hub: "production", homepage: "build" }, ); const p = status(i).plan; assert.deepEqual(kinds(p), [ "update-index _index", "build-hub _hub", "deploy-hub _hub", "build-homepage _homepage", ]); assert.equal(p.steps[1].indexAfter, NOW); assert.equal(p.steps[2].builtAfter, NOW); // Off (the default): nothing. assert.deepEqual(kinds(status(inputs({ index: i.index, ingestByChannel: i.ingestByChannel })).plan), [ "update-index _index", ]); // The homepage is stale when the index moved past its build. const moved = inputs({ index: { stamp: stamp({ stampId: "stamp-2" }), lastIngestDoneAt: null, configChangedAt: null } }, { homepage: "build" }); assert.deepEqual(kinds(status(moved).plan), ["build-homepage _homepage"]); }); test("plan: never forced, and the lane's exclusions and skipIndex leave steps out", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN }, index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null }, }); i.sites[0].policy = "production"; const s = status(i); for (const step of s.plan.steps) assert.equal(step.force, undefined, `${step.kind} is not forced`); const exclude = new Set([stepKey({ kind: "build-site", target: "alpha" })]); assert.deepEqual(kinds(planPublishRun(s, { exclude, skipIndex: true })), ["deploy-site alpha"]); assert.equal(stepKey({ kind: "deploy-site", target: "a", preview: "p" }), "deploy-site:a:preview:p"); assert.equal(stepKey({ kind: "deploy-site", target: "a" }), "deploy-site:a:production"); assert.equal(stepKey({ kind: "deploy-site", target: "a", to: "local" }), "deploy-site:a:local"); // runStart is the caller's when given. assert.equal(planPublishRun(s, { runStart: 5 }).steps[1].indexAfter, 5); }); test("every plan step names a real stage and passes the stage's own argv parser", () => { const i = inputs({ ingestByChannel: { "alpha-ch": NOW - 10 * MIN, "shared-ch": NOW - 10 * MIN }, index: { stamp: stamp(), lastIngestDoneAt: NOW - 10 * MIN, configChangedAt: null }, }, { hub: "preview", homepage: "production" }); i.sites[0].policy = "preview"; for (const step of status(i).plan.steps) { const stage = STAGES[step.kind]; assert.ok(stage, step.kind); const argv = stage.argv({ ...step, runId: "r1" }); assert.equal(argv[0], "stage"); assert.equal(argv[1], step.kind); } }); // --- the lane's pass decision ------------------------------------------------------- test("pass decision: off, held, quiet hours and busy never run", () => { const base = { settings: { ...defaultPublish(), enabled: true }, stamp: null, indexFresh: false, busy: false, now: NOW, lastPassAt: null, policyWork: false, }; assert.deepEqual(publishPassDecision(base), { run: true, reason: "no index stamp yet" }); assert.equal(publishPassDecision({ ...base, settings: defaultPublish() }).reason, "the lane is off"); assert.equal(publishPassDecision({ ...base, settings: { ...base.settings, held: true } }).reason, "held"); assert.equal(publishPassDecision({ ...base, busy: true }).run, false); const hour = new Date(NOW).getHours(); const quiet = { start: hour, end: (hour + 1) % 24 }; assert.equal(publishPassDecision({ ...base, settings: { ...base.settings, quietHours: quiet } }).reason, "quiet hours"); const notQuiet = { start: (hour + 1) % 24, end: (hour + 2) % 24 }; assert.equal(publishPassDecision({ ...base, settings: { ...base.settings, quietHours: notQuiet } }).run, true); }); test("pass decision: a stale index waits out refreshEveryMinutes from the stamp's builtAt", () => { const settings = { ...defaultPublish(), enabled: true, refreshEveryMinutes: 60 }; const a = { settings, indexFresh: false, busy: false, now: NOW, lastPassAt: null, policyWork: false }; assert.equal(publishPassDecision({ ...a, stamp: stamp({ builtAt: NOW - 61 * MIN }) }).run, true); const early = publishPassDecision({ ...a, stamp: stamp({ builtAt: NOW - 30 * MIN }) }); assert.equal(early.run, false); assert.match(early.reason, /the index is stale; the next update is due 30 min from now/); // 0 = whenever stale. assert.equal( publishPassDecision({ ...a, settings: { ...settings, refreshEveryMinutes: 0 }, stamp: stamp({ builtAt: NOW }) }).run, true, ); // A fresh index with nothing to do never runs. assert.equal(publishPassDecision({ ...a, indexFresh: true, stamp: stamp() }).run, false); // Policy work left: once per interval since the last pass. assert.equal(publishPassDecision({ ...a, indexFresh: true, stamp: stamp(), policyWork: true }).run, true); assert.equal( publishPassDecision({ ...a, indexFresh: true, stamp: stamp(), policyWork: true, lastPassAt: NOW - 10 * MIN }).run, false, ); }); test("the status carries the lane's words and the plan Publish now would run", () => { const s = status(inputs({}, { enabled: true })); assert.equal(s.lane.enabled, true); assert.equal(s.lane.due, false); assert.match(s.lane.reason, /nothing to do/); assert.equal(s.lane.chip.text, "lane on, runner not running"); const off = status(inputs()); assert.equal(off.lane.chip.text, "lane off"); assert.equal(off.lane.blockedReason, "the lane is off"); assert.deepEqual(off.plan.steps, []); });