// THE LIVE CHECK (release 18): after every deploy, read what the URL actually // serves. // // A deploy that exits 0 says wrangler uploaded a bundle; it does not say a // visitor gets it. Cloudflare's edge keeps serving a cached object after a // deploy that changed or removed it (the hub served a withdrawn X shard from a // 7-day cache after the deploy that took it out), and a deploy can land on a // preview when production was meant. So the deploy stage reads // `/corpus.json` twice — PLAIN, as a visitor would, and CACHE-BUSTED // (`?cb=`), which the edge has never seen and so fetches from the // deployment — and compares each `generatedAt` with the build's own // (`built.corpusGeneratedAt`): // // ok both serve this build (or the plain one does and the busted // read failed: a visitor gets this build) // stale-edge the busted read serves this build, the plain one answers 2xx // with something else: the deployment is right and the edge // still has the old object // mismatch the busted read serves something else: not this build // unreachable neither read answered 2xx, or the plain read failed (a // visitor gets nothing, whatever the deployment holds) // skipped E2E_LIVE_CHECK=skip (the e2e suite, whose fake wrangler // deploys nothing) // // stale-edge and mismatch are WARNINGS: the deploy itself succeeded and the job // ends `done`; the verdict is recorded in deployed.json and said in the log. // Three tries, 10 s apart, before a verdict short of ok stands — a fresh // deployment takes a few seconds to answer everywhere. // // The hub's deploy also probes its TOMBSTONES (publish/tombstones.ts): each // path must answer, plain and busted, with the empty object that replaced the // withdrawn content. // // Everything that touches the network or the clock is injected: `fetch`, // `sleep`, `now`, `env`. // The record shapes are S1's (publish/stamps.ts): `at` is ms, `age` seconds. import type { LiveCheck, Probe } from "./stamps"; export type { LiveCheck, Probe } from "./stamps"; export type LiveCheckVerdict = LiveCheck["verdict"]; export type TombstoneProbe = NonNullable[number]; export type LiveCheckDeps = { // The stage's Cancel: stops between reads and tries, aborts a read in flight. signal?: AbortSignal; fetch?: typeof fetch; sleep?: (ms: number) => Promise; now?: () => Date; env?: Record; tries?: number; intervalMs?: number; timeoutMs?: number; }; export const LIVE_CHECK_TRIES = 3; export const LIVE_CHECK_INTERVAL_MS = 10_000; const LIVE_CHECK_TIMEOUT_MS = 15_000; /** The root-relative URL of `rel` under `base` (a site URL, alias or deployment URL). */ export function liveUrl(base: string, rel: string): string { return `${base.replace(/\/+$/, "")}/${rel.replace(/^\/+/, "")}`; } /** The cache-busted form of `url`: a query the edge has never seen. */ export function cacheBusted(url: string, builtStampId: string, attempt = 1): string { const cb = encodeURIComponent(builtStampId); return `${url}${url.includes("?") ? "&" : "?"}cb=${cb}${attempt > 1 ? `&try=${attempt}` : ""}`; } const reached = (p: Probe) => p.status !== null && p.status >= 200 && p.status < 300; /** * The verdict for one plain + busted pair against the build's `expected` * `generatedAt` (null: the build named none, so answering 2xx is all that is * asked). Pure. */ export function liveCheckVerdict( plain: Probe, busted: Probe, expected: string | null, ): Exclude { const serves = (p: Probe) => reached(p) && (expected === null || p.generatedAt === expected); if (!reached(plain) && !reached(busted)) return "unreachable"; if (serves(busted)) { if (serves(plain)) return "ok"; // Stale only when the edge ANSWERS with another object; a plain read that // fails is not staleness — a visitor gets nothing. return reached(plain) ? "stale-edge" : "unreachable"; } // The busted read failed but the plain one serves this build: a visitor // gets it, which is what the check is for. if (!reached(busted) && serves(plain)) return "ok"; return "mismatch"; } // Whether a parsed body is the tombstone that belongs at `rel`. function isTombstoneBody(rel: string, body: unknown): boolean { if (/\/page-\d+\.json$/.test(rel)) return Array.isArray(body) && body.length === 0; if (rel === "posts/manifest.json") { const ch = (body as { channels?: unknown } | null)?.channels; return Array.isArray(ch) && ch.length === 0; } return (body as { pageCount?: unknown } | null)?.pageCount === 0; } type Read = { probe: Probe; body: unknown }; async function read(url: string, f: typeof fetch, timeoutMs: number, cancel?: AbortSignal): Promise { try { const timeout = AbortSignal.timeout(timeoutMs); const signal = cancel ? AbortSignal.any([cancel, timeout]) : timeout; const res = await f(url, { redirect: "follow", signal }); const probe: Probe = { status: res.status }; const h = (name: string) => res.headers.get(name) ?? undefined; if (h("cf-cache-status")) probe.cfCacheStatus = h("cf-cache-status"); const age = Number(h("age")); if (h("age") !== undefined && Number.isFinite(age)) probe.age = age; if (h("cache-control")) probe.cacheControl = h("cache-control"); let body: unknown; try { body = JSON.parse(await res.text()); } catch { body = undefined; } const generatedAt = (body as { generatedAt?: unknown } | undefined)?.generatedAt; if (typeof generatedAt === "string") probe.generatedAt = generatedAt; return { probe, body }; } catch (err) { return { probe: { status: null, error: err instanceof Error ? err.message : String(err) }, body: undefined, }; } } /** * Read `/` (default `corpus.json`) plain and cache-busted, and each * of `tombstones` the same way; retry up to `tries` times, `intervalMs` apart, * until everything reads ok. Never throws. A `signal` that aborts stops it * between reads and tries (the result then carries what was read, or an * `unreachable` naming the cancel); the caller decides what a cancelled check * means. */ export async function runLiveCheck( opts: { url: string; builtStampId: string; expected: string | null; path?: string; tombstones?: readonly string[]; }, deps: LiveCheckDeps = {}, ): Promise { const now = deps.now ?? (() => new Date()); const env = deps.env ?? process.env; const base: Omit = { at: now().getTime(), url: opts.url, expected: opts.expected, }; if (env.E2E_LIVE_CHECK === "skip") { return { ...base, plain: { status: null }, busted: { status: null }, verdict: "skipped" }; } const f = deps.fetch ?? fetch; const cancel = deps.signal; const sleep = deps.sleep ?? ((ms: number) => new Promise((resolve) => { const t = setTimeout(done, ms); function done() { clearTimeout(t); cancel?.removeEventListener("abort", done); resolve(); } cancel?.addEventListener("abort", done, { once: true }); })); const tries = Math.max(1, deps.tries ?? LIVE_CHECK_TRIES); const interval = deps.intervalMs ?? LIVE_CHECK_INTERVAL_MS; const timeout = deps.timeoutMs ?? LIVE_CHECK_TIMEOUT_MS; const target = liveUrl(opts.url, opts.path ?? "corpus.json"); let check: LiveCheck | null = null; for (let attempt = 1; attempt <= tries; attempt++) { if (attempt > 1) await sleep(interval); if (cancel?.aborted) break; const plain = (await read(target, f, timeout, cancel)).probe; const busted = (await read(cacheBusted(target, opts.builtStampId, attempt), f, timeout, cancel)).probe; let verdict: LiveCheckVerdict = liveCheckVerdict(plain, busted, opts.expected); let tombstones: TombstoneProbe[] | undefined; if (opts.tombstones && opts.tombstones.length > 0) { tombstones = []; let staleOnly = true; for (const rel of opts.tombstones) { if (cancel?.aborted) break; const u = liveUrl(opts.url, rel); const p = await read(u, f, timeout, cancel); const b = await read(cacheBusted(u, opts.builtStampId, attempt), f, timeout, cancel); const pOk = reached(p.probe) && isTombstoneBody(rel, p.body); const bOk = reached(b.probe) && isTombstoneBody(rel, b.body); tombstones.push({ path: rel, plain: p.probe, busted: b.probe, ok: pOk && bOk }); if (!(pOk && bOk) && !bOk) staleOnly = false; } // The corpus reads ok but a tombstone does not: the edge still serves // the withdrawn object (stale-edge) — or the deployment never had it. if (verdict === "ok" && tombstones.some((t) => !t.ok)) { verdict = staleOnly ? "stale-edge" : "mismatch"; } } check = { ...base, plain, busted, verdict, ...(tombstones ? { tombstones } : {}) }; if (verdict === "ok" || cancel?.aborted) break; } return ( check ?? { ...base, plain: { status: null, error: "cancelled" }, busted: { status: null, error: "cancelled" }, verdict: "unreachable", } ); } function describe(p: Probe): string { if (p.status === null) return p.error ? `no answer (${p.error})` : "no answer"; const parts = [`HTTP ${p.status}`]; if (p.generatedAt) parts.push(`generatedAt ${p.generatedAt}`); if (p.cfCacheStatus) parts.push(`cf-cache-status ${p.cfCacheStatus}`); if (p.age) parts.push(`age ${p.age}`); if (p.cacheControl) parts.push(`cache-control "${p.cacheControl}"`); return parts.join(", "); } /** * The log lines a live check ends a deploy on: one verdict line (WARNING for * anything short of ok but skipped), and one per tombstone that read wrong. */ export function liveCheckLines(check: LiveCheck): string[] { const where = check.url; const want = check.expected ? `this build (generatedAt ${check.expected})` : "this build"; switch (check.verdict) { case "skipped": return ["[live] skipped (E2E_LIVE_CHECK=skip)."]; case "ok": { const n = check.tombstones?.length ?? 0; return [ `[live] ok — ${where} serves ${want}; plain: ${describe(check.plain)}` + (n > 0 ? `; ${n} withdrawn path(s) read as tombstones.` : "."), ]; } case "unreachable": return [ `[live] WARNING unreachable — ${where} did not answer: plain ${describe(check.plain)}; ` + `cache-busted ${describe(check.busted)}. The deploy itself succeeded.`, ]; case "stale-edge": case "mismatch": { const bad = (check.tombstones ?? []).filter((t) => !t.ok); const corpusOk = liveCheckVerdict(check.plain, check.busted, check.expected) === "ok"; const lines = corpusOk ? [ `[live] WARNING ${check.verdict} — ${where} serves ${want}, but ${bad.length} withdrawn ` + `path(s) do not read as tombstones` + (check.verdict === "stale-edge" ? ": the deployment has them, Cloudflare's edge still serves the old objects until they expire." : ": the deployment does not serve them."), ] : check.verdict === "stale-edge" ? [ `[live] WARNING stale-edge — the deployment serves ${want}, but ${where} still answers ` + `with an older object from Cloudflare's edge: plain ${describe(check.plain)}; ` + `cache-busted ${describe(check.busted)}. It is replaced when the edge's copy expires.`, ] : [ `[live] WARNING mismatch — ${where} does not serve ${want}: plain ${describe(check.plain)}; ` + `cache-busted ${describe(check.busted)}.`, ]; for (const t of bad) { lines.push( `[live] tombstone ${t.path}: plain ${describe(t.plain)}; cache-busted ${describe(t.busted)}.`, ); } return lines; } } }