commit 3e1db6bda4556127cb4887572046cf23e919f79f parent 4572213d70dd41e717892d9cbc4c35c58154afe4 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Thu, 24 Sep 2026 12:54:21 -0400 lib: one JSON reader and one per-path-chained atomic JSON writer common/lib/jsonFile-server.ts: readJsonFile (+sync twin) answers {ok, value} | {ok:false, reason: absent|unreadable|unparseable}; writeJsonAtomic writes tmp + rename with a UNIQUE temp name (`${file}.tmp-${pid}-${seq}`) and chains writes per absolute path, so two same-process writers serialise instead of sharing one temp file. writeJsonAtomicSync for the three synchronous stores. Folded: the seven private writeJsonAtomic copies (digest-server, chartsStore, aliasesStore, curatedTagsStore, buildIndex, buildStats, compose-homepage) and the inline tmp writes in site.ts, settings.ts, controller/channels.ts, runYtdlp.ts, channelSnapshot.ts, the seven single-file sidecar servers, and the write halves of savedVideo-server, posts-server and clipWindow-server. getSettings reads through readJsonFileSync. Every file keeps its historical bytes (indent and trailing newline are per-caller options). Pure refactor. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Diffstat:
24 files changed, 350 insertions(+), 106 deletions(-)
diff --git a/common/bin/compose-homepage.ts b/common/bin/compose-homepage.ts @@ -12,8 +12,9 @@ // prebuild chains it), same as the export pipeline. import path from "node:path"; -import { mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { mkdir, readFile } from "node:fs/promises"; import { getPaths } from "../lib/paths"; +import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server"; import { buildStats } from "../controller/buildStats"; import { listSites } from "../lib/site"; import { @@ -35,10 +36,9 @@ function homepagePublicDir(monorepoRoot: string): string { ); } -async function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { - const tmp = `${filePath}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(value)); - await rename(tmp, filePath); +// Compact, no trailing newline — the homepage files' historical bytes. +function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { + return writeJsonAtomicShared(filePath, value, { indent: 0, newline: false }); } // Read the whole-pool stats dataset back from the pages buildStats just wrote, so diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts @@ -24,6 +24,7 @@ import { createWriteStream } from "node:fs"; import type { Dirent, WriteStream } from "node:fs"; import { once } from "node:events"; import { open } from "lmdb"; +import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server"; import { parseVtt, type Cue } from "../lib/vtt"; import { parseTranscriptJson } from "../lib/whisper"; import { parseLiveChat } from "../lib/liveChat"; @@ -410,13 +411,9 @@ function indexKeysEqual(a: IndexKey, b: IndexKey): boolean { return a[0] === b[0] && a[1] === b[1] && a[2] === b[2]; } -async function writeJsonAtomic( - filePath: string, - value: unknown, -): Promise<void> { - const tmp = `${filePath}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(value)); - await rename(tmp, filePath); +// Compact, no trailing newline — the export pages' historical bytes. +function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { + return writeJsonAtomicShared(filePath, value, { indent: 0, newline: false }); } export type BuildIndexResult = { diff --git a/common/controller/buildStats.ts b/common/controller/buildStats.ts @@ -23,6 +23,7 @@ import { } from "node:fs/promises"; import type { Dirent } from "node:fs"; import { open } from "lmdb"; +import { writeJsonAtomic as writeJsonAtomicShared } from "../lib/jsonFile-server"; import type { Cue } from "../lib/vtt"; import { transcriptCoverage } from "../lib/transcriptCoverage"; import { @@ -243,10 +244,9 @@ async function writePages( return { pageCount: pages.length, pagesWritten: pages.length }; } -async function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { - const tmp = `${filePath}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(value)); - await rename(tmp, filePath); +// Compact, no trailing newline — the stats manifests' historical bytes. +function writeJsonAtomic(filePath: string, value: unknown): Promise<void> { + return writeJsonAtomicShared(filePath, value, { indent: 0, newline: false }); } export async function buildStats({ diff --git a/common/controller/channelSnapshot.ts b/common/controller/channelSnapshot.ts @@ -1,6 +1,7 @@ +import { writeJsonAtomic } from "../lib/jsonFile-server"; import path from "node:path"; import type { Dirent } from "node:fs"; -import { readdir, readFile, rename, stat, writeFile } from "node:fs/promises"; +import { readdir, readFile, stat } from "node:fs/promises"; import pLimit from "p-limit"; import { readArchive } from "../lib/archive"; import { @@ -1621,7 +1622,5 @@ async function writeChannelSnapshot( snapshot: ChannelSnapshot, ): Promise<void> { const file = snapshotPath(paths, slug); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(snapshot, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, snapshot); } diff --git a/common/controller/channels.ts b/common/controller/channels.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "../lib/jsonFile-server"; import path from "node:path"; -import { readdir, readFile, writeFile, rename, rm, stat, mkdir } from "node:fs/promises"; +import { readdir, readFile, rm, stat, mkdir } from "node:fs/promises"; import type { Dirent } from "node:fs"; import { parseChannelConfig, @@ -458,9 +459,7 @@ export async function writeChannelConfig( const dir = path.join(paths.channelsDir, slug); await mkdir(dir, { recursive: true }); const file = path.join(dir, "config.json"); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(config, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, config); } export async function createChannel( diff --git a/common/lib/aliasesStore.ts b/common/lib/aliasesStore.ts @@ -7,8 +7,8 @@ // ships useful suggestions out of the box; a missing per-site file = no // overrides. See common/lib/searchAliases.ts for the pure model + merge. -import path from "node:path"; -import { readFileSync, writeFileSync, renameSync, mkdirSync } from "node:fs"; +import { readFileSync } from "node:fs"; +import { writeJsonAtomicSync } from "./jsonFile-server"; import type { Paths } from "./paths"; import { siteAliasesFile } from "./site"; import { @@ -19,11 +19,9 @@ import { type SearchAlias, } from "./searchAliases"; +// Indented, no trailing newline — these files' historical bytes. function writeJsonAtomic(filePath: string, value: unknown): void { - mkdirSync(path.dirname(filePath), { recursive: true }); - const tmp = `${filePath}.tmp-${process.pid}`; - writeFileSync(tmp, JSON.stringify(value, null, 2)); - renameSync(tmp, filePath); + writeJsonAtomicSync(filePath, value, { newline: false, mkdir: true }); } // Global dictionary. Absent/unreadable → the seeded defaults (a fresh install diff --git a/common/lib/attribution-server.ts b/common/lib/attribution-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { ATTRIBUTION_FILENAME, type AttributionRecord } from "./attribution"; export function attributionPath(videoDir: string): string { @@ -42,7 +43,5 @@ export async function writeAttribution( record: AttributionRecord, ): Promise<void> { const file = attributionPath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record, { indent: 0 }); } diff --git a/common/lib/availability-server.ts b/common/lib/availability-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { AVAILABILITY_FILENAME, AVAILABILITY_VALUES, @@ -43,9 +44,7 @@ export async function writeAvailability( record: AvailabilityRecord, ): Promise<void> { const file = availabilityPath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record); } // Append-on-change writer that all availability persistence funnels through. diff --git a/common/lib/chartsStore.ts b/common/lib/chartsStore.ts @@ -4,7 +4,8 @@ // bundle as /chart-templates.json for the static viewer to fetch. import path from "node:path"; -import { readFileSync, writeFileSync, renameSync, mkdirSync } from "node:fs"; +import { readFileSync } from "node:fs"; +import { writeJsonAtomicSync } from "./jsonFile-server"; import type { Paths } from "./paths"; import { siteChartTemplatesFile, siteIndexDir } from "./site"; import { @@ -45,11 +46,9 @@ export function readTemplates(paths: Paths, siteId: string): ChartTemplates { } } +// Indented, no trailing newline — these files' historical bytes. function writeJsonAtomic(filePath: string, value: unknown): void { - mkdirSync(path.dirname(filePath), { recursive: true }); - const tmp = `${filePath}.tmp-${process.pid}`; - writeFileSync(tmp, JSON.stringify(value, null, 2)); - renameSync(tmp, filePath); + writeJsonAtomicSync(filePath, value, { newline: false, mkdir: true }); } export function writeTemplates( diff --git a/common/lib/clipWindow-server.ts b/common/lib/clipWindow-server.ts @@ -2,8 +2,9 @@ // layout and for why a `clips/` subdirectory is invisible to every video-dir // enumerator in the repo. +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { mkdir, readFile, readdir, rename, stat, writeFile } from "node:fs/promises"; +import { mkdir, readFile, readdir, stat } from "node:fs/promises"; import { clipsDirFor, clipWindowFile, @@ -94,9 +95,7 @@ export async function writeClipProvenance( const dir = clipsDirFor(videoDir); await mkdir(dir, { recursive: true }); const file = path.join(dir, clipWindowSidecar(from, to)); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(provenance, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, provenance); return file; } diff --git a/common/lib/curatedTagsStore.ts b/common/lib/curatedTagsStore.ts @@ -42,8 +42,8 @@ // touches. A per-video loop would be N read-modify-writes and could interleave // with another writer. -import path from "node:path"; -import { readFileSync, writeFileSync, renameSync, mkdirSync } from "node:fs"; +import { readFileSync } from "node:fs"; +import { writeJsonAtomicSync } from "./jsonFile-server"; import type { Paths } from "./paths"; import { siteTagsFile } from "./site"; import { @@ -57,11 +57,9 @@ import { type CuratedTagsConfig, } from "./curatedTags"; +// Indented, no trailing newline — these files' historical bytes. function writeJsonAtomic(filePath: string, value: unknown): void { - mkdirSync(path.dirname(filePath), { recursive: true }); - const tmp = `${filePath}.tmp-${process.pid}`; - writeFileSync(tmp, JSON.stringify(value, null, 2)); - renameSync(tmp, filePath); + writeJsonAtomicSync(filePath, value, { newline: false, mkdir: true }); } export function emptyTagsConfig(): CuratedTagsConfig { diff --git a/common/lib/diarization-server.ts b/common/lib/diarization-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { DIARIZATION_FILENAME, type DiarizationRecord, @@ -41,7 +42,5 @@ export async function writeDiarization( record: DiarizationRecord, ): Promise<void> { const file = diarizationPath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record, { indent: 0 }); } diff --git a/common/lib/digest-server.ts b/common/lib/digest-server.ts @@ -9,7 +9,8 @@ // never clobbers local chapters (and vice versa). import path from "node:path"; -import { readFile, rename, rm, writeFile } from "node:fs/promises"; +import { readFile, rm } from "node:fs/promises"; +import { writeJsonAtomic } from "./jsonFile-server"; import { DIGEST_FILENAME, DIGEST_OVERRIDES_FILENAME, @@ -161,12 +162,6 @@ function sanitizeTags(value: unknown): DigestTag[] { // Writes // --------------------------------------------------------------------------- -async function writeJsonAtomic(file: string, value: unknown): Promise<void> { - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(value, null, 2) + "\n"); - await rename(tmp, file); -} - export async function writeDigest( videoDir: string, record: DigestRecord, diff --git a/common/lib/doNotClean-server.ts b/common/lib/doNotClean-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, rm, writeFile } from "node:fs/promises"; +import { readFile, rm } from "node:fs/promises"; import { DO_NOT_CLEAN_FILENAME, type DoNotCleanRecord, @@ -46,7 +47,5 @@ export async function setDoNotClean( setAt: new Date().toISOString(), ...(note ? { note } : {}), }; - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record); } diff --git a/common/lib/downloadOutcome-server.ts b/common/lib/downloadOutcome-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { DOWNLOAD_OUTCOME_FILENAME, DOWNLOAD_OUTCOME_STATUS_VALUES, @@ -39,9 +40,7 @@ export async function writeDownloadOutcome( record: DownloadOutcomeRecord, ): Promise<void> { const file = downloadOutcomePath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record); } // Narrow re-export so callers don't have to import from both modules. diff --git a/common/lib/excludeTruncatedCheck-server.ts b/common/lib/excludeTruncatedCheck-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, rm, writeFile } from "node:fs/promises"; +import { readFile, rm } from "node:fs/promises"; import { EXCLUDE_TRUNCATED_CHECK_FILENAME, type ExcludeTruncatedCheckRecord, @@ -47,7 +48,5 @@ export async function setExcludedFromTruncatedCheck( setAt: new Date().toISOString(), ...(note ? { note } : {}), }; - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record); } diff --git a/common/lib/jsonFile-server.test.ts b/common/lib/jsonFile-server.test.ts @@ -0,0 +1,103 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import { mkdtemp, readFile, readdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { + jsonText, + pendingJsonWrites, + readJsonFile, + readJsonFileSync, + tmpPathFor, + writeJsonAtomic, + writeJsonAtomicSync, +} from "./jsonFile-server"; + +async function scratch(): Promise<string> { + return mkdtemp(path.join(os.tmpdir(), "jsonfile-")); +} + +test("readJsonFile: ok, absent, unparseable, unreadable — and the sync twin agrees", async () => { + const dir = await scratch(); + const good = path.join(dir, "good.json"); + const bad = path.join(dir, "bad.json"); + await writeFile(good, '{"a":1}'); + await writeFile(bad, '{"a":'); + const cases: Array<[string, unknown]> = [ + [good, { ok: true, value: { a: 1 } }], + [path.join(dir, "missing.json"), { ok: false, reason: "absent" }], + [path.join(dir, "no-such-dir", "x.json"), { ok: false, reason: "absent" }], + [bad, { ok: false, reason: "unparseable" }], + // A directory is readable as a path but not as a file: EISDIR. + [dir, { ok: false, reason: "unreadable" }], + ]; + for (const [file, want] of cases) { + assert.deepEqual(await readJsonFile(file), want, file); + assert.deepEqual(readJsonFileSync(file), want, file); + } +}); + +test("jsonText: indent 2 vs 0, newline on by default", () => { + const v = { a: [1, 2] }; + assert.equal(jsonText(v), '{\n "a": [\n 1,\n 2\n ]\n}\n'); + assert.equal(jsonText(v, { indent: 0 }), '{"a":[1,2]}\n'); + assert.equal(jsonText(v, { indent: 0, newline: false }), '{"a":[1,2]}'); + assert.equal(jsonText(v, { newline: false }), '{\n "a": [\n 1,\n 2\n ]\n}'); +}); + +test("writeJsonAtomic: the bytes on disk are jsonText's, per option", async () => { + const dir = await scratch(); + for (const opts of [{}, { indent: 0 as const }, { indent: 0 as const, newline: false }]) { + const file = path.join(dir, `f-${JSON.stringify(opts)}.json`); + await writeJsonAtomic(file, { x: "y" }, opts); + assert.equal(await readFile(file, "utf8"), jsonText({ x: "y" }, opts)); + } +}); + +test("temp names are unique per write, not per process", () => { + const a = tmpPathFor("/x/config.json"); + const b = tmpPathFor("/x/config.json"); + assert.notEqual(a, b); + assert.ok(a.startsWith(`/x/config.json.tmp-${process.pid}-`)); +}); + +test("two same-path writers serialise: the last ISSUED lands, no temp is left, no write fails", async () => { + const dir = await scratch(); + const file = path.join(dir, "config.json"); + const writes = []; + for (let i = 0; i < 50; i++) writes.push(writeJsonAtomic(file, { i })); + await Promise.all(writes); + assert.deepEqual(JSON.parse(await readFile(file, "utf8")), { i: 49 }); + assert.deepEqual(await readdir(dir), ["config.json"]); + assert.equal(pendingJsonWrites(), 0); +}); + +test("the value is serialised at the call, not when the chain reaches it", async () => { + const dir = await scratch(); + const file = path.join(dir, "v.json"); + const v = { n: 1 }; + const first = writeJsonAtomic(file, { hold: true }); + const second = writeJsonAtomic(file, v); + v.n = 2; + await Promise.all([first, second]); + assert.deepEqual(JSON.parse(await readFile(file, "utf8")), { n: 1 }); +}); + +test("a failed write does not block the next one on the same path, and cleans its temp", async () => { + const dir = await scratch(); + const file = path.join(dir, "sub", "x.json"); + // No parent dir and no mkdir: this one fails. + await assert.rejects(writeJsonAtomic(file, { a: 1 })); + await writeJsonAtomic(file, { a: 2 }, { mkdir: true }); + assert.deepEqual(JSON.parse(await readFile(file, "utf8")), { a: 2 }); + assert.deepEqual(await readdir(path.dirname(file)), ["x.json"]); +}); + +test("writeJsonAtomicSync: same bytes, mkdir, no temp left", async () => { + const dir = await scratch(); + const file = path.join(dir, "a", "b.json"); + writeJsonAtomicSync(file, { k: 1 }, { newline: false, mkdir: true }); + assert.equal(fs.readFileSync(file, "utf8"), '{\n "k": 1\n}'); + assert.deepEqual(await readdir(path.dirname(file)), ["b.json"]); +}); diff --git a/common/lib/jsonFile-server.ts b/common/lib/jsonFile-server.ts @@ -0,0 +1,176 @@ +// ONE JSON READER AND ONE ATOMIC JSON WRITER for every config file and sidecar. +// +// one-core phase 3 slice 4b. Before this module the repo had seven private +// copies of `writeJsonAtomic` and some twenty inline `tmp + rename` writes, all +// of them spelling the temp file `${file}.tmp-${process.pid}`. That name is the +// same for every write this process makes to one file, so two writers in the +// same process — a sync stamping `lastSyncedAt` while the channel form saves +// `config.json` — shared one temp file: the second `writeFile` truncated the +// first's bytes and one `rename` then found no file. This module fixes both +// halves of that: +// +// - every temp name is unique (`${file}.tmp-${pid}-${seq}`), and +// - writes to one ABSOLUTE PATH are CHAINED: a write starts only once the +// previous write to that path has settled, so two same-process writers +// serialise and the last one to be ISSUED is the one on disk. +// +// The chain is per process. A CLI run beside a live editor (e.g. +// `migrate-channel-priority.ts`) is a different process and is NOT covered — +// the rename is still atomic, so a reader never sees a torn file, but the +// last rename wins. +// +// The chain serialises WRITES, not read-modify-write cycles. A caller that reads, +// changes and writes back has to hold its read inside the same critical section +// for the read to be current — see `patchChannelConfig` in +// controller/channels.ts, which re-reads immediately before its write. +// +// BYTES ARE PRESERVED, per caller. `indent` and `newline` are options, not a +// house style, because the bytes on disk are what the phase-3 numbers diff: the +// attribution and diarization sidecars are compact with a trailing newline, +// the export build's pages are compact without one, the chart/alias/tag stores +// are indented without one, and everything else is indented with one. +// +// SERVER-ONLY (node:fs). Named `-server` so a `"use client"` graph never +// reaches it. + +import fs from "node:fs"; +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; + +export type ReadJsonResult = + | { ok: true; value: unknown } + | { ok: false; reason: "absent" | "unreadable" | "unparseable" }; + +function readFailure(err: unknown): ReadJsonResult { + const code = (err as NodeJS.ErrnoException | null)?.code; + return { + ok: false, + reason: code === "ENOENT" || code === "ENOTDIR" ? "absent" : "unreadable", + }; +} + +function parseText(text: string): ReadJsonResult { + try { + return { ok: true, value: JSON.parse(text) as unknown }; + } catch { + return { ok: false, reason: "unparseable" }; + } +} + +// The file as JSON. Never throws: `absent` is a missing file (or a missing +// parent), `unreadable` any other read error (EACCES, EISDIR, EIO, …), +// `unparseable` a file that is not JSON — which includes a truncated one. +export async function readJsonFile(file: string): Promise<ReadJsonResult> { + let text: string; + try { + text = await readFile(file, "utf8"); + } catch (err) { + return readFailure(err); + } + return parseText(text); +} + +// The synchronous twin, for the two readers that are synchronous by contract: +// `getSettings` (lib/settings.ts) and `getSite` (lib/site.ts). +export function readJsonFileSync(file: string): ReadJsonResult { + let text: string; + try { + text = fs.readFileSync(file, "utf8"); + } catch (err) { + return readFailure(err); + } + return parseText(text); +} + +export type WriteJsonOptions = { + // 2 (the default) = `JSON.stringify(value, null, 2)`; 0 = compact. + indent?: 0 | 2; + // Append "\n". Defaults to true — the common case; pass false to keep a + // file's historical bytes. + newline?: boolean; + // Create the parent directory first (`mkdir -p`). + mkdir?: boolean; +}; + +// Exactly the bytes a write puts on disk. +export function jsonText(value: unknown, opts: WriteJsonOptions = {}): string { + const indent = opts.indent ?? 2; + const body = + indent === 0 ? JSON.stringify(value) : JSON.stringify(value, null, indent); + return opts.newline === false ? body : body + "\n"; +} + +let tmpSeq = 0; + +// Unique per write, not per process: `${file}.tmp-${pid}-${seq}`. +export function tmpPathFor(file: string): string { + return `${file}.tmp-${process.pid}-${++tmpSeq}`; +} + +const chains = new Map<string, Promise<void>>(); + +async function writeNow( + file: string, + text: string, + makeDir: boolean, +): Promise<void> { + if (makeDir) await mkdir(path.dirname(file), { recursive: true }); + const tmp = tmpPathFor(file); + try { + await writeFile(tmp, text); + await rename(tmp, file); + } catch (err) { + await rm(tmp, { force: true }).catch(() => {}); + throw err; + } +} + +// Write `value` as JSON to `file` atomically (tmp + rename), after every write +// to the same absolute path this process issued earlier has settled. The value +// is serialised NOW, at the call, so a caller that goes on mutating its object +// cannot change what lands. A failed earlier write does not block a later one; +// each caller sees only its own write's error. +export function writeJsonAtomic( + file: string, + value: unknown, + opts: WriteJsonOptions = {}, +): Promise<void> { + const key = path.resolve(file); + const text = jsonText(value, opts); + const prev = chains.get(key) ?? Promise.resolve(); + const next = prev.then( + () => writeNow(key, text, opts.mkdir === true), + () => writeNow(key, text, opts.mkdir === true), + ); + chains.set(key, next); + const release = () => { + if (chains.get(key) === next) chains.delete(key); + }; + next.then(release, release); + return next; +} + +// The synchronous twin, for the three stores whose API is synchronous +// (chartsStore, aliasesStore, curatedTagsStore). A synchronous write cannot +// wait on the async chain, and needs no chain of its own: nothing else in this +// process runs while it does. It shares the unique temp name. +export function writeJsonAtomicSync( + file: string, + value: unknown, + opts: WriteJsonOptions = {}, +): void { + if (opts.mkdir === true) fs.mkdirSync(path.dirname(file), { recursive: true }); + const tmp = tmpPathFor(file); + try { + fs.writeFileSync(tmp, jsonText(value, opts)); + fs.renameSync(tmp, file); + } catch (err) { + fs.rmSync(tmp, { force: true }); + throw err; + } +} + +// For tests: how many paths currently have a write in flight. +export function pendingJsonWrites(): number { + return chains.size; +} diff --git a/common/lib/posts-server.ts b/common/lib/posts-server.ts @@ -11,8 +11,9 @@ // fetchers stop the same way), and uses the same "<extractor> <id>" line format // so common/lib/archive.ts parses it unchanged. +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readdir, mkdir, readFile, rename, writeFile } from "node:fs/promises"; +import { readdir, mkdir, readFile } from "node:fs/promises"; import { appendFile } from "node:fs/promises"; import { readArchive } from "./archive"; import { @@ -256,9 +257,7 @@ export async function writePostAvailability( ): Promise<void> { const file = postsAvailabilityPath(channelRoot); await mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(map, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, map); } // Fold new observations in, appending to history ONLY when the availability @@ -339,7 +338,5 @@ export async function writePostFetchState( ): Promise<void> { const file = postFetchStatePath(channelRoot); await mkdir(path.dirname(file), { recursive: true }); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(state, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, state); } diff --git a/common/lib/savedVideo-server.ts b/common/lib/savedVideo-server.ts @@ -1,3 +1,4 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; import { copyFile, @@ -6,7 +7,6 @@ import { rename, rm, stat, - writeFile, } from "node:fs/promises"; import { SAVED_VIDEO_POINTER_FILENAME, @@ -45,9 +45,7 @@ async function writePointer( pointer: SavedVideoPointer, ): Promise<void> { const file = savedVideoPointerPath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(pointer, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, pointer); } // Repoint a video's saved-video.json at a new absolute store dir. Used when a diff --git a/common/lib/settings.ts b/common/lib/settings.ts @@ -22,7 +22,7 @@ // One schema, both directions: the only differences between what a read and a // write produce are those migrations and those two validators. -import fs from "node:fs"; +import { readJsonFileSync, writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; import { getPaths } from "./paths"; import { @@ -56,11 +56,8 @@ type RawSettings = Record<string, unknown>; // The file as JSON, or `undefined` when it is missing or not JSON. Never // throws: a settings read is on every request path. function readRawSettings(file: string): unknown { - try { - return JSON.parse(fs.readFileSync(file, "utf8")); - } catch { - return undefined; - } + const read = readJsonFileSync(file); + return read.ok ? read.value : undefined; } // Only a plain object is a settings file. `null`, `[]`, `3` and a truncated @@ -244,7 +241,5 @@ export async function writeSettings(next: SiteSettings): Promise<void> { socialLinks: validatedSocialLinks(next.socialLinks), }); const file = getPaths().settingsFile; - const tmp = `${file}.tmp-${process.pid}`; - await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2) + "\n"); - await fs.promises.rename(tmp, file); + await writeJsonAtomic(file, merged); } diff --git a/common/lib/site.ts b/common/lib/site.ts @@ -1,3 +1,4 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import fs from "node:fs"; import path from "node:path"; import { @@ -494,9 +495,7 @@ export async function writeSite( const dir = siteDir(paths, site.siteId); await fs.promises.mkdir(dir, { recursive: true }); const file = siteConfigFile(paths, site.siteId); - const tmp = `${file}.tmp-${process.pid}`; - await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2) + "\n"); - await fs.promises.rename(tmp, file); + await writeJsonAtomic(file, merged); } export async function deleteSite( diff --git a/common/lib/transcribeOutcome-server.ts b/common/lib/transcribeOutcome-server.ts @@ -1,5 +1,6 @@ +import { writeJsonAtomic } from "./jsonFile-server"; import path from "node:path"; -import { readFile, rename, writeFile } from "node:fs/promises"; +import { readFile } from "node:fs/promises"; import { TRANSCRIBE_OUTCOME_FILENAME, type TranscribeOutcomeRecord, @@ -32,7 +33,5 @@ export async function writeTranscribeOutcome( record: TranscribeOutcomeRecord, ): Promise<void> { const file = transcribeOutcomePath(videoDir); - const tmp = `${file}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(record, null, 2) + "\n"); - await rename(tmp, file); + await writeJsonAtomic(file, record); } diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts @@ -1,3 +1,4 @@ +import { writeJsonAtomic } from "../lib/jsonFile-server"; import path from "node:path"; import { mkdir, readdir, readFile, rename, writeFile } from "node:fs/promises"; import { execa } from "execa"; @@ -1838,9 +1839,7 @@ async function updateConfigField( const parsed = parseChannelConfig(JSON.parse(raw)); if (!parsed) return; parsed[field] = value; - const tmp = `${configPath}.tmp-${process.pid}`; - await writeFile(tmp, JSON.stringify(parsed, null, 2) + "\n"); - await rename(tmp, configPath); + await writeJsonAtomic(configPath, parsed); } // The channel's archive file stores yt-dlp's native extractor ids (e.g.