commit 80034b71d5c7757db39c71d9a80da52e89c7c4c2
parent 75ed0f0ec2260aba9c586216ce1eae68a1948c51
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 24 Sep 2026 18:56:36 -0400
common: process-global and per-video JSON writers on writeJsonAtomic
Scheduler state, worker defaults, widget presets, homepage.json, the
priority migration, relocation markers, shard configs, the duplicate and
media-scan reports and overrides, the saved-video backup manifest, both
cue normalizers, and the video page's remark-empty transcript.json. Bytes
per site as before: null, 2 + "\n" except the two reports and the two
cue files (compact, no newline) and transcript.json (compact + "\n", the
literal it replaced). mkdir kept exactly where a site had one. Tests pin
the four compact writers' bytes and the literal.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
15 files changed, 146 insertions(+), 72 deletions(-)
diff --git a/common/bin/migrate-channel-priority.ts b/common/bin/migrate-channel-priority.ts
@@ -1,6 +1,7 @@
#!/usr/bin/env tsx
import path from "node:path";
-import { readdir, readFile, rename, writeFile } from "node:fs/promises";
+import { readdir, readFile, writeFile } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { getPaths } from "../lib/paths";
import { getSettings, writeSettings } from "../lib/settings";
import { siteChannelIndex } from "../lib/site";
@@ -203,9 +204,7 @@ async function clearExcludeFromSync(slug: string): Promise<boolean> {
}
if (!("excludeFromSync" in parsed)) return false;
delete parsed.excludeFromSync;
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(parsed, null, 2) + "\n");
- await rename(tmp, file);
+ await writeJsonAtomic(file, parsed);
return true;
}
diff --git a/common/controller/backupSavedVideos.ts b/common/controller/backupSavedVideos.ts
@@ -1,7 +1,8 @@
import path from "node:path";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
-import { mkdir, readFile, readdir, rename, stat, writeFile } from "node:fs/promises";
+import { mkdir, readFile, readdir, stat } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { execa } from "execa";
import type { Paths } from "../lib/paths";
import {
@@ -115,9 +116,7 @@ export async function backupSavedVideos({
entries: manifestEntries,
};
const manifestPath = path.join(destRoot, BACKUP_MANIFEST_FILENAME);
- const tmp = `${manifestPath}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(manifest, null, 2) + "\n");
- await rename(tmp, manifestPath);
+ await writeJsonAtomic(manifestPath, manifest);
log(
`Backup complete: ${backedUp}/${saved.length} container(s), ${bytes} bytes. Manifest at ${manifestPath}`,
);
diff --git a/common/controller/compactJsonWriters.test.ts b/common/controller/compactJsonWriters.test.ts
@@ -0,0 +1,98 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import path from "node:path";
+import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import type { Paths } from "../lib/paths";
+import {
+ CUES_JSON_FILENAME,
+ LIVE_CHAT_CUES_FILENAME,
+ LIVE_CHAT_FILENAME,
+ META_FILENAME,
+ VTT_FILENAME,
+} from "../lib/videoStatus";
+import { DUPLICATES_FILENAME } from "../lib/duplicates";
+import { MEDIA_SCAN_FILENAME } from "../lib/mediaScan";
+import { normalizeTranscript } from "./normalizeTranscript";
+import { normalizeLiveChat } from "./normalizeLiveChat";
+import { detectDuplicateShorts } from "./duplicateShorts";
+import { scanCorruptMedia } from "./scanCorruptMedia";
+
+// THE FOUR COMPACT WRITERS keep their historical bytes after the fold onto
+// writeJsonAtomic (release 4 slice W): each wrote `JSON.stringify(x)` — no
+// indent, NO trailing newline — and a reader-side diff (the phase-3 numbers
+// tools, a site's published bytes) would move if that changed. Each test pins
+// that the file on disk is exactly `JSON.stringify` of its own parse.
+
+async function withDir(fn: (dir: string) => Promise<void>): Promise<void> {
+ const dir = await mkdtemp(path.join(tmpdir(), "compact-writers-"));
+ try {
+ await fn(dir);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+function assertCompact(text: string, what: string): void {
+ assert.ok(!text.endsWith("\n"), `${what}: no trailing newline`);
+ assert.equal(text, JSON.stringify(JSON.parse(text)), `${what}: compact`);
+}
+
+async function videoDir(dir: string): Promise<string> {
+ const v = path.join(dir, "vid1");
+ await mkdir(v, { recursive: true });
+ await writeFile(
+ path.join(v, META_FILENAME),
+ JSON.stringify({ id: "vid1", title: "A video", duration: 60 }),
+ );
+ return v;
+}
+
+test("normalizeTranscript writes transcript.cues.json compact, no newline", async () => {
+ await withDir(async (dir) => {
+ const v = await videoDir(dir);
+ await writeFile(
+ path.join(v, VTT_FILENAME),
+ "WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nhello\n",
+ );
+ const out = await normalizeTranscript({ videoDir: v, channelSlug: "c" });
+ assert.equal(out.status, "wrote");
+ assertCompact(await readFile(path.join(v, CUES_JSON_FILENAME), "utf8"), "cues");
+ });
+});
+
+test("normalizeLiveChat writes the live-chat cues compact, no newline", async () => {
+ await withDir(async (dir) => {
+ const v = await videoDir(dir);
+ await writeFile(path.join(v, LIVE_CHAT_FILENAME), "");
+ const out = await normalizeLiveChat({ videoDir: v, channelSlug: "c" });
+ assert.equal(out.status, "wrote");
+ assertCompact(
+ await readFile(path.join(v, LIVE_CHAT_CUES_FILENAME), "utf8"),
+ "live chat cues",
+ );
+ });
+});
+
+function emptyCorpus(dir: string): Paths {
+ return {
+ transcriptsDir: dir,
+ channelsDir: path.join(dir, "channels"),
+ } as Paths;
+}
+
+test("detectDuplicateShorts writes duplicates.json compact, no newline", async () => {
+ await withDir(async (dir) => {
+ await mkdir(path.join(dir, "channels"), { recursive: true });
+ await detectDuplicateShorts({ paths: emptyCorpus(dir), onLog: () => {} });
+ assertCompact(await readFile(path.join(dir, DUPLICATES_FILENAME), "utf8"), "duplicates");
+ });
+});
+
+test("scanCorruptMedia writes media-scan.json compact, no newline", async () => {
+ await withDir(async (dir) => {
+ await mkdir(path.join(dir, "channels"), { recursive: true });
+ await scanCorruptMedia({ paths: emptyCorpus(dir), onLog: () => {} });
+ assertCompact(await readFile(path.join(dir, MEDIA_SCAN_FILENAME), "utf8"), "media scan");
+ });
+});
diff --git a/common/controller/duplicateShorts.ts b/common/controller/duplicateShorts.ts
@@ -38,7 +38,8 @@
// global transcripts/duplicates.json. Flag-only: nothing is merged or deleted.
import path from "node:path";
-import { mkdir, rename, writeFile, readFile } from "node:fs/promises";
+import { readFile } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { createHash } from "node:crypto";
import { open } from "lmdb";
import pLimit from "p-limit";
@@ -636,10 +637,8 @@ export async function detectDuplicateShorts(
};
const outPath = path.join(opts.paths.transcriptsDir, DUPLICATES_FILENAME);
- await mkdir(path.dirname(outPath), { recursive: true });
- const tmp = `${outPath}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(report));
- await rename(tmp, outPath);
+ // Compact, no trailing newline: the report's historical bytes.
+ await writeJsonAtomic(outPath, report, { indent: 0, newline: false, mkdir: true });
noteRss();
const suspectClusters = clusters.filter((c) => c.needsReview).length;
log(
@@ -730,10 +729,7 @@ export async function updateDuplicateOverride(
}
const out = sanitizeDuplicateOverrides(current);
const file = duplicateOverridesPath(paths);
- await mkdir(path.dirname(file), { recursive: true });
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(out, null, 2) + "\n");
- await rename(tmp, file);
+ await writeJsonAtomic(file, out, { mkdir: true });
return out;
}
diff --git a/common/controller/normalizeLiveChat.ts b/common/controller/normalizeLiveChat.ts
@@ -4,7 +4,8 @@
// matches NormalizedTranscript with source: "live_chat".
import path from "node:path";
-import { readFile, rename, stat, writeFile } from "node:fs/promises";
+import { readFile, stat } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { parseLiveChat } from "../lib/liveChat";
import type { Cue } from "../lib/vtt";
import { summarize, type RawMetadata } from "../lib/transcripts-server";
@@ -95,9 +96,8 @@ export async function normalizeLiveChat(
cues,
};
- const tmp = `${cuesPath}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(out));
- await rename(tmp, cuesPath);
+ // Compact, no trailing newline: live_chat.cues.json's historical bytes.
+ await writeJsonAtomic(cuesPath, out, { indent: 0, newline: false });
opts.log?.(
`Normalized live chat ${opts.channelSlug}/${path.basename(opts.videoDir)} (${cues.length} cues)`,
);
diff --git a/common/controller/normalizeTranscript.ts b/common/controller/normalizeTranscript.ts
@@ -4,7 +4,8 @@
// shape that buildIndex would emit, plus a `source` marker.
import path from "node:path";
-import { readdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { parseVtt, type Cue } from "../lib/vtt";
import {
detectTranscriptFormat,
@@ -136,9 +137,8 @@ export async function normalizeTranscript(
cues,
};
- const tmp = `${cuesPath}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(out));
- await rename(tmp, cuesPath);
+ // Compact, no trailing newline: transcript.cues.json's historical bytes.
+ await writeJsonAtomic(cuesPath, out, { indent: 0, newline: false });
opts.log?.(
`Normalized ${opts.channelSlug}/${path.basename(opts.videoDir)} (${transcriptFormat}, ${cues.length} cues)`,
);
diff --git a/common/controller/relocateDir.ts b/common/controller/relocateDir.ts
@@ -4,11 +4,10 @@ import {
readdir,
readFile,
readlink,
- rename,
rm,
stat,
- writeFile,
} from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { execa } from "execa";
import {
formatRsyncProgressDetail,
@@ -136,9 +135,7 @@ export async function writeDirMarker(
file: string,
marker: DirRelocationMarker,
): Promise<void> {
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(marker, null, 2) + "\n");
- await rename(tmp, file);
+ await writeJsonAtomic(file, marker);
}
export async function clearDirMarker(file: string): Promise<void> {
diff --git a/common/controller/scanCorruptMedia.ts b/common/controller/scanCorruptMedia.ts
@@ -29,7 +29,8 @@
// first, delete second — not a dry-run flag on a deleting command.
import path from "node:path";
-import { mkdir, readdir, readFile, rename, stat, writeFile } from "node:fs/promises";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import { execa } from "execa";
import type { Paths } from "../lib/paths";
import {
@@ -389,10 +390,8 @@ async function mergeAndWriteReport(
),
};
const outPath = path.join(paths.transcriptsDir, MEDIA_SCAN_FILENAME);
- await mkdir(path.dirname(outPath), { recursive: true });
- const tmp = `${outPath}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(merged));
- await rename(tmp, outPath);
+ // Compact, no trailing newline: the report's historical bytes.
+ await writeJsonAtomic(outPath, merged, { indent: 0, newline: false, mkdir: true });
return merged;
}
@@ -450,9 +449,6 @@ export async function updateMediaScanOverride(
reviewed,
};
const file = mediaScanOverridesPath(paths);
- await mkdir(path.dirname(file), { recursive: true });
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(out, null, 2) + "\n");
- await rename(tmp, file);
+ await writeJsonAtomic(file, out, { mkdir: true });
return out;
}
diff --git a/common/controller/shard.ts b/common/controller/shard.ts
@@ -1,5 +1,6 @@
import path from "node:path";
-import { readFile, rename, unlink, writeFile } from "node:fs/promises";
+import { readFile, unlink } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import type { Paths } from "../lib/paths";
export const SHARD_OPS = [
@@ -52,10 +53,7 @@ export async function saveShardConfig(
op: ShardOp,
cfg: ShardConfig,
): Promise<void> {
- const file = shardFile(paths, slug, op);
- const tmp = `${file}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(cfg, null, 2) + "\n");
- await rename(tmp, file);
+ await writeJsonAtomic(shardFile(paths, slug, op), cfg);
}
export async function clearShardConfig(
diff --git a/common/jobs/syncSchedulerState.ts b/common/jobs/syncSchedulerState.ts
@@ -1,5 +1,5 @@
-import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
-import path from "node:path";
+import { readFile } from "node:fs/promises";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import type { Paths } from "../lib/paths";
// Persistent state for the cron-driven sync scheduler. Unlike the in-memory job
@@ -118,10 +118,7 @@ export async function writeSchedulerState(
lastSavedVideoBackupAt: state.lastSavedVideoBackupAt ?? null,
lastSyncAllAt: state.lastSyncAllAt ?? null,
};
- await mkdir(path.dirname(paths.schedulerStateFile), { recursive: true });
- const tmp = `${paths.schedulerStateFile}.tmp-${process.pid}`;
- await writeFile(tmp, JSON.stringify(out, null, 2) + "\n");
- await rename(tmp, paths.schedulerStateFile);
+ await writeJsonAtomic(paths.schedulerStateFile, out, { mkdir: true });
}
// Get (or lazily create) the mutable state entry for a channel.
diff --git a/common/jobs/workerDefaults.ts b/common/jobs/workerDefaults.ts
@@ -1,5 +1,5 @@
import fs from "node:fs";
-import path from "node:path";
+import { writeJsonAtomic } from "../lib/jsonFile-server";
import type { Paths } from "../lib/paths";
// Persisted "default" worker arrangement. Unlike the in-memory worker pool
@@ -55,10 +55,5 @@ export async function writeWorkerDefaults(
if (typeof id === "string" && id) seen.add(id);
}
const out: WorkerDefaults = { enabledWorkerIds: Array.from(seen) };
- await fs.promises.mkdir(path.dirname(paths.workerDefaultsFile), {
- recursive: true,
- });
- const tmp = `${paths.workerDefaultsFile}.tmp-${process.pid}`;
- await fs.promises.writeFile(tmp, JSON.stringify(out, null, 2) + "\n");
- await fs.promises.rename(tmp, paths.workerDefaultsFile);
+ await writeJsonAtomic(paths.workerDefaultsFile, out, { mkdir: true });
}
diff --git a/common/lib/homepage.ts b/common/lib/homepage.ts
@@ -1,4 +1,5 @@
import fs from "node:fs";
+import { writeJsonAtomic } from "./jsonFile-server";
import { getPaths, type Paths } from "./paths";
import { PROJECT_NAME, PROJECT_TAGLINE } from "./project";
import {
@@ -124,9 +125,6 @@ export async function writeHomepageConfig(
? { cloudflareProject: config.cloudflareProject.trim() }
: {}),
};
- await fs.promises.mkdir(paths.homepageDir, { recursive: true });
- const file = paths.homepageConfigFile;
- const tmp = `${file}.tmp-${process.pid}`;
- await fs.promises.writeFile(tmp, JSON.stringify(merged, null, 2) + "\n");
- await fs.promises.rename(tmp, file);
+ // mkdir: the parent of homepageConfigFile IS homepageDir (lib/paths.ts).
+ await writeJsonAtomic(paths.homepageConfigFile, merged, { mkdir: true });
}
diff --git a/common/lib/jsonFile-server.test.ts b/common/lib/jsonFile-server.test.ts
@@ -193,3 +193,9 @@ test("copyFileAtomic: dest gets src's bytes, src stays, no temp left", async ()
await assert.rejects(copyFileAtomic(path.join(dir, "nope"), dest));
assert.deepEqual(await readdir(path.join(dir, "out")), ["dest.bin"]);
});
+
+test("the remark-empty transcript is the literal it replaced (videoActions)", () => {
+ // editor/app/channels/[slug]/videos/[id]/videoActions.ts wrote this literal
+ // by hand before slice W; it now writes the object with { indent: 0 }.
+ assert.equal(jsonText({ transcription: [] }, { indent: 0 }), '{"transcription":[]}\n');
+});
diff --git a/common/lib/widgetPresets.ts b/common/lib/widgetPresets.ts
@@ -1,5 +1,5 @@
import fs from "node:fs";
-import path from "node:path";
+import { writeJsonAtomic } from "./jsonFile-server";
import type { Paths } from "./paths";
// Persisted monitor-widget presets: named arrangements the /widget/builder board
@@ -77,12 +77,7 @@ async function writePresets(
presets: WidgetPreset[],
): Promise<void> {
const out: PresetsFile = { v: 1, presets };
- await fs.promises.mkdir(path.dirname(paths.widgetPresetsFile), {
- recursive: true,
- });
- const tmp = `${paths.widgetPresetsFile}.tmp-${process.pid}`;
- await fs.promises.writeFile(tmp, JSON.stringify(out, null, 2) + "\n");
- await fs.promises.rename(tmp, paths.widgetPresetsFile);
+ await writeJsonAtomic(paths.widgetPresetsFile, out, { mkdir: true });
}
// Save a preset under `name`. Saving over an existing name OVERWRITES its query
diff --git a/editor/app/channels/[slug]/videos/[id]/videoActions.ts b/editor/app/channels/[slug]/videos/[id]/videoActions.ts
@@ -71,6 +71,7 @@ import {
import { requestChannelSnapshot } from "yt-dlp-transcript-common/jobs/snapshotScheduler";
import { makeTaskTracker } from "yt-dlp-transcript-common/jobs/taskHooks";
import { fixIncompleteTranscriptOne } from "../../lib/fixIncompleteTranscript";
+import { writeJsonAtomic } from "yt-dlp-transcript-common/lib/jsonFile-server";
function videoQueueKey(config: ChannelConfig, override: string | undefined): string {
return resolveQueueKey(downloadQueueKey(config), override);
@@ -607,9 +608,8 @@ export async function markVideoUntranscribableAction(
} catch {
// good — file does not exist
}
- const tmp = path.join(videoDir, `transcript.tmp-${process.pid}.json`);
- await writeFile(tmp, '{"transcription":[]}\n');
- await rename(tmp, transcriptPath);
+ // Compact + "\n": exactly the literal `{"transcription":[]}\n` this wrote.
+ await writeJsonAtomic(transcriptPath, { transcription: [] }, { indent: 0 });
const failureListFile = path.join(
getPaths().channelsDir,
slug,
