commit 30224b9528b004afed82103f56581d66824dccd3
parent 7c0b0556f4121b9f19118e68b8cd251514b8a649
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Thu, 1 Oct 2026 21:10:17 -0400
common: the index scan lstats a tierable sub track and reads a tiered raw live chat only through the watchdog (review H1)
subsMs takes the link's times for transcript.live_chat.json; a stale-cues
fallback on a tiered raw reads through onDrive(mediaDir) only while the
channel's media is ok/in-place, and otherwise keeps the cues the last build
held. Case (k): a stalled media drive with a tiered live chat holds neither
the index nor the stats build, and no call reaches it.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
2 files changed, 138 insertions(+), 9 deletions(-)
diff --git a/common/controller/buildIndex.test.ts b/common/controller/buildIndex.test.ts
@@ -86,6 +86,11 @@ const COMMON = fileURLToPath(new URL("..", import.meta.url));
// holds. The last case reads it.
const writes: string[] = [];
let afterStat: ((p: string) => void) | null = null;
+// A STALLED MEDIA DRIVE (release 17): while set, a promise-API stat, readFile
+// or open whose path is a symlink resolving under it never settles — what a
+// call blocked on a stalled drive looks like from here — and is recorded.
+let hangUnder: string | null = null;
+const driveCalls: string[] = [];
{
const req = createRequire(import.meta.url);
const fsCjs = req("node:fs") as Record<string, unknown>;
@@ -132,6 +137,31 @@ let afterStat: ((p: string) => void) | null = null;
if (p !== null) afterStat?.(path.resolve(p));
return result;
};
+ const throughLinkUnder = (p: string, root: string): boolean => {
+ try {
+ if (!fsCjsLstat(p).isSymbolicLink()) return false;
+ // Through every link on the way (`media` is itself a link to the drive).
+ return fsCjsRealpath(p).startsWith(root + path.sep);
+ } catch {
+ return false;
+ }
+ };
+ const fsCjsLstat = (fsCjs.lstatSync as (p: string) => { isSymbolicLink(): boolean }).bind(fsCjs);
+ const fsCjsRealpath = (fsCjs.realpathSync as (p: string) => string).bind(fsCjs);
+ for (const n of ["stat", "readFile", "open"]) {
+ const orig = fspCjs[n] as (...a: unknown[]) => Promise<unknown>;
+ fspCjs[n] = function (this: unknown, ...args: unknown[]) {
+ const p = asPath(args[0]);
+ if (hangUnder && p !== null) {
+ const abs = path.resolve(p);
+ if (abs.startsWith(hangUnder + path.sep) || throughLinkUnder(abs, hangUnder)) {
+ driveCalls.push(`${n} ${abs}`);
+ return new Promise(() => {});
+ }
+ }
+ return orig.apply(this, args);
+ };
+ }
syncBuiltinESMExports();
}
@@ -689,6 +719,56 @@ test("(j) release 17: an unmounted MEDIA drive does not hold the index — the t
}
});
+test("(k) release 17: a STALLED media drive with a tiered live chat holds neither the index nor the stats build, and is not asked", async () => {
+ resetCorpus();
+ seedVideo("local");
+ seedDriveChannel();
+ // d1's raw live chat is tiered: the bytes on the drive, a relative link in
+ // data/d1 carrying the file's times, and no cues yet (so the build reads
+ // the raw — the fallback this case pins).
+ const chat =
+ JSON.stringify({ replayChatItemAction: { actions: [{ addChatItemAction: { item: { liveChatTextMessageRenderer: { message: { runs: [{ text: "hello" }] }, authorName: { simpleText: "a" }, timestampUsec: "1000000" } } } }], videoOffsetTimeMsec: "1000" } }) + "\n";
+ mkdirSync(path.join(DRIVE_MEDIA(), "d1"), { recursive: true });
+ const bytes = path.join(DRIVE_MEDIA(), "d1", "transcript.live_chat.json");
+ writeFileSync(bytes, chat);
+ symlinkSync("../../media/d1/transcript.live_chat.json", path.join(videoDir("d1", DRIVE_CHANNEL), "transcript.live_chat.json"));
+ const first = await runIndex();
+ assert.deepEqual(first.res.heldChannels, []);
+ const liveChatOf = () =>
+ withIndex((db) =>
+ [...db("subs").getRange()]
+ .filter(({ key }) => (key as unknown as string[])[2] === "d1")
+ .flatMap(({ value }) => (value as { track: string }[]).map((t) => t.track)),
+ );
+ assert.ok(liveChatOf().includes("live_chat"), "read through the link while the drive answered");
+
+ // The drive stalls: marked (the health pass does this in the editor), and
+ // every call that would reach it hangs. A metadata rewrite makes d1 changed.
+ const { recordLocationHealth, resetStorageHealth } = await import("../lib/storageHealth");
+ recordLocationHealth({ id: "usb", label: "USB drive", root: MEDIA }, "stalled");
+ hangUnder = MEDIA;
+ driveCalls.length = 0;
+ try {
+ seedVideo("d1", DRIVE_CHANNEL, { subs: true, title: "Retitled" });
+ const { res, log } = await runIndex();
+ assert.deepEqual(res.heldChannels, [], log.join("\n"));
+ assert.equal(res.changed, 1);
+ assert.equal(res.removed, 0);
+ assert.ok(liveChatOf().includes("live_chat"), "the cues the last build held are kept");
+ const { buildStats } = await import("./buildStats");
+ const stats = await buildStats({
+ paths,
+ onLog: () => {},
+ wholePoolStatsDir: path.join(ROOT, "pool-stats"),
+ });
+ assert.deepEqual(stats.heldChannels, []);
+ assert.deepEqual(driveCalls, [], "no call reached the stalled drive");
+ } finally {
+ hangUnder = null;
+ resetStorageHealth();
+ }
+});
+
test("(z) no write this file caused landed outside its temp root", () => {
// LMDB writes natively, past the spy: its file must be under the root too.
assert.ok(paths.lmdbPath.startsWith(ROOT + path.sep), paths.lmdbPath);
diff --git a/common/controller/buildIndex.ts b/common/controller/buildIndex.ts
@@ -28,6 +28,7 @@
import path from "node:path";
import { createHash } from "node:crypto";
import {
+ lstat,
mkdir,
readdir,
readFile,
@@ -95,6 +96,8 @@ import {
} from "../lib/channelConfig";
import { readChannelConfigFile } from "./channels";
import { inspectChannelMedia } from "../lib/channelMedia";
+import { isTierable } from "../lib/mediaTier";
+import { onDrive } from "../lib/storageHealth";
import {
HELD_WAYS_OUT,
describeHeld,
@@ -312,17 +315,19 @@ async function scanSource(
live: LiveEntry[];
channels: Map<string, ChannelConfig>;
held: Map<string, string>;
+ mediaAccess: Map<string, MediaAccess>;
}> {
const locations = getSettings().storage.locations;
const channels = new Map<string, ChannelConfig>();
const live: LiveEntry[] = [];
const held = new Map<string, string>();
+ const mediaAccess = new Map<string, MediaAccess>();
let channelEntries: Dirent[];
try {
channelEntries = await readdir(channelsDir, { withFileTypes: true });
} catch {
// Fresh transcripts dir with no channels yet.
- return { live, channels, held };
+ return { live, channels, held, mediaAccess };
}
for (const ch of channelEntries) {
if (!ch.isDirectory()) continue;
@@ -357,6 +362,10 @@ async function scanSource(
held.set(ch.name, heldReason(media, cfg.mediaDir ?? cfg.dataDir, locations));
continue;
}
+ mediaAccess.set(ch.name, {
+ readable: media.status === "ok" || media.status === "in-place",
+ drive: cfg.mediaDir?.trim() || undefined,
+ });
const dataDir = path.join(channelDir, "data");
let videoEntries: Dirent[];
try {
@@ -416,7 +425,13 @@ async function scanSource(
let subsMs: number | null = null;
for (const t of subTracks) {
try {
- const ms = (await stat(path.join(fullVideoDir, t.filename))).mtimeMs;
+ // A TIERED FILE IS `lstat`ED (release 17): `transcript.live_chat.json`
+ // is a sub track AND media — on a tiered channel a link into
+ // channels/<slug>/media, possibly on another drive. The link carries
+ // the file's times (the tier hook's `lutimes`), so this answers from
+ // the corpus disk and never reaches the media drive.
+ const p = path.join(fullVideoDir, t.filename);
+ const ms = (await (isTierable(t.filename) ? lstat(p) : stat(p))).mtimeMs;
if (subsMs === null || ms > subsMs) subsMs = ms;
} catch {
// ignore
@@ -478,9 +493,13 @@ async function scanSource(
}
for (const e of channelLive) live.push(e);
}
- return { live, channels, held };
+ return { live, channels, held, mediaAccess };
}
+// Whether a channel's media tier may be read in this build, and through which
+// drive (scanSource, from the same inspect the text guard asked).
+type MediaAccess = { readable: boolean; drive?: string };
+
function pathKeyId(k: PathKey): string {
return `${k[0]}\x00${k[1]}`;
}
@@ -650,6 +669,7 @@ export async function buildIndex({
live,
channels: channelConfigs,
held,
+ mediaAccess,
} = await scanSource(channelsDir, log);
// A full rebuild clears every channel's records, and a held channel cannot be
@@ -860,6 +880,10 @@ export async function buildIndex({
const pk: PathKey = [s.channelSlug, s.videoDir];
const prev = mtimes.get(pk);
+ // What the last build held for this video's sub tracks, read before
+ // a re-keyed record is removed: a tiered live chat whose raw cannot
+ // be read now keeps the cues it had (below).
+ const prevSubs = prev ? subs.get(prev.indexKey) : undefined;
if (prev && !indexKeysEqual(prev.indexKey, indexKey)) {
sums.remove(prev.indexKey);
cues.remove(prev.indexKey);
@@ -885,12 +909,37 @@ export async function buildIndex({
}
}
if (!trackCues) {
- const raw = await readFile(
- path.join(path.dirname(s.metaPath), t.filename),
- "utf8",
- );
- trackCues =
- t.track === "live_chat" ? parseLiveChat(raw) : parseVtt(raw);
+ const rawPath = path.join(path.dirname(s.metaPath), t.filename);
+ // THE RAW REPLAY IS MEDIA (release 17). Tiered — a link into
+ // channels/<slug>/media — it is read only while the channel's
+ // media is reachable, through the watchdog; otherwise (or when
+ // the drive does not answer) the cues the last build held are
+ // kept, and the track is re-read on a later build.
+ const tiered =
+ t.track === "live_chat" &&
+ (await lstat(rawPath).then((l) => l.isSymbolicLink(), () => false));
+ if (tiered) {
+ const access = mediaAccess.get(s.channelSlug);
+ let raw: string | null = null;
+ if (access?.readable) {
+ try {
+ const read = () => readFile(rawPath, "utf8");
+ raw = await (access.drive ? onDrive(access.drive, read) : read());
+ } catch {
+ raw = null;
+ }
+ }
+ if (raw === null) {
+ const kept = prevSubs?.find((x) => x.track === t.track);
+ if (kept) parsedSubs.push(kept);
+ continue;
+ }
+ trackCues = parseLiveChat(raw);
+ } else {
+ const raw = await readFile(rawPath, "utf8");
+ trackCues =
+ t.track === "live_chat" ? parseLiveChat(raw) : parseVtt(raw);
+ }
}
if (trackCues.length > 0) {
parsedSubs.push({ track: t.track, cues: trackCues });
