commit a064ef7b07f218a2902bfef7bad7ebd1f2acaa86
parent b7688f22158827ee9ebc2d1fe9af48eedd10385a
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 2 Oct 2026 00:56:58 -0400
editor: e2e rebased on the media tier
channel-storage: a classic channel's audio and raw live chat are tiered by the
preview and moved; data/<id>/audio.mp3 and transcript.live_chat.json are links;
the videos list and the file route serve the transcript and stream the audio
(200, a range 206) through the link; with the media root renamed away the
video page still reads, the audio is 503 with retry-after, the rack shows
media held and the hold says the text stays readable; Move back leaves a real
media/. New: a legacy channel reads Media layout retired and Sync all's skip
names migrate-tier; a digest runs on a channel whose media drive is away.
storage-locations and channels-storage-columns build the relocated media
tier (media link, mediaDir) instead of the retired data link.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Diffstat:
3 files changed, 390 insertions(+), 110 deletions(-)
diff --git a/editor/e2e/channel-storage.spec.ts b/editor/e2e/channel-storage.spec.ts
@@ -3,6 +3,8 @@ import {
lstat,
mkdir,
readdir,
+ readlink,
+ rename,
rm,
stat,
symlink,
@@ -21,11 +23,18 @@ import {
resetData,
resolvePath,
writeChannelConfig,
+ writeDigestVideo,
writeSettings,
} from "./helpers";
// MOVING A CHANNEL'S MEDIA TO ANOTHER DIRECTORY, AND BACK.
//
+// RELEASE 17: what moves is the MEDIA TIER — `channels/<slug>/media`, the big
+// files (the audio, the raw live chat), each reached from `data/<id>/<name>`
+// by a relative link. The text never leaves `data/` on the corpus disk. A
+// classic channel (its audio still a real file in `data/<id>/`) is tiered by
+// the move's preview and preflight first.
+//
// The thing this spec is really pinning is the claim the whole design rests on:
// that a relocated channel is indistinguishable from an in-place one to every
// reader. So the assertions after the move are deliberately NOT about the move —
@@ -52,6 +61,28 @@ const SLUG = "test-youtube";
const VIDEO = "20240101_test1234567";
const dataDir = () => resolvePath(`test-transcripts/channels/${SLUG}/data`);
+const mediaLink = () => resolvePath(`test-transcripts/channels/${SLUG}/media`);
+const videoFile = (name: string) => join(dataDir(), VIDEO, name);
+
+// A small audio file — the route streams bytes, it does not decode them.
+const AUDIO_BYTES = 4096;
+const AUDIO = "ID3" + "x".repeat(AUDIO_BYTES - 3);
+
+// THE TIERED LAYOUT, BY HAND: the bytes in `<mediaDir>/<id>/<name>` and the
+// RELATIVE link `data/<id>/<name> -> ../../media/<id>/<name>` — what the hook
+// leaves behind (lib/mediaTier-server.ts). `mediaDir` is the channel's real
+// `media/` (tiered in place) or a relocated target the `media` link points at.
+async function tierByHand(
+ mediaDir: string,
+ files: Record<string, string>,
+): Promise<void> {
+ await mkdir(join(mediaDir, VIDEO), { recursive: true });
+ for (const [name, content] of Object.entries(files)) {
+ await writeFile(join(mediaDir, VIDEO, name), content);
+ await rm(videoFile(name), { force: true });
+ await symlink(join("..", "..", "media", VIDEO, name), videoFile(name));
+ }
+}
// Wait until the channel has no running or queued job. Polled from the same
// endpoint /jobs draws, because the guard reads the same registry.
@@ -80,17 +111,26 @@ async function quiet(page: Page): Promise<void> {
test("relocate a channel's media to another root, and move it back", async ({
page,
}, testInfo) => {
- test.setTimeout(90_000);
+ test.setTimeout(120_000);
await resetData("one-youtube-channel-with-data");
await generateReport(page, SLUG);
await quiet(page);
const root = testInfo.outputPath("media-root");
await mkdir(root, { recursive: true });
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // A CLASSIC CHANNEL: its audio and its raw live chat are real files in
+ // `data/<id>/`, which is how every channel downloaded before release 17 is.
+ await writeFile(videoFile("audio.mp3"), AUDIO);
+ await writeFile(videoFile("transcript.live_chat.json"), "[]");
// --- before -------------------------------------------------------------
await page.goto(channelStage(SLUG, "storage"));
- await expect(page.getByLabel("media path")).toHaveText(dataDir());
+ // Two rows under one heading: the media tier (not there yet — in place) and
+ // the text, which never moves.
+ await expect(page.getByLabel("media path")).toHaveText(
+ `${mediaLink()} (in place)`,
+ );
+ await expect(page.getByLabel("text path")).toContainText(dataDir());
// In place draws no badge at all — a badge on every channel saying "normal"
// is what makes the one that matters hard to find.
await expect(page.getByLabel(/^media location:/)).toHaveCount(0);
@@ -114,8 +154,15 @@ test("relocate a channel's media to another root, and move it back", async ({
await page.getByRole("button", { name: "Preview", exact: true }).click();
const preview = page.getByLabel("relocation preview");
await expect(preview).toBeVisible({ timeout: 15_000 });
- // The fixture is two files in one video dir.
+ // The media tier only: the audio and the raw live chat — not the transcript
+ // or the metadata, which stay.
await expect(page.getByLabel("bytes to move")).toContainText("2 file(s)");
+ // THE PREVIEW TIERED THE CLASSIC CHANNEL FIRST, on this disk, and says so.
+ await expect(page.getByLabel("tiered first")).toContainText(
+ "2 file(s) tiered first",
+ );
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isDirectory()).toBe(true);
// --- the move -----------------------------------------------------------
await expect(moveButton).toBeEnabled();
@@ -136,39 +183,59 @@ test("relocate a channel's media to another root, and move it back", async ({
/Copying… .* · \d+ % · /,
);
- // data/ is a symlink now, and config.json records the target — written only
- // by the job, on success, after the copy verified.
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ // `media` is ONE symlink now, `data/` is still a real directory, and each big
+ // file in it is a link into `media/` — config.json records the target,
+ // written only by the job, on success, after the copy verified.
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
+ expect(await readlink(mediaLink())).toBe(target);
+ expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ expect((await lstat(dataDir())).isSymbolicLink()).toBe(false);
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
expect(
- (
- await readJson<{ dataDir?: string }>(
- `test-transcripts/channels/${SLUG}/config.json`,
- )
- ).dataDir,
- ).toBe(target);
+ (await lstat(videoFile("transcript.live_chat.json"))).isSymbolicLink(),
+ ).toBe(true);
+ expect((await lstat(videoFile("transcript.en.vtt"))).isFile()).toBe(true);
+ expect(await existsAbs(join(target, VIDEO, "audio.mp3"))).toBe(true);
+ const config = await readJson<{ mediaDir?: string; dataDir?: string }>(
+ `test-transcripts/channels/${SLUG}/config.json`,
+ );
+ expect(config.mediaDir).toBe(target);
+ expect(config.dataDir).toBe(undefined);
// The source was reclaimed: no parked copy left holding a second copy of the
- // channel on the volume the move exists to free, and no marker.
+ // media on the volume the move exists to free, and no marker.
const siblings = await readdir(
resolvePath(`test-transcripts/channels/${SLUG}`),
);
- expect(siblings.filter((n) => n.startsWith("data."))).toEqual([]);
+ expect(siblings.filter((n) => n.startsWith("media."))).toEqual([]);
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
// --- NOTHING ELSE NOTICED ----------------------------------------------
// The videos list is read off data/ through the same joined path it always
- // was, and resolves through the link.
- await page.goto(channelVideos(SLUG));
+ // was; the selected video lists the tiered audio beside the transcript.
+ await page.goto(channelVideos(SLUG, { video: VIDEO }));
await expect(page.getByText(VIDEO).first()).toBeVisible();
+ await expect(page.getByLabel("file name audio.mp3")).toBeVisible();
+ await expect(page.getByLabel("file name transcript.en.vtt")).toBeVisible();
// So does the per-file route, which resolves channelsDir/<slug>/data/<id>/
- // and has no idea any of this happened.
- const file = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
- );
+ // and has no idea any of this happened: the transcript off the corpus disk,
+ // the audio streamed through its link onto the other drive.
+ const fileUrl = (name: string) =>
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/${name}`;
+ const file = await page.request.get(fileUrl("transcript.en.vtt"));
expect(file.status()).toBe(200);
expect(await file.text()).toContain("WEBVTT");
+ const audio = await page.request.get(fileUrl("audio.mp3"));
+ expect(audio.status()).toBe(200);
+ expect(audio.headers()["content-type"]).toBe("audio/mpeg");
+ expect((await audio.body()).length).toBe(AUDIO_BYTES);
+ const ranged = await page.request.get(fileUrl("audio.mp3"), {
+ headers: { range: "bytes=0-99" },
+ });
+ expect(ranged.status()).toBe(206);
+ expect((await ranged.body()).toString()).toBe(AUDIO.slice(0, 100));
// --- the badge ----------------------------------------------------------
await page.goto("/channels");
@@ -176,6 +243,34 @@ test("relocate a channel's media to another root, and move it back", async ({
page.getByLabel(/^media location: Media relocated/),
).toBeVisible();
+ // --- THE MEDIA DRIVE GOES AWAY; THE TEXT DOES NOT ------------------------
+ // Renamed away (the buildStats.test.ts trick): the `media` link dangles, as
+ // it does with the drive unmounted. The video page still reads — its text is
+ // on the corpus disk — the audio is listed with its size unknown, the file
+ // route answers 503 with a retry for it and 200 for the transcript, and the
+ // rack says the media is held.
+ const away = join(root, `${SLUG}.away`);
+ await rename(join(root, SLUG), away);
+ await forgetCaches();
+ await page.goto(`/channels/${SLUG}/videos/${VIDEO}`);
+ await expect(page.getByLabel("file name transcript.en.vtt")).toBeVisible();
+ await expect(page.getByLabel("file name audio.mp3")).toBeVisible();
+ await expect(page.getByText("media drive not reachable").first()).toBeVisible();
+ const text = await page.request.get(fileUrl("transcript.en.vtt"));
+ expect(text.status()).toBe(200);
+ const held = await page.request.get(fileUrl("audio.mp3"));
+ expect(held.status()).toBe(503);
+ expect(held.headers()["retry-after"]).toBe("15");
+ await page.goto("/channels");
+ const chip = page.getByLabel(`media hold for ${SLUG}`);
+ await expect(chip).toContainText("media held");
+ await page.goto(channelStage(SLUG, "storage"));
+ await expect(page.getByLabel("media hold", { exact: true })).toContainText(
+ "Its text stays readable",
+ );
+ await rename(away, join(root, SLUG));
+ await forgetCaches();
+
// --- back, WHICH IS A DESTINATION AND NOT A SECOND BUTTON ----------------
// "Move back in place" was its own section with its own button; it is now
// the one destination the select offers while the media is on a location,
@@ -200,19 +295,24 @@ test("relocate a channel's media to another root, and move it back", async ({
{ timeout: 60_000 },
);
- // A real directory again, the config field gone, and the target reclaimed.
- expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ // `media/` a REAL directory on the corpus disk again, the config field gone,
+ // the target reclaimed — and the per-file link untouched, resolving here now.
+ expect((await lstat(mediaLink())).isDirectory()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(false);
+ expect((await lstat(videoFile("audio.mp3"))).isSymbolicLink()).toBe(true);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(undefined);
- const back = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
- );
+ expect(await existsAbs(target)).toBe(false);
+ const back = await page.request.get(fileUrl("transcript.en.vtt"));
expect(back.status()).toBe(200);
+ const backAudio = await page.request.get(fileUrl("audio.mp3"));
+ expect(backAudio.status()).toBe(200);
+ expect((await backAudio.body()).length).toBe(AUDIO_BYTES);
});
test("the Configure form shows the media location read-only", async ({
@@ -223,7 +323,7 @@ test("the Configure form shows the media location read-only", async ({
// exact: the Storage panel's badge is "media location: …", and a substring
// match would find either.
const line = page.getByLabel("media location", { exact: true });
- await expect(line).toHaveText("In the channel directory (data/)");
+ await expect(line).toHaveText("In the channel directory (media/)");
// Not an input: it is a record of what is on disk, and the only writer is a
// move that succeeded. A text box here would let config and disk disagree
// with a keystroke.
@@ -233,8 +333,8 @@ test("the Configure form shows the media location read-only", async ({
// THE BULK MOVE, from /channels, with the cold root coming out of Settings.
//
// Two channels are selected and exactly one moves. The other is a channel that
-// is ALREADY relocated, built directly on disk — an absolute `data` symlink plus
-// `config.dataDir`, which is precisely what a finished move leaves behind —
+// is ALREADY relocated, built directly on disk — an absolute `media` symlink
+// plus `config.mediaDir`, which is precisely what a finished move leaves behind —
// rather than by running a second relocation first. A real move here would cost
// a second rsync, a second job wait and a second 60s timeout to assert a skip
// that is decided before any byte is read; what is under test is the skip, and
@@ -280,16 +380,24 @@ test("the /channels bulk move queues one job per channel and skips the rest", as
});
const PRE = "pre-moved";
- const preTarget = join(root, PRE, "data");
- await mkdir(join(preTarget, "20240102_pre1234567"), { recursive: true });
+ const preTarget = join(root, PRE, "media");
+ const preVideo = "20240102_pre1234567";
+ await mkdir(join(preTarget, preVideo), { recursive: true });
+ await writeFile(join(preTarget, preVideo, "audio.mp3"), AUDIO);
+ const preData = resolvePath(`test-transcripts/channels/${PRE}/data/${preVideo}`);
+ await mkdir(preData, { recursive: true });
await writeFile(
- join(preTarget, "20240102_pre1234567", "transcript.en.vtt"),
+ join(preData, "transcript.en.vtt"),
"WEBVTT\n\n00:00.000 --> 00:01.000\nhello\n",
);
- await writeChannelConfig(PRE, { dataDir: preTarget });
+ await symlink(
+ join("..", "..", "media", preVideo, "audio.mp3"),
+ join(preData, "audio.mp3"),
+ );
+ await writeChannelConfig(PRE, { mediaDir: preTarget });
await symlink(
preTarget,
- resolvePath(`test-transcripts/channels/${PRE}/data`),
+ resolvePath(`test-transcripts/channels/${PRE}/media`),
);
await page.goto("/channels");
@@ -341,30 +449,31 @@ test("the /channels bulk move queues one job per channel and skips the rest", as
await expect(badges.first()).toHaveText("on Cold");
await expect(badges.nth(1)).toHaveText("on Cold");
- // The moved channel: config records the target under the tmp root, `data/` is
- // a link, and the videos list still lists the video through it.
+ // The moved channel: config records the target under the tmp root, `media` is
+ // a link, `data/` is still real, and the videos list lists the video.
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
- ).toBe(join(root, SLUG, "data"));
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ ).mediaDir,
+ ).toBe(join(root, SLUG, "media"));
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
+ expect((await lstat(dataDir())).isSymbolicLink()).toBe(false);
await page.goto(channelVideos(SLUG));
await expect(page.getByText(VIDEO).first()).toBeVisible();
// The skipped channel was not touched: same target, still a link, no marker.
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${PRE}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(preTarget);
expect(
(
- await lstat(resolvePath(`test-transcripts/channels/${PRE}/data`))
+ await lstat(resolvePath(`test-transcripts/channels/${PRE}/media`))
).isSymbolicLink(),
).toBe(true);
expect(
@@ -514,7 +623,7 @@ test("the Storage panel moves to a location picked by name", async ({
await expect(destination).toContainText("Cold");
await destination.selectOption("cold");
- const target = join(cold, SLUG, "data");
+ const target = join(cold, SLUG, "media");
// EXACT. The panel has a second preview since the clip-window card joined it
// ("Preview eviction"), and getByRole's name match is a case-insensitive
// SUBSTRING by default — so the bare name resolves to two buttons and the
@@ -533,10 +642,10 @@ test("the Storage panel moves to a location picked by name", async ({
});
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(target);
// AND THE BADGE READS THE NAME. `on Cold`, not sixty columns of absolute
@@ -566,14 +675,18 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
const root = testInfo.outputPath("resume-root");
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // The channel was tiered in place by the move's preflight before it was
+ // killed: its two big files in `media/`, linked from `data/`.
+ await tierByHand(mediaLink(), {
+ "audio.mp3": AUDIO,
+ "transcript.live_chat.json": "[]",
+ });
// The partial copy: the video dir is there with ONE of its two files.
await mkdir(join(target, VIDEO), { recursive: true });
await copyFile(
- resolvePath(
- `test-transcripts/channels/${SLUG}/data/${VIDEO}/transcript.en.vtt`,
- ),
- join(target, VIDEO, "transcript.en.vtt"),
+ join(mediaLink(), VIDEO, "audio.mp3"),
+ join(target, VIDEO, "audio.mp3"),
);
// AND A STALE SCRATCH DIR the source no longer has — the 2026-10-01 case: a
// transcriber's `.audio.mp3.parakeet/` copied mid-transcription, then deleted
@@ -592,6 +705,7 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
},
null,
2,
@@ -624,25 +738,29 @@ test("Resume move finishes an interrupted move and clears its marker", async ({
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(target);
- // And the bytes are readable through the same URL as ever.
+ // And the bytes are readable through the same URLs as ever.
const file = await page.request.get(
`${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
);
expect(file.status()).toBe(200);
+ const audio = await page.request.get(
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/audio.mp3`,
+ );
+ expect(audio.status()).toBe(200);
// The stale scratch dir is gone from the copy, and the copy is the source's
- // two files and nothing else.
+ // two media files and nothing else.
expect(await existsAbs(scratch)).toBe(false);
expect((await readdir(join(target, VIDEO))).sort()).toEqual([
- "metadata.info.json",
- "transcript.en.vtt",
+ "audio.mp3",
+ "transcript.live_chat.json",
]);
});
@@ -721,11 +839,13 @@ test("a move that starts while a job writes into the channel refuses, naming the
`Nothing has been touched.`,
);
- // NOTHING WAS TOUCHED: no marker, no copy, the media a real directory.
+ // NOTHING WAS TOUCHED: no marker, no copy, no `media` made, the text a real
+ // directory.
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
expect(await existsAbs(join(root, SLUG))).toBe(false);
+ expect(await existsAbs(mediaLink())).toBe(false);
expect((await lstat(dataDir())).isDirectory()).toBe(true);
// And the panel says it before anyone clicks: the move is blocked, and the
@@ -814,7 +934,7 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
await resetData("one-youtube-channel-with-data");
await generateReport(page, SLUG);
await quiet(page);
- const target = join(testInfo.outputPath("hold-root"), SLUG, "data");
+ const target = join(testInfo.outputPath("hold-root"), SLUG, "media");
await writeFile(
resolvePath(`test-transcripts/channels/${SLUG}/.relocating.json`),
JSON.stringify({
@@ -822,6 +942,7 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
}),
);
await forgetCaches();
@@ -841,6 +962,8 @@ test("the rack and the Storage panel show the hold while a marker stands, and it
await expect(hold).toContainText(
"when the move completes, or its marker is cleared below",
);
+ // A MEDIA move holds the media only (release 17).
+ await expect(hold).toContainText("Its text stays readable");
// Abandoned: the marker goes, and the hold with it. The click is retried
// until the marker is gone: one that lands before hydration does nothing.
@@ -869,11 +992,16 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
const root = testInfo.outputPath("reconcile-root");
- const target = join(root, SLUG, "data");
+ const target = join(root, SLUG, "media");
+ // Tiered in place by the interrupted move's preflight.
+ await tierByHand(mediaLink(), {
+ "audio.mp3": AUDIO,
+ "transcript.live_chat.json": '[{"fresh":true}]',
+ });
await mkdir(join(target, VIDEO), { recursive: true });
// The full copy, timestamps and all, as a copy pass leaves it…
- for (const name of ["metadata.info.json", "transcript.en.vtt"]) {
- const src = join(dataDir(), VIDEO, name);
+ for (const name of ["audio.mp3", "transcript.live_chat.json"]) {
+ const src = join(mediaLink(), VIDEO, name);
await copyFile(src, join(target, VIDEO, name));
const { atime, mtime } = await stat(src);
await utimes(join(target, VIDEO, name), atime, mtime);
@@ -882,7 +1010,10 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
const scratch = join(target, VIDEO, ".audio.mp3.parakeet");
await mkdir(scratch, { recursive: true });
await writeFile(join(scratch, "meta.json"), "{}");
- await writeFile(join(target, VIDEO, "transcript.en.vtt"), "WEBVTT\n\nstale\n");
+ await writeFile(
+ join(target, VIDEO, "transcript.live_chat.json"),
+ '[{"stale":true}]',
+ );
await writeFile(
resolvePath(`test-transcripts/channels/${SLUG}/.relocating.json`),
JSON.stringify({
@@ -890,6 +1021,7 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
direction: "out",
startedAt: new Date().toISOString(),
phase: "copy",
+ scope: "media",
}),
);
@@ -907,20 +1039,22 @@ test("Reconcile and resume settles an extra and a changed file on the destinatio
await expect(out).toContainText(
/Reconciling: the destination copy differs from the source — \d+ extra on the destination \([^)]*\.audio\.mp3\.parakeet/,
);
- await expect(out).toContainText(/changed \([^)]*transcript\.en\.vtt/);
+ await expect(out).toContainText(
+ /changed \([^)]*transcript\.live_chat\.json/,
+ );
// Finished: link, config, no marker; the copy is the source's two files.
expect(
await pathExists(`test-transcripts/channels/${SLUG}/.relocating.json`),
).toBe(false);
- expect((await lstat(dataDir())).isSymbolicLink()).toBe(true);
+ expect((await lstat(mediaLink())).isSymbolicLink()).toBe(true);
expect(await existsAbs(scratch)).toBe(false);
expect((await readdir(join(target, VIDEO))).sort()).toEqual([
- "metadata.info.json",
- "transcript.en.vtt",
+ "audio.mp3",
+ "transcript.live_chat.json",
]);
const file = await page.request.get(
- `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.live_chat.json`,
);
expect(file.status()).toBe(200);
expect(await file.text()).not.toContain("stale");
@@ -986,8 +1120,10 @@ test("a move to an unmounted root refuses before it creates anything", async ({
// NOTHING WAS CREATED on the way to finding out — that IS the bug.
expect(await pathExists(root)).toBe(false);
expect(await pathExists(join(root, SLUG))).toBe(false);
- // The media is still a real directory in the corpus, unmoved.
+ // The channel is untouched in the corpus — not even tiered: the root is
+ // refused before the preview's tiering runs.
expect((await lstat(dataDir())).isDirectory()).toBe(true);
+ expect(await existsAbs(mediaLink())).toBe(false);
});
// EVICTING THIS CHANNEL'S FETCHED CLIP WINDOWS, from the panel that already
@@ -1094,13 +1230,13 @@ test("Sync all skips a channel whose media drive is not mounted", async ({
test.setTimeout(90_000);
await resetData("one-youtube-channel-with-data");
await writeSettings({ minFreeDiskGB: 0 });
- // A relocation whose drive went away: the link and the config agree with each
- // other and with nothing on disk. Deliberately NOT created — an unmounted
- // mountpoint whose parent is missing too is the honest version.
- const target = join(testInfo.outputPath("never-mounted"), SLUG, "data");
- await rm(dataDir(), { recursive: true, force: true });
- await symlink(target, dataDir());
- await writeChannelConfig(SLUG, { dataDir: target });
+ // A relocation whose drive went away: the `media` link and the config agree
+ // with each other and with nothing on disk. Deliberately NOT created — an
+ // unmounted mountpoint whose parent is missing too is the honest version.
+ // The text stays in `data/` on the corpus disk, as it always does now.
+ const target = join(testInfo.outputPath("never-mounted"), SLUG, "media");
+ await symlink(target, mediaLink());
+ await writeChannelConfig(SLUG, { mediaDir: target });
await page.goto("/channels");
await page.getByRole("button", { name: "sync every channel" }).click();
@@ -1115,3 +1251,104 @@ test("Sync all skips a channel whose media drive is not mounted", async ({
new RegExp(`${SLUG}: media unreachable:.*drive not mounted`),
);
});
+
+// THE RETIRED LAYOUT (release 17). A channel whose whole `data/` was moved
+// before the media tier — an absolute `data` link and `config.dataDir`, the old
+// mover's work — is `legacy`: its text is on the far drive too, so it is held
+// by both guards until `archilyzer storage migrate-tier` brings the text home.
+// The row says "Media layout retired", the Storage panel refuses to move it
+// with the same sentence, and a media job refused on it names the command.
+test("a legacy channel reads Media layout retired, and a media job's refusal names migrate-tier", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(90_000);
+ await resetData("one-youtube-channel-with-data");
+ await writeSettings({ minFreeDiskGB: 0 });
+ const target = join(testInfo.outputPath("legacy-root"), SLUG, "data");
+ await mkdir(join(target, VIDEO), { recursive: true });
+ for (const name of await readdir(join(dataDir(), VIDEO))) {
+ await copyFile(join(dataDir(), VIDEO, name), join(target, VIDEO, name));
+ }
+ await rm(dataDir(), { recursive: true, force: true });
+ await symlink(target, dataDir());
+ await writeChannelConfig(SLUG, { dataDir: target });
+ await forgetCaches();
+
+ await page.goto("/channels");
+ await expect(
+ page.getByLabel(/^media location: Media layout retired/),
+ ).toBeVisible();
+
+ await page.goto(channelStage(SLUG, "storage"));
+ await expect(page.getByLabel("move back refused")).toContainText(
+ `run archilyzer storage migrate-tier ${SLUG}`,
+ );
+
+ await page.goto("/channels");
+ await page.getByRole("button", { name: "sync every channel" }).click();
+ const result = page.getByLabel("sync all result");
+ await expect(result).toContainText(/Queued 0 . skipped 1/, {
+ timeout: 15_000,
+ });
+ await expect(result).toHaveAttribute(
+ "title",
+ new RegExp(`${SLUG}: .*archilyzer storage migrate-tier ${SLUG}`),
+ );
+});
+
+// THE TEXT OUTLIVES ITS MEDIA DRIVE (release 17): a digest reads and writes
+// the text tier, so it runs on a channel whose media is relocated onto a drive
+// that is not there — the job is asked the text guard, not the media one, and
+// the digest lane makes the same decision (autoRunner.ts isChannelHeldForLane,
+// pinned in its unit tests). The ollama stub answers, as in digest.spec.ts.
+test("a digest runs on a channel whose media drive is away", async ({
+ page,
+}, testInfo) => {
+ test.setTimeout(120_000);
+ const CHANNEL = "digest-channel";
+ const DIGEST_VIDEO = "digestvid0001";
+ await resetData(null);
+ await writeSettings({
+ adminTitle: "Test Admin",
+ maxTranscriptPageBytes: 8388608,
+ sleepBetweenDownloadsSeconds: 0,
+ minFreeDiskGB: 0,
+ digest: {
+ localAppId: "ollama-direct",
+ remoteAppId: "claude-code",
+ sections: ["chapters"],
+ },
+ });
+ // Relocated media, renamed away: the `media` link dangles; `data/` is real.
+ const target = join(testInfo.outputPath("away-root"), CHANNEL, "media");
+ await writeChannelConfig(CHANNEL, { mediaDir: target });
+ await writeDigestVideo({ channelSlug: CHANNEL, videoId: DIGEST_VIDEO });
+ await symlink(
+ target,
+ resolvePath(`test-transcripts/channels/${CHANNEL}/media`),
+ );
+ await symlink(
+ join("..", "..", "media", DIGEST_VIDEO, "audio.mp3"),
+ resolvePath(
+ `test-transcripts/channels/${CHANNEL}/data/${DIGEST_VIDEO}/audio.mp3`,
+ ),
+ );
+ await forgetCaches();
+
+ await generateReport(page, CHANNEL);
+ await page.goto("/channels");
+ await expect(page.getByLabel(`media hold for ${CHANNEL}`)).toContainText(
+ "media held",
+ );
+ await page.goto(channelStage(CHANNEL, "digest"));
+ await page.getByRole("button", { name: "Digest channel" }).click();
+ await expect(page.getByLabel("Digest channel output")).toContainText(
+ "Digest batch:",
+ { timeout: 60_000 },
+ );
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${CHANNEL}/data/${DIGEST_VIDEO}/ai-digest.json`,
+ ),
+ ).toBe(true);
+});
diff --git a/editor/e2e/channels-storage-columns.spec.ts b/editor/e2e/channels-storage-columns.spec.ts
@@ -1,4 +1,4 @@
-import { mkdir, rename, symlink } from "node:fs/promises";
+import { mkdir, readdir, rename, symlink, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { test, expect } from "@playwright/test";
import { baseUrl } from "./baseUrl";
@@ -32,21 +32,47 @@ import {
// 549d).
const SLUG = "test-youtube";
+const VIDEO = "20240101_test1234567";
+
+// THE SIZE COLUMN IS THE MEDIA TIER (release 17): the audio and the raw live
+// chat, what a move carries. The fixture has none, so one is written.
+async function giveAudio(slug: string, id: string, bytes = 4096) {
+ await writeFile(
+ resolvePath(`test-transcripts/channels/${slug}/data/${id}/audio.mp3`),
+ "x".repeat(bytes),
+ );
+}
-// Put a channel's media on `root` the way a finished relocation leaves it: the
-// real directory on the "drive", an absolute symlink at channels/<slug>/data,
-// and config.dataDir naming the target. Built directly rather than by running a
-// move — what is under test is the COLUMN, and a real rsync buys nothing here.
+// Put a channel's media tier on `root` the way a finished relocation leaves it
+// (release 17): each audio file moved to `<root>/<slug>/media/<id>/` with its
+// relative link left in `data/<id>/`, an absolute symlink at
+// channels/<slug>/media, and config.mediaDir naming the target. Built directly
+// rather than by running a move — what is under test is the COLUMN, and a real
+// rsync buys nothing here.
async function relocateOnDisk(slug: string, root: string): Promise<string> {
const channelDir = resolvePath(`test-transcripts/channels/${slug}`);
- const target = join(root, slug, "data");
- await mkdir(join(root, slug), { recursive: true });
- await rename(join(channelDir, "data"), target);
- await symlink(target, join(channelDir, "data"));
+ const target = join(root, slug, "media");
+ await mkdir(target, { recursive: true });
+ for (const id of await readdir(join(channelDir, "data"))) {
+ const names = await readdir(join(channelDir, "data", id)).catch(
+ () => [] as string[],
+ );
+ if (!names.includes("audio.mp3")) continue;
+ await mkdir(join(target, id), { recursive: true });
+ await rename(
+ join(channelDir, "data", id, "audio.mp3"),
+ join(target, id, "audio.mp3"),
+ );
+ await symlink(
+ join("..", "..", "media", id, "audio.mp3"),
+ join(channelDir, "data", id, "audio.mp3"),
+ );
+ }
+ await symlink(target, join(channelDir, "media"));
const config = await readJson<Record<string, unknown>>(
`test-transcripts/channels/${slug}/config.json`,
);
- await writeChannelConfig(slug, { ...config, dataDir: target });
+ await writeChannelConfig(slug, { ...config, mediaDir: target });
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
return target;
}
@@ -66,6 +92,7 @@ test("the Location and Size columns say which disk and how much", async ({
defaultLocationId: "cold",
},
});
+ await giveAudio(SLUG, VIDEO);
await generateReport(page, SLUG);
await page.goto("/channels");
@@ -73,8 +100,8 @@ test("the Location and Size columns say which disk and how much", async ({
// is on the corpus volume, and that is a fact about it, not an absence.
const location = page.getByLabel(`media location for ${SLUG}`);
await expect(location).toContainText("Internal");
- // The fixture is two small files, so the figure is not zero and not "—": the
- // report measured it.
+ // The fixture's audio is a small file, so the figure is not zero and not
+ // "—": the report measured it.
const size = page.getByLabel(`media size for ${SLUG}`);
await expect(size).not.toHaveText("—");
await expect(size).toContainText(/B|KB|MB/);
@@ -131,6 +158,8 @@ test("filter by volume, sort by size, and free up N GB", async ({
resolvePath(`test-transcripts/channels/${SECOND}/data/20240102_second0001`),
{ recursive: true },
);
+ await giveAudio(SLUG, VIDEO);
+ await giveAudio(SECOND, "20240102_second0001", 1024);
await writeSettings({
adminTitle: "Test Admin",
minFreeDiskGB: 0,
diff --git a/editor/e2e/storage-locations.spec.ts b/editor/e2e/storage-locations.spec.ts
@@ -26,8 +26,10 @@ import {
// back at a different mountpoint, ONE button makes every channel on it readable
// again, and it does so WITHOUT MOVING A BYTE. So the assertions after the
// re-point are deliberately not about the re-point — they are `readlink`,
-// `config.dataDir` and the transcript route serving the same URL it always did,
-// with the media now reached through a different path entirely.
+// `config.mediaDir` and the file route serving the same URLs it always did,
+// with the media now reached through a different path entirely. (Release 17:
+// what lives on a location is a channel's media tier, `channels/<slug>/media`;
+// its text never leaves `data/` on the corpus disk.)
//
// THE DISK IS A FAKE BINARY. `FINDMNT_BIN` and `UDISKSCTL_BIN` point at
// `e2e/fixtures/bin/fake-{findmnt,udisksctl}.mjs` (see editor/package.json), and
@@ -41,7 +43,7 @@ const VIDEO = "20240101_test1234567";
const UUID = "e2e-platter-uuid";
const channelDir = () => resolvePath(`test-transcripts/channels/${SLUG}`);
-const dataLink = () => join(channelDir(), "data");
+const mediaLink = () => join(channelDir(), "media");
const controlFile = () => resolvePath("test-transcripts/.fake-findmnt.json");
async function writeControl(control: {
@@ -53,22 +55,27 @@ async function writeControl(control: {
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
}
-// Put the fixture channel's media on `root`, the way a finished relocation
-// leaves it: the real directory on the "drive", an absolute symlink at
-// channels/<slug>/data, and config.dataDir naming the target. Built directly
+// Put the fixture channel's media tier on `root`, the way a finished
+// relocation leaves it: the audio in `<root>/<slug>/media/<id>/` on the
+// "drive", its relative link in `data/<id>/`, an absolute symlink at
+// channels/<slug>/media, and config.mediaDir naming the target. Built directly
// rather than by running a move — what is under test here is the re-point, and
// a real rsync first would buy nothing but a minute.
async function relocateOnDisk(root: string): Promise<string> {
- const target = join(root, SLUG, "data");
- await mkdir(join(root, SLUG), { recursive: true });
- await rename(dataLink(), target);
- await symlink(target, dataLink());
+ const target = join(root, SLUG, "media");
+ await mkdir(join(target, VIDEO), { recursive: true });
+ await writeFile(join(target, VIDEO, "audio.mp3"), "ID3audio");
+ await symlink(
+ join("..", "..", "media", VIDEO, "audio.mp3"),
+ join(channelDir(), "data", VIDEO, "audio.mp3"),
+ );
+ await symlink(target, mediaLink());
const config = await readJson<Record<string, unknown>>(
`test-transcripts/channels/${SLUG}/config.json`,
);
await writeFile(
join(channelDir(), "config.json"),
- JSON.stringify({ ...config, dataDir: target }, null, 2) + "\n",
+ JSON.stringify({ ...config, mediaDir: target }, null, 2) + "\n",
);
await fetch(`${baseUrl}/api/test/invalidate-cache`).catch(() => {});
return target;
@@ -138,7 +145,7 @@ test("the page lists a location with its status, counts and refusals", async ({
// DELETE IS REFUSED WHILE ANYBODY LIVES THERE. Deleting the location moves no
// bytes — which is exactly why it is refused: it would erase the only record
- // of which disk that channel's absolute dataDir belongs to.
+ // of which disk that channel's absolute mediaDir belongs to.
await expect(cold.getByLabel("delete cold")).toBeDisabled();
await expect(cold.getByLabel("delete withheld")).toContainText(
"1 channel(s) still have their media under",
@@ -223,7 +230,7 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
// --- the disk comes back somewhere else ---------------------------------
// The media moves with it, because it IS the disk. Nothing in the corpus
- // changes: config.dataDir and the symlink still name the old mountpoint,
+ // changes: config.mediaDir and the symlink still name the old mountpoint,
// which is precisely the breakage this page exists for.
await rename(join(rootA, SLUG), join(rootB, SLUG));
await rm(rootA, { recursive: true, force: true });
@@ -235,7 +242,7 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
// is intact and its target is gone, which is the whole of the breakage. A
// stat here follows the link and reports "no data/ at all", which is the
// reading this module exists to stop anybody making.
- expect(await readlink(dataLink())).toBe(targetA);
+ expect(await readlink(mediaLink())).toBe(targetA);
await page.goto("/storage");
// The probe memo is ten seconds wide and keyed by id+root, neither of which
@@ -261,17 +268,17 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
});
// The link and the config now name the new mountpoint...
- const targetB = join(rootB, SLUG, "data");
+ const targetB = join(rootB, SLUG, "media");
expect(targetA).not.toBe(targetB);
await expect
- .poll(async () => readlink(dataLink()), { timeout: 30_000 })
+ .poll(async () => readlink(mediaLink()), { timeout: 30_000 })
.toBe(targetB);
expect(
(
- await readJson<{ dataDir?: string }>(
+ await readJson<{ mediaDir?: string }>(
`test-transcripts/channels/${SLUG}/config.json`,
)
- ).dataDir,
+ ).mediaDir,
).toBe(targetB);
// ...and so does the location.
const settings = await readJson<{
@@ -282,12 +289,19 @@ test("a volume that came up somewhere else is re-pointed in one click", async ({
);
// NOTHING ELSE NOTICED. The per-file route joins channelsDir/<slug>/data/<id>
- // exactly as it always has and has no idea any of this happened.
+ // exactly as it always has and has no idea any of this happened: the
+ // transcript off the corpus disk, the audio through its link and the
+ // re-pointed `media` onto the drive at its new mountpoint.
const file = await page.request.get(
`${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/transcript.en.vtt`,
);
expect(file.status()).toBe(200);
expect(await file.text()).toContain("WEBVTT");
+ const audio = await page.request.get(
+ `${baseUrl}/api/channels/${SLUG}/videos/${VIDEO}/files/audio.mp3`,
+ );
+ expect(audio.status()).toBe(200);
+ expect(await audio.text()).toBe("ID3audio");
// And the page agrees: the channel is reachable again.
await page.goto("/storage");
