commit c717a41f8a23270321d894ed965ab62d3d25f6d0
parent b1188708cc2c59dd2c22ec1ab4ba7ca18d599557
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sat, 30 May 2026 17:00:35 -0400
more shard fixes
Diffstat:
12 files changed, 375 insertions(+), 56 deletions(-)
diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts
@@ -32,6 +32,9 @@ export type CheckAvailabilityOpts = {
concurrency?: number;
shardTotal?: number;
shardIndex?: number;
+ // When set, resolve + persist the shard slice and return immediately without
+ // checking anything. Backs the "Save shard" button.
+ saveShardOnly?: boolean;
onLog?: (msg: string) => void;
signal?: AbortSignal;
};
@@ -91,6 +94,7 @@ export async function runAvailabilityCheck({
concurrency,
shardTotal,
shardIndex,
+ saveShardOnly = false,
onLog,
signal,
}: CheckAvailabilityOpts): Promise<CheckAvailabilityResult> {
@@ -103,7 +107,7 @@ export async function runAvailabilityCheck({
const limit = pLimit(concurrency && concurrency > 0 ? Math.floor(concurrency) : 1);
const allDirs = await readdir(dataDir).catch(() => [] as string[]);
- const { items: videoDirs } = await resolveShardItems({
+ const shardResult = await resolveShardItems({
paths,
slug: channelSlug,
op: "availability",
@@ -112,6 +116,13 @@ export async function runAvailabilityCheck({
shardIndex: shardIndex,
onLog: log,
});
+ const videoDirs = shardResult.items;
+ if (saveShardOnly) {
+ log(
+ `Shard availability: save-only — slice ${(shardResult.config?.shardIndex ?? shardIndex ?? 0) + 1}/${shardResult.config?.totalShards ?? shardTotal} (${videoDirs.length} item(s)) persisted; not checking.`,
+ );
+ return { attempted: 0, checked: 0, skipped: 0, byStatus: emptyByStatus() };
+ }
let attempted = 0;
let checked = 0;
diff --git a/common/controller/shard.ts b/common/controller/shard.ts
@@ -137,7 +137,7 @@ export async function resolveShardItems({
saved.shardIndex === shardIndex
) {
log(
- `Shard ${op}: using saved slice ${saved.shardIndex}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
+ `Shard ${op}: using saved slice ${saved.shardIndex + 1}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
);
return { items: saved.items, source: "saved", config: saved };
}
@@ -152,17 +152,20 @@ export async function resolveShardItems({
};
await saveShardConfig(paths, slug, op, cfg);
log(
- `Shard ${op}: computed slice ${shardIndex}/${totalShards} (${slice.length} of ${fullItems.length} item(s)); persisted.`,
+ `Shard ${op}: computed slice ${shardIndex! + 1}/${totalShards} (${slice.length} of ${fullItems.length} item(s)); persisted.`,
);
return { items: slice, source: "computed", config: cfg };
}
if (saved) {
log(
- `Shard ${op}: using saved slice ${saved.shardIndex}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
+ `Shard ${op}: using saved slice ${saved.shardIndex + 1}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
);
return { items: saved.items, source: "saved", config: saved };
}
+ log(
+ `Shard ${op}: no shard inputs provided; running the full set (${fullItems.length} item(s)).`,
+ );
return { items: fullItems, source: "full", config: null };
}
diff --git a/common/controller/whisperBatch.ts b/common/controller/whisperBatch.ts
@@ -22,6 +22,10 @@ export type WhisperBatchOptions = {
reverse?: boolean;
shardTotal?: number;
shardIndex?: number;
+ // When set, resolve + persist the shard slice and return immediately without
+ // transcribing anything. Backs the "Save shard" button so a slice can be
+ // pinned (and confirmed saved) before dividing work across synced machines.
+ saveShardOnly?: boolean;
// When sharding is active, refine the job's progress total to this slice
// (progressBaseline + shard items still needing a transcript) so the Active
// Jobs bar measures the shard, not the whole channel.
@@ -58,6 +62,7 @@ export async function runWhisperBatch({
reverse = false,
shardTotal,
shardIndex,
+ saveShardOnly = false,
setProgress,
progressBaseline,
ids,
@@ -81,16 +86,30 @@ export async function runWhisperBatch({
.split("\n")
.filter(Boolean);
+ const failedSet = new Set<string>(originalFailures);
+
const allDirs = await readdir(dataDir);
// When the caller supplies an explicit id set (bucket-scoped run), filter
// to that intersection so we never try to transcribe ids whose dirs aren't
// on disk. Otherwise consider every video dir.
- const fullItems = ids
+ const candidateIds = ids
? (() => {
const onDisk = new Set(allDirs);
return ids.filter((id) => onDisk.has(id));
})()
: allDirs;
+ // Shard only the *remaining* work — videos this run would actually attempt:
+ // no transcript yet (neither whisper's transcript.json nor a yt-dlp VTT) and
+ // not on the failed list. This is what makes a shard slice meaningful (e.g.
+ // 6 untranscribed videos split 2 ways → 3 each, not 29 of the whole channel).
+ const fullItems: string[] = [];
+ for (const id of candidateIds) {
+ if (failedSet.has(id)) continue;
+ const vp = path.join(dataDir, id);
+ if (await pathExists(path.join(vp, WHISPER_FILENAME))) continue;
+ if (await pathExists(path.join(vp, VTT_FILENAME))) continue;
+ fullItems.push(id);
+ }
const shardResult = await resolveShardItems({
paths,
slug: channelSlug,
@@ -101,6 +120,12 @@ export async function runWhisperBatch({
onLog: log,
});
const shardItems = shardResult.items;
+ if (saveShardOnly) {
+ log(
+ `Shard transcribe-missing: save-only — slice ${(shardResult.config?.shardIndex ?? shardIndex ?? 0) + 1}/${shardResult.config?.totalShards ?? shardTotal} (${shardItems.length} item(s)) persisted; not transcribing.`,
+ );
+ return { attempted: 0, succeeded: 0, failed: 0, skipped: 0 };
+ }
// With a shard active, refine the job's progress total to this slice:
// baseline (channel-wide transcriptCount at run start) + the shard's items
// still needing a transcript. `current` is read channel-wide on the Active
@@ -124,8 +149,6 @@ export async function runWhisperBatch({
const videoDirs = [...shardItems];
if (reverse) videoDirs.reverse();
- const failedSet = new Set<string>(originalFailures);
-
let attempted = 0;
let succeededCount = 0;
let failed = 0;
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -57,6 +57,9 @@ export type RunYtdlpOpts = {
// shard. Saved/resumed via channels/<slug>/shard-download-missing.json.
shardTotal?: number;
shardIndex?: number;
+ // download-missing only: when set, resolve + persist the shard slice and
+ // return immediately without downloading. Backs the "Save shard" button.
+ saveShardOnly?: boolean;
// download-missing only: when sharding is active, the run refines the job's
// progress target to its own slice (progressBaseline + shard items left to
// fetch) so the Active Jobs bar measures the shard, not the whole channel.
@@ -420,6 +423,21 @@ async function downloadPlaylistManaged(
: null;
const items = shardResult ? shardResult.items : tofetch;
+ // "Save shard" probe: the slice is already persisted by resolveShardItems
+ // above; stop here without downloading anything.
+ if (opts.saveShardOnly) {
+ if (shardResult && shardResult.config) {
+ opts.onLog(
+ `Shard download-missing: save-only — slice ${shardResult.config.shardIndex + 1}/${shardResult.config.totalShards} (${shardResult.config.items.length} item(s)) persisted; not downloading.\n`,
+ );
+ } else {
+ opts.onLog(
+ `Shard download-missing: save-only requested but no shard inputs were provided; nothing persisted.\n`,
+ );
+ }
+ return;
+ }
+
// With a shard active, refine the job's progress total to this slice:
// baseline (channel-wide downloadCount at run start) + the shard's items
// still needing download. `current` is read channel-wide on the Active Jobs
diff --git a/editor/CHANGELOG.md b/editor/CHANGELOG.md
@@ -1,6 +1,7 @@
# Changelog
## [Unreleased]
+- **Shard controls overhauled: "Save" button, 1-based `i / N` inputs, remaining-only transcribe slices, and always-on logging.** The shard inputs now read **`i / N`** (this shard's number first, 1-based — a 2-way split is `1/2` and `2/2`) and each box has its own hover title so they're no longer easy to mix up. A new **Save** button persists a slice to `shard-<op>.json` *without* starting the job — handy for dividing the remaining videos across machines (that share an identical synced copy) up front, and for confirming the slice actually saved; the saved-shard pill updates immediately. **Transcribe** sharding now splits only the videos that still need a transcript (e.g. 6 untranscribed videos split 2 ways → 3 each) instead of slicing the whole channel's video list. And every Download/Transcribe/Availability run now logs whether it's running a computed/saved shard slice or the full set, so a run that silently ignored shard inputs is no longer indistinguishable from one that honored it.
- **Shard configs now show up immediately, and a shard job's progress bar totals only its slice.** Previously, setting a shard (N/i) and running **Download videos** / **Transcribe missing** did persist the slice to `shard-*.json`, but the saved-shard indicator never updated until you manually reloaded the whole page — so it looked like nothing saved. The indicator on the Download/Transcribe/Diagnostics controls (and channel counts, buckets, and failed-lists generally) now refresh as soon as a run finishes — including after a cancel — without a reload. The download slice is still taken over the *missing* set and snapshotted to the file, so a resume run with the same N/i reuses that exact slice instead of re-slicing the now-smaller set. A sharded job's progress bar on `/jobs/active` now totals only its own slice rather than the whole channel.
- **Multi-site support (major change).** One editor instance can now power several public sites (e.g. "Jeralyzer" and "Rekietalyzer") over a single shared channel pool — a channel's downloads are stored once and reused by every site that includes it. A new **Sites** area (sidebar) manages each site's `sites/<id>/site.json`: its branding (title, header, description, tagline), social links, channel-group layout, and **which channels it exposes** (with a per-site group for each member, so the same channel can sit in different groups on different sites) plus its Cloudflare Pages project. Site-level branding/groups/social have moved **out** of global Settings, which now holds only operational config plus a new **Admin title** for the editor's own shell; the per-channel "Group" field is gone (grouping is configured per site). The **Charts** tab is per-site (pick the site at the top; each site has its own dashboard at `sites/<id>/chart-templates.json`). **Build index** / **Build stats dataset** now build the shared per-channel data once and a filtered bundle per site; the Deploy page's **Build static export** and **Deploy** gained a site selector and deploy each site to its own Cloudflare project. Upgrading an existing single-site install: the Sites page shows a one-click **Migrate** button (also a `migrate-to-sites` CLI) that lifts your current branding, channel groups, per-channel group assignments, and chart dashboard into a first site and reduces `settings.json` to operational config.
- **"Audio + Whisper (skip pipeline)" no longer mistakes a live-chat sidecar for audio.** yt-dlp writes the live-chat track as `audio.live_chat.json` (it ignores the subtitle output path), so an interrupted download could leave an `audio.live_chat.json.part` behind. One-click whisper saw the `audio.` prefix, decided audio was already on disk, skipped the download, and then failed with "no audio file found". Live-chat sidecars (`.part` or completed) are now excluded everywhere audio is detected, so whisper downloads the real audio and transcribes as expected.
diff --git a/editor/app/channels/[slug]/components/stages/DiagnosticsStage.tsx b/editor/app/channels/[slug]/components/stages/DiagnosticsStage.tsx
@@ -15,6 +15,7 @@ import { ConcurrencyControl } from "../../../../components/ConcurrencyControl";
import {
ShardControl,
parseShardField,
+ parseShardIndexField,
type ShardConfigSummary,
} from "../../../../components/ShardControl";
import { RetryBucketControl } from "../RetryBucketControl";
@@ -317,7 +318,7 @@ function AvailabilityButtons({
shard ? String(shard.totalShards) : "",
);
const [shardIndex, setShardIndex] = useState(
- shard ? String(shard.shardIndex) : "",
+ shard ? String(shard.shardIndex + 1) : "",
);
const extra = (
<>
@@ -341,7 +342,7 @@ function AvailabilityButtons({
);
const limit = parseConcurrency(concurrency);
const total = parseShardField(shardTotal);
- const index = parseShardField(shardIndex);
+ const index = parseShardIndexField(shardIndex);
return (
<div className="flex flex-col gap-3 sm:flex-row sm:flex-wrap sm:gap-4">
<StreamActionLog
diff --git a/editor/app/channels/[slug]/components/stages/DownloadStage.tsx b/editor/app/channels/[slug]/components/stages/DownloadStage.tsx
@@ -7,6 +7,7 @@ import { QueueControl } from "../../../../components/QueueControl";
import {
ShardControl,
parseShardField,
+ parseShardIndexField,
type ShardConfigSummary,
} from "../../../../components/ShardControl";
import { cancelJobAction } from "../../../../jobs/actions";
@@ -52,7 +53,7 @@ export function DownloadStage({
missingShard ? String(missingShard.totalShards) : "",
);
const [missingShardIndex, setMissingShardIndex] = useState(
- missingShard ? String(missingShard.shardIndex) : "",
+ missingShard ? String(missingShard.shardIndex + 1) : "",
);
if (!hasUrl) {
@@ -140,7 +141,7 @@ export function DownloadStage({
ignoreArchive,
missingAbortOnError,
parseShardField(missingShardTotal),
- parseShardField(missingShardIndex),
+ parseShardIndexField(missingShardIndex),
)
}
cancelAction={cancelJobAction}
diff --git a/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx b/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx
@@ -11,6 +11,7 @@ import { QueueControl } from "../../../../components/QueueControl";
import {
ShardControl,
parseShardField,
+ parseShardIndexField,
type ShardConfigSummary,
} from "../../../../components/ShardControl";
import { cancelJobAction } from "../../../../jobs/actions";
@@ -181,7 +182,7 @@ function TranscribeMissingSection({
missingShard ? String(missingShard.totalShards) : "",
);
const [missingShardIndex, setMissingShardIndex] = useState(
- missingShard ? String(missingShard.shardIndex) : "",
+ missingShard ? String(missingShard.shardIndex + 1) : "",
);
const wrapperClass = demoted
? "flex flex-col gap-2 rounded border border-zinc-200 dark:border-zinc-800 p-3"
@@ -203,7 +204,7 @@ function TranscribeMissingSection({
missingAudioFormat === "any" ? undefined : missingAudioFormat,
missingAudioFormat !== "any" && missingStrictFormat,
parseShardField(missingShardTotal),
- parseShardField(missingShardIndex),
+ parseShardIndexField(missingShardIndex),
)
}
cancelAction={cancelJobAction}
diff --git a/editor/app/channels/[slug]/shardActions.ts b/editor/app/channels/[slug]/shardActions.ts
@@ -5,8 +5,13 @@ import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
SHARD_OPS,
clearShardConfig,
+ loadShardConfig,
type ShardOp,
} from "yt-dlp-transcript-common/controller/shard";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp";
+import { runWhisperBatch } from "yt-dlp-transcript-common/controller/whisperBatch";
+import { runAvailabilityCheck } from "yt-dlp-transcript-common/controller/checkAvailability";
export type ClearShardResult =
| { ok: true; cleared: boolean }
@@ -23,3 +28,101 @@ export async function clearShardConfigAction(
revalidatePath(`/channels/${slug}`);
return { ok: true, cleared };
}
+
+export type SaveShardResult =
+ | { ok: true; totalShards: number; shardIndex: number; itemCount: number }
+ | { ok: false; error: string };
+
+// Persist a shard slice WITHOUT running the job. Each op's runner is invoked
+// with `saveShardOnly: true`, so the saved slice is computed over the exact
+// same population a real run would use (transcribe = all video dirs, download
+// = the post-prefilter missing set, availability = data dirs) and then returns
+// before doing any work. This backs the "Save shard" button: it lets a slice
+// be pinned up front before dividing work across synced machines, and proves
+// the save path independently of a full run.
+export async function saveShardConfigAction(
+ slug: string,
+ op: ShardOp,
+ total?: number,
+ index?: number,
+): Promise<SaveShardResult> {
+ if (!(SHARD_OPS as readonly string[]).includes(op)) {
+ return { ok: false, error: `Unsupported shard op: ${op}` };
+ }
+ if (
+ total === undefined ||
+ index === undefined ||
+ !Number.isInteger(total) ||
+ !Number.isInteger(index) ||
+ total < 1 ||
+ index < 0 ||
+ index >= total
+ ) {
+ return {
+ ok: false,
+ error: `Enter a valid shard before saving: total ≥ 1 and 0 ≤ index < total (got ${total}/${index}).`,
+ };
+ }
+
+ const paths = getPaths();
+ const noopLog = () => {};
+ const signal = new AbortController().signal;
+
+ try {
+ if (op === "transcribe-missing") {
+ await runWhisperBatch({
+ channelSlug: slug,
+ paths,
+ shardTotal: total,
+ shardIndex: index,
+ saveShardOnly: true,
+ onLog: noopLog,
+ signal,
+ });
+ } else if (op === "download-missing") {
+ const channelConfig = await readChannelConfig(paths, slug);
+ if (!channelConfig) {
+ return { ok: false, error: `Channel "${slug}" not found` };
+ }
+ await runYtdlp({
+ channelSlug: slug,
+ mode: "download-missing",
+ channelConfig,
+ paths,
+ onLog: noopLog,
+ signal,
+ shardTotal: total,
+ shardIndex: index,
+ saveShardOnly: true,
+ });
+ } else {
+ await runAvailabilityCheck({
+ channelSlug: slug,
+ paths,
+ mode: "resume",
+ shardTotal: total,
+ shardIndex: index,
+ saveShardOnly: true,
+ onLog: noopLog,
+ signal,
+ });
+ }
+ } catch (e) {
+ return { ok: false, error: (e as Error).message };
+ }
+
+ const saved = await loadShardConfig(paths, slug, op);
+ revalidatePath(`/channels/${slug}`);
+ if (!saved) {
+ return {
+ ok: false,
+ error: "Shard slice was not saved (no matching population on disk).",
+ };
+ }
+ return {
+ ok: true,
+ totalShards: saved.totalShards,
+ shardIndex: saved.shardIndex,
+ itemCount: saved.items.length,
+ };
+}
diff --git a/editor/app/components/ShardControl.tsx b/editor/app/components/ShardControl.tsx
@@ -3,7 +3,10 @@
import { useRouter } from "next/navigation";
import { useState, useTransition } from "react";
import type { ShardOp } from "yt-dlp-transcript-common/controller/shard";
-import { clearShardConfigAction } from "../channels/[slug]/shardActions";
+import {
+ clearShardConfigAction,
+ saveShardConfigAction,
+} from "../channels/[slug]/shardActions";
export type ShardConfigSummary = {
totalShards: number;
@@ -35,8 +38,19 @@ export function ShardControl({
}: Props) {
const router = useRouter();
const [clearing, startTransition] = useTransition();
+ const [saving, startSaveTransition] = useTransition();
const [error, setError] = useState<string | null>(null);
+ const parsedTotal = parseShardField(total);
+ // The index input is 1-based for humans ("shard 1 of 2"); parseShardIndexField
+ // converts it to the 0-based index the slicing math uses.
+ const parsedIndex = parseShardIndexField(index);
+ const inputsValid =
+ parsedTotal !== undefined &&
+ parsedIndex !== undefined &&
+ parsedTotal >= 1 &&
+ parsedIndex < parsedTotal;
+
function handleClear() {
setError(null);
startTransition(async () => {
@@ -49,6 +63,25 @@ export function ShardControl({
});
}
+ // Persist the slice without running the job (the "Save shard" probe).
+ function handleSave() {
+ setError(null);
+ if (!inputsValid) {
+ setError(
+ "Enter a shard first: a number from 1 to the total (e.g. 1 / 2).",
+ );
+ return;
+ }
+ startSaveTransition(async () => {
+ const r = await saveShardConfigAction(slug, op, parsedTotal, parsedIndex);
+ if (!r.ok) {
+ setError(r.error);
+ return;
+ }
+ router.refresh();
+ });
+ }
+
// Aria-labels intentionally key on `op` (a stable slug) instead of
// actionLabel so they don't substring-match the action button name
// (e.g. "Transcribe missing") and break role-based locators in tests.
@@ -64,22 +97,24 @@ export function ShardControl({
min={1}
step={1}
inputMode="numeric"
- value={total}
- placeholder="N"
- aria-label={`shard ${op} total`}
- onChange={(e) => onTotalChange(e.target.value)}
+ value={index}
+ placeholder="i"
+ aria-label={`shard ${op} index`}
+ title="This shard's number (1-based: 1…total)"
+ onChange={(e) => onIndexChange(e.target.value)}
className="w-14 font-mono px-2 py-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 text-zinc-900 dark:text-zinc-100"
/>
<span>/</span>
<input
type="number"
- min={0}
+ min={1}
step={1}
inputMode="numeric"
- value={index}
- placeholder="i"
- aria-label={`shard ${op} index`}
- onChange={(e) => onIndexChange(e.target.value)}
+ value={total}
+ placeholder="N"
+ aria-label={`shard ${op} total`}
+ title="Total number of shards"
+ onChange={(e) => onTotalChange(e.target.value)}
className="w-14 font-mono px-2 py-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 text-zinc-900 dark:text-zinc-100"
/>
{existing ? (
@@ -87,7 +122,7 @@ export function ShardControl({
aria-label={`shard ${op} saved`}
className="text-zinc-600 dark:text-zinc-400"
>
- saved: {existing.shardIndex}/{existing.totalShards} ({existing.itemCount} items)
+ saved: {existing.shardIndex + 1}/{existing.totalShards} ({existing.itemCount} items)
</span>
) : (
<span
@@ -99,6 +134,15 @@ export function ShardControl({
)}
<button
type="button"
+ onClick={handleSave}
+ disabled={saving}
+ aria-label={`shard ${op} save`}
+ className="px-2 py-0.5 rounded border border-zinc-300 dark:border-zinc-700 text-zinc-600 dark:text-zinc-400 hover:bg-zinc-100 dark:hover:bg-zinc-800 disabled:opacity-40"
+ >
+ {saving ? "Saving…" : "Save"}
+ </button>
+ <button
+ type="button"
onClick={handleClear}
disabled={!existing || clearing}
aria-label={`shard ${op} clear`}
@@ -126,3 +170,12 @@ export function parseShardField(s: string): number | undefined {
if (!Number.isFinite(n) || n < 0) return undefined;
return Math.floor(n);
}
+
+// The shard index input is shown 1-based ("shard 1 of 2"). Convert it to the
+// 0-based index used by computeShardSlice / the server actions. Returns
+// undefined for blank or sub-1 input (0 is not a valid 1-based shard number).
+export function parseShardIndexField(s: string): number | undefined {
+ const n = parseShardField(s);
+ if (n === undefined || n < 1) return undefined;
+ return n - 1;
+}
diff --git a/editor/e2e/shard.spec.ts b/editor/e2e/shard.spec.ts
@@ -4,6 +4,8 @@ import { pathExists, readJson, resetData, resolvePath } from "./helpers";
const AVAILABILITY_CHANNEL = "availability-test";
+// The shard inputs are 1-based in the UI ("shard i of N"); the persisted
+// shardIndex stays 0-based. So filling index "1" → shardIndex 0 on disk.
type ShardFile = {
totalShards: number;
shardIndex: number;
@@ -12,25 +14,25 @@ type ShardFile = {
};
test.describe("Availability check sharding", () => {
- test("shard 2/0 only checks half the videos and persists the slice", async ({
+ test("shard 1/2 only checks half the videos and persists the slice", async ({
page,
}) => {
await resetData("availability-baseline");
await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
await page.getByLabel("shard availability total").first().fill("2");
- await page.getByLabel("shard availability index").first().fill("0");
+ await page.getByLabel("shard availability index").first().fill("1");
await page.getByRole("button", { name: "Resume checks" }).click();
const log = page.getByLabel("Resume availability checks output");
- await expect(log).toContainText("Shard availability: computed slice 0/2", {
+ await expect(log).toContainText("Shard availability: computed slice 1/2", {
timeout: 20_000,
});
await expect(log).toContainText("Availability check done", {
timeout: 20_000,
});
- // 6 fixture videos sorted lex; shard 0/2 takes 3 of them.
+ // 6 fixture videos sorted lex; shard 1 of 2 (0-based index 0) takes 3.
const cfg = await readJson<ShardFile>(
`test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`,
);
@@ -65,24 +67,24 @@ test.describe("Availability check sharding", () => {
await resetData("availability-baseline");
await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
await page.getByLabel("shard availability total").first().fill("2");
- await page.getByLabel("shard availability index").first().fill("0");
+ await page.getByLabel("shard availability index").first().fill("1");
await page.getByRole("button", { name: "Resume checks" }).click();
await expect(
page.getByLabel("Resume availability checks output"),
- ).toContainText("computed slice 0/2", { timeout: 20_000 });
+ ).toContainText("computed slice 1/2", { timeout: 20_000 });
// Re-click — should report "using saved slice" rather than recompute.
await page.getByRole("button", { name: "Resume checks" }).click();
await expect(
page.getByLabel("Resume availability checks output"),
- ).toContainText("using saved slice 0/2", { timeout: 20_000 });
+ ).toContainText("using saved slice 1/2", { timeout: 20_000 });
});
test("Clear button removes the saved shard config", async ({ page }) => {
await resetData("availability-baseline");
await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
await page.getByLabel("shard availability total").first().fill("2");
- await page.getByLabel("shard availability index").first().fill("0");
+ await page.getByLabel("shard availability index").first().fill("1");
await page.getByRole("button", { name: "Resume checks" }).click();
await expect(
page.getByLabel("Resume availability checks output"),
@@ -112,7 +114,7 @@ test.describe("Availability check sharding", () => {
});
test.describe("Transcribe missing sharding", () => {
- test("shard 2/1 only transcribes its half and persists the slice", async ({
+ test("shard 2/2 only transcribes its half and persists the slice", async ({
page,
}) => {
await resetData("one-transcribe-channel-with-audio");
@@ -125,16 +127,15 @@ test.describe("Transcribe missing sharding", () => {
await page
.getByLabel("shard transcribe-missing index")
.first()
- .fill("1");
+ .fill("2");
await page.getByRole("button", { name: "Transcribe missing" }).click();
const log = page.getByLabel("Transcribe missing output");
- await expect(log).toContainText("Shard transcribe-missing: computed slice 1/2", {
+ await expect(log).toContainText("Shard transcribe-missing: computed slice 2/2", {
timeout: 30_000,
});
- // Part C: the job's progress total is scoped to the shard subset, not the
- // whole channel (3 videos sorted, index 1 of 2 → 1 item still needs a
- // transcript).
+ // The job's progress total is scoped to the shard subset, not the whole
+ // channel (3 untranscribed videos sorted, shard 2 of 2 → 1 item).
await expect(log).toContainText(
"Shard transcribe-missing: progress scoped to 1 of 1 shard item(s)",
{ timeout: 30_000 },
@@ -146,15 +147,119 @@ test.describe("Transcribe missing sharding", () => {
);
expect(cfg.totalShards).toBe(2);
expect(cfg.shardIndex).toBe(1);
- // 3 videos (vidA, vidB, vidC) sorted; index 1 of 2 picks 1 of them.
+ // 3 untranscribed videos sorted; shard 2 of 2 picks 1 of them.
+ expect(cfg.items).toHaveLength(1);
+
+ // The saved-shard pill reflects the just-saved config WITHOUT a manual
+ // page reload (StreamActionLog refreshes server data on completion).
+ await expect(
+ page.getByLabel("shard transcribe-missing saved").first(),
+ ).toBeVisible({ timeout: 10_000 });
+
+ // Only the sharded id should have a transcript.
+ let transcribed = 0;
+ for (const id of ["vidA", "vidB", "vidC"]) {
+ if (
+ await pathExists(
+ `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`,
+ )
+ ) {
+ transcribed++;
+ }
+ }
+ expect(transcribed).toBe(1);
+ });
+
+ test("shards only the remaining (untranscribed) videos, not the whole channel", async ({
+ page,
+ }) => {
+ await resetData("one-transcribe-channel-with-audio");
+ // Pre-transcribe vidB so only vidA & vidC remain. A 1-shard slice must then
+ // own 2 items (the remaining set), never 3 (the whole channel).
+ await writeFile(
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/data/vidB/transcript.json",
+ ),
+ JSON.stringify({ cues: [] }),
+ );
+ await page.goto("/channels/test-transcribe");
+
+ await page.getByLabel("shard transcribe-missing total").first().fill("1");
+ await page.getByLabel("shard transcribe-missing index").first().fill("1");
+ await page.getByLabel("shard transcribe-missing save").first().click();
+ await expect(
+ page.getByLabel("shard transcribe-missing saved").first(),
+ ).toBeVisible({ timeout: 10_000 });
+
+ const cfg = await readJson<ShardFile>(
+ "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json",
+ );
+ // The remaining set is {vidA, vidC} — vidB is already transcribed.
+ expect(cfg.items).toHaveLength(2);
+ expect(cfg.items).not.toContain("vidB");
+ });
+});
+
+test.describe("Save shard (no run)", () => {
+ test("Save persists the slice and shows the pill without running", async ({
+ page,
+ }) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await page.goto("/channels/test-transcribe");
+
+ await page.getByLabel("shard transcribe-missing total").first().fill("2");
+ await page.getByLabel("shard transcribe-missing index").first().fill("2");
+
+ // Save-only: persist the slice WITHOUT transcribing.
+ await page.getByLabel("shard transcribe-missing save").first().click();
+
+ // The saved-shard pill appears via router.refresh, no manual reload.
+ await expect(
+ page.getByLabel("shard transcribe-missing saved").first(),
+ ).toBeVisible({ timeout: 10_000 });
+
+ const cfg = await readJson<ShardFile>(
+ "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json",
+ );
+ expect(cfg.totalShards).toBe(2);
+ expect(cfg.shardIndex).toBe(1);
+ // 3 untranscribed videos sorted; shard 2 of 2 owns exactly 1.
expect(cfg.items).toHaveLength(1);
- // Part B: the saved-shard pill reflects the just-saved config WITHOUT a
- // manual page reload (StreamActionLog refreshes server data on completion).
+ // Save-only must NOT have produced any transcript.
+ let transcribed = 0;
+ for (const id of ["vidA", "vidB", "vidC"]) {
+ if (
+ await pathExists(
+ `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`,
+ )
+ ) {
+ transcribed++;
+ }
+ }
+ expect(transcribed).toBe(0);
+ });
+
+ test("a run after Save reuses the saved slice", async ({ page }) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await page.goto("/channels/test-transcribe");
+
+ await page.getByLabel("shard transcribe-missing total").first().fill("2");
+ await page.getByLabel("shard transcribe-missing index").first().fill("2");
+ await page.getByLabel("shard transcribe-missing save").first().click();
await expect(
page.getByLabel("shard transcribe-missing saved").first(),
).toBeVisible({ timeout: 10_000 });
+ // Now actually run — resolveShardItems should report the saved slice.
+ await page.getByRole("button", { name: "Transcribe missing" }).click();
+ const log = page.getByLabel("Transcribe missing output");
+ await expect(log).toContainText(
+ "Shard transcribe-missing: using saved slice 2/2",
+ { timeout: 30_000 },
+ );
+ await expect(log).toContainText("succeeded", { timeout: 30_000 });
+
// Only the sharded id should have a transcript.
let transcribed = 0;
for (const id of ["vidA", "vidB", "vidC"]) {
@@ -174,8 +279,8 @@ test.describe("Download missing sharding", () => {
const channelRoot = "test-transcripts/channels/test-transcribe";
// Make all three videos missing (clear their audio) and set a 3-URL playlist.
- // The shard slices the *missing* set: vidA/vidB/vidC sorted, shard 2/0 owns
- // indices 0 and 2 → vidA, vidC.
+ // The shard slices the *missing* set: vidA/vidB/vidC sorted, shard 1 of 2
+ // (0-based index 0) owns indices 0 and 2 → vidA, vidC.
async function setUpMissing() {
for (const id of ["vidA", "vidB", "vidC"]) {
await rm(resolvePath(`${channelRoot}/data/${id}/audio.m4a`), {
@@ -195,7 +300,7 @@ test.describe("Download missing sharding", () => {
);
}
- test("shard 2/0 persists the missing-set slice before fetching and shows it without reload", async ({
+ test("shard 1/2 persists the missing-set slice before fetching and shows it without reload", async ({
page,
}) => {
await resetData("one-transcribe-channel-with-audio");
@@ -203,16 +308,16 @@ test.describe("Download missing sharding", () => {
await page.goto("/channels/test-transcribe");
await page.getByLabel("shard download-missing total").first().fill("2");
- await page.getByLabel("shard download-missing index").first().fill("0");
+ await page.getByLabel("shard download-missing index").first().fill("1");
await page.getByRole("button", { name: "Download videos" }).click();
const log = page.getByLabel("Download videos output");
// The slice is computed + persisted before any download starts.
await expect(log).toContainText(
- "Shard download-missing: computed slice 0/2",
+ "Shard download-missing: computed slice 1/2",
{ timeout: 30_000 },
);
- // Part C: the job's progress total is scoped to the shard subset.
+ // The job's progress total is scoped to the shard subset.
await expect(log).toContainText(
"Shard download-missing: progress scoped to 2 shard item(s)",
{ timeout: 30_000 },
@@ -230,7 +335,7 @@ test.describe("Download missing sharding", () => {
expect(cfg.items.some((u) => u.includes("vidC"))).toBe(true);
expect(cfg.items.some((u) => u.includes("vidB"))).toBe(false);
- // Part B: saved-shard pill appears without a manual reload.
+ // The saved-shard pill appears without a manual reload.
await expect(
page.getByLabel("shard download-missing saved").first(),
).toBeVisible({ timeout: 10_000 });
@@ -244,11 +349,11 @@ test.describe("Download missing sharding", () => {
await page.goto("/channels/test-transcribe");
await page.getByLabel("shard download-missing total").first().fill("2");
- await page.getByLabel("shard download-missing index").first().fill("0");
+ await page.getByLabel("shard download-missing index").first().fill("1");
await page.getByRole("button", { name: "Download videos" }).click();
await expect(
page.getByLabel("Download videos output"),
- ).toContainText("computed slice 0/2", { timeout: 30_000 });
+ ).toContainText("computed slice 1/2", { timeout: 30_000 });
// Wait for the first run to fully finish — the saved-shard pill appears via
// the post-run refresh — then start a fresh run. (Reload to get a clean
@@ -266,6 +371,6 @@ test.describe("Download missing sharding", () => {
// re-slicing the now-smaller missing set.
await expect(
page.getByLabel("Download videos output"),
- ).toContainText("using saved slice 0/2", { timeout: 30_000 });
+ ).toContainText("using saved slice 1/2", { timeout: 30_000 });
});
});
diff --git a/editor/e2e/whisper.spec.ts b/editor/e2e/whisper.spec.ts
@@ -19,20 +19,19 @@ test("transcribes every audio file with no transcript", async ({ page }) => {
}
});
-test("Transcribe missing skips VTT-only videos instead of failing them", async ({
+test("Transcribe missing excludes VTT-only videos instead of failing them", async ({
page,
}) => {
await resetData("youtube-with-playlist");
await page.goto("/channels/test-youtube");
await page.getByRole("button", { name: "Transcribe missing" }).click();
const log = page.getByLabel("Transcribe missing output");
+ // A video that already has a yt-dlp VTT transcript is not "remaining work",
+ // so it's excluded from the batch up front — never attempted, never failed.
await expect(log).toContainText(
- "Whisper batch: 0 succeeded, 0 failed, 1 skipped, 0 attempted.",
+ "Whisper batch: 0 succeeded, 0 failed, 0 skipped, 0 attempted.",
{ timeout: 30_000 },
);
- await expect(log).toContainText(
- "Transcription for fake00000001 already exists",
- );
const failPath =
"test-transcripts/channels/test-youtube/failed-transcriptions";
