commit f20c42c95d703949265935c7a0af2a58f5a4bd63
parent e7a8d1bad1b0bb7a11934e9653b98a0daa37f34f
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Tue, 12 May 2026 14:06:05 -0400
various fixes
Diffstat:
34 files changed, 1356 insertions(+), 232 deletions(-)
diff --git a/common/bin/retry-failures.ts b/common/bin/retry-failures.ts
@@ -1,26 +0,0 @@
-#!/usr/bin/env tsx
-import { getPaths } from "../lib/paths";
-import { runWhisperBatch } from "../controller/whisperBatch";
-import { parseFlags } from "./_parseFlags";
-
-const flags = parseFlags(process.argv.slice(2));
-const channelSlug = flags.channel;
-if (!channelSlug) {
- console.error("Usage: retry-failures.ts --channel <slug>");
- process.exit(2);
-}
-
-runWhisperBatch({
- channelSlug,
- paths: getPaths(),
- mode: "retry-failures",
-})
- .then((result) => {
- console.log(
- `Done: ${result.succeeded} succeeded, ${result.failed} failed, ${result.skipped} skipped, ${result.attempted} attempted.`,
- );
- })
- .catch((err) => {
- console.error(err);
- process.exit(1);
- });
diff --git a/common/bin/transform.ts b/common/bin/transform.ts
@@ -13,7 +13,6 @@ if (!channelSlug) {
runWhisperBatch({
channelSlug,
paths: getPaths(),
- mode: "all",
})
.then((result) => {
console.log(
diff --git a/common/controller/backfillAvailability.ts b/common/controller/backfillAvailability.ts
@@ -0,0 +1,75 @@
+import path from "node:path";
+import { readdir, readFile } from "node:fs/promises";
+import {
+ availabilityFromJsonField,
+ type AvailabilityRecord,
+} from "../lib/availability";
+import { loadAvailability, writeAvailability } from "../lib/availability-server";
+import type { Paths } from "../lib/paths";
+
+export type BackfillAvailabilityOpts = {
+ channelSlug: string;
+ paths: Paths;
+ onLog?: (msg: string) => void;
+ signal?: AbortSignal;
+};
+
+export type BackfillAvailabilityResult = {
+ wrote: number;
+ skippedExisting: number;
+ skippedNoMeta: number;
+};
+
+export async function backfillAvailabilityFromMetadata({
+ channelSlug,
+ paths,
+ onLog,
+ signal,
+}: BackfillAvailabilityOpts): Promise<BackfillAvailabilityResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const dataDir = path.join(paths.channelsDir, channelSlug, "data");
+ const ids = await readdir(dataDir).catch(() => [] as string[]);
+ let wrote = 0;
+ let skippedExisting = 0;
+ let skippedNoMeta = 0;
+
+ for (const id of ids) {
+ if (signal?.aborted) break;
+ const videoDir = path.join(dataDir, id);
+ if (await loadAvailability(videoDir)) {
+ skippedExisting++;
+ continue;
+ }
+ let metaRaw: string;
+ try {
+ metaRaw = await readFile(
+ path.join(videoDir, "metadata.info.json"),
+ "utf8",
+ );
+ } catch {
+ skippedNoMeta++;
+ continue;
+ }
+ let parsed: { availability?: unknown; webpage_url?: unknown };
+ try {
+ parsed = JSON.parse(metaRaw);
+ } catch {
+ skippedNoMeta++;
+ continue;
+ }
+ const record: AvailabilityRecord = {
+ checkedAt: new Date().toISOString(),
+ availability: availabilityFromJsonField(parsed.availability),
+ };
+ if (typeof parsed.webpage_url === "string") {
+ record.webpageUrl = parsed.webpage_url;
+ }
+ await writeAvailability(videoDir, record);
+ wrote++;
+ }
+
+ log(
+ `Availability backfill: wrote ${wrote}, skipped ${skippedExisting} existing, ${skippedNoMeta} without metadata.`,
+ );
+ return { wrote, skippedExisting, skippedNoMeta };
+}
diff --git a/common/controller/checkAvailability.ts b/common/controller/checkAvailability.ts
@@ -11,6 +11,7 @@ import {
} from "../lib/availability";
import { loadAvailability, writeAvailability } from "../lib/availability-server";
import { readChannelConfig } from "./channels";
+import { resolveShardItems } from "./shard";
import type { Paths } from "../lib/paths";
export type CheckAvailabilityMode =
@@ -29,6 +30,8 @@ export type CheckAvailabilityOpts = {
paths: Paths;
mode: CheckAvailabilityMode;
concurrency?: number;
+ shardTotal?: number;
+ shardIndex?: number;
onLog?: (msg: string) => void;
signal?: AbortSignal;
};
@@ -86,6 +89,8 @@ export async function runAvailabilityCheck({
paths,
mode,
concurrency,
+ shardTotal,
+ shardIndex,
onLog,
signal,
}: CheckAvailabilityOpts): Promise<CheckAvailabilityResult> {
@@ -101,15 +106,35 @@ export async function runAvailabilityCheck({
const limit = pLimit(concurrency && concurrency > 0 ? Math.floor(concurrency) : 1);
- const videoDirs = await readdir(dataDir).catch(() => [] as string[]);
+ const allDirs = await readdir(dataDir).catch(() => [] as string[]);
+ const { items: videoDirs } = await resolveShardItems({
+ paths,
+ slug: channelSlug,
+ op: "availability",
+ fullItems: allDirs,
+ totalShards: shardTotal,
+ shardIndex: shardIndex,
+ onLog: log,
+ });
let attempted = 0;
let checked = 0;
let skipped = 0;
const byStatus = emptyByStatus();
+ // Pre-scan: count how many already have a parsed availability.json so the
+ // "skipped" total in the summary is unambiguous (no, we are not skipping
+ // every video that has a transcript — only those with a valid sidecar).
+ let alreadyChecked = 0;
+ await Promise.all(
+ videoDirs.map(async (id) => {
+ const rec = await loadAvailability(path.join(dataDir, id));
+ if (rec) alreadyChecked++;
+ }),
+ );
+ const unchecked = videoDirs.length - alreadyChecked;
log(
- `Availability check (${mode}): scanning ${videoDirs.length} video dir(s) under ${dataDir}`,
+ `Availability check (${mode}): ${videoDirs.length} video dir(s) under ${dataDir} — ${alreadyChecked} already checked, ${unchecked} unchecked. concurrency=${concurrency && concurrency > 0 ? Math.floor(concurrency) : 1}`,
);
await Promise.all(
diff --git a/common/controller/cleanAudioFromTranscribed.ts b/common/controller/cleanAudioFromTranscribed.ts
@@ -43,7 +43,8 @@ export async function cleanAudioFromTranscribed({
(e) =>
e.startsWith("audio.") &&
!e.includes(".tmp-") &&
- !e.endsWith(".info.json"),
+ !e.endsWith(".info.json") &&
+ !e.endsWith(".part"),
);
if (audioFiles.length === 0) continue;
for (const f of audioFiles) {
diff --git a/common/controller/cleanExtraAudioFormats.ts b/common/controller/cleanExtraAudioFormats.ts
@@ -56,7 +56,8 @@ export async function cleanExtraAudioFormats({
e.startsWith("audio.") &&
e !== targetAudioFile &&
!e.includes(".tmp-") &&
- !e.endsWith(".info.json"),
+ !e.endsWith(".info.json") &&
+ !e.endsWith(".part"),
);
if (extras.length === 0) continue;
for (const f of extras) {
diff --git a/common/controller/failedTranscriptions.ts b/common/controller/failedTranscriptions.ts
@@ -38,3 +38,21 @@ export async function pruneFailedTranscriptions(
pruned: original.length - filtered.length,
};
}
+
+export async function clearFailedTranscriptions(
+ paths: Paths,
+ slug: string,
+): Promise<{ cleared: number }> {
+ const file = failedTranscriptionsFile(paths, slug);
+ let cleared = 0;
+ try {
+ const raw = await readFile(file, "utf8");
+ cleared = raw.split("\n").filter(Boolean).length;
+ } catch {
+ return { cleared: 0 };
+ }
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, "");
+ await rename(tmp, file);
+ return { cleared };
+}
diff --git a/common/controller/shard.ts b/common/controller/shard.ts
@@ -0,0 +1,168 @@
+import path from "node:path";
+import { readFile, rename, unlink, writeFile } from "node:fs/promises";
+import type { Paths } from "../lib/paths";
+
+export const SHARD_OPS = [
+ "download-missing",
+ "transcribe-missing",
+ "availability",
+] as const;
+export type ShardOp = (typeof SHARD_OPS)[number];
+
+export type ShardConfig = {
+ totalShards: number;
+ shardIndex: number;
+ items: string[];
+ createdAt: string;
+};
+
+export function shardFile(paths: Paths, slug: string, op: ShardOp): string {
+ return path.join(paths.channelsDir, slug, `shard-${op}.json`);
+}
+
+export async function loadShardConfig(
+ paths: Paths,
+ slug: string,
+ op: ShardOp,
+): Promise<ShardConfig | null> {
+ try {
+ const raw = await readFile(shardFile(paths, slug, op), "utf8");
+ const parsed = JSON.parse(raw) as Partial<ShardConfig>;
+ if (
+ typeof parsed?.totalShards === "number" &&
+ parsed.totalShards >= 1 &&
+ typeof parsed?.shardIndex === "number" &&
+ parsed.shardIndex >= 0 &&
+ parsed.shardIndex < parsed.totalShards &&
+ Array.isArray(parsed.items) &&
+ parsed.items.every((x) => typeof x === "string") &&
+ typeof parsed.createdAt === "string"
+ ) {
+ return parsed as ShardConfig;
+ }
+ return null;
+ } catch {
+ return null;
+ }
+}
+
+export async function saveShardConfig(
+ paths: Paths,
+ slug: string,
+ op: ShardOp,
+ cfg: ShardConfig,
+): Promise<void> {
+ const file = shardFile(paths, slug, op);
+ const tmp = `${file}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(cfg, null, 2) + "\n");
+ await rename(tmp, file);
+}
+
+export async function clearShardConfig(
+ paths: Paths,
+ slug: string,
+ op: ShardOp,
+): Promise<boolean> {
+ try {
+ await unlink(shardFile(paths, slug, op));
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+// Sort lexicographically first so the slice is stable across machines that
+// happen to readdir in different orders. Same (items, total, index) on every
+// box → same slice.
+export function computeShardSlice(
+ items: string[],
+ totalShards: number,
+ shardIndex: number,
+): string[] {
+ if (totalShards < 1 || shardIndex < 0 || shardIndex >= totalShards) {
+ throw new Error(
+ `Invalid shard config: total=${totalShards}, index=${shardIndex}`,
+ );
+ }
+ const sorted = [...items].sort();
+ return sorted.filter((_, i) => i % totalShards === shardIndex);
+}
+
+export type ResolveShardOpts = {
+ paths: Paths;
+ slug: string;
+ op: ShardOp;
+ fullItems: string[];
+ // When undefined: no inputs provided this run. If a saved slice exists,
+ // operate on it; otherwise fall through to fullItems.
+ totalShards?: number;
+ shardIndex?: number;
+ onLog?: (msg: string) => void;
+};
+
+export type ResolveShardResult = {
+ items: string[];
+ source: "saved" | "computed" | "full";
+ config: ShardConfig | null;
+};
+
+// Decide what to operate on for a sharded run:
+// 1. If totalShards+shardIndex provided AND a saved slice exists with the
+// same (total, index): return saved (resume).
+// 2. Else if totalShards+shardIndex provided: compute, persist, return.
+// 3. Else if a saved slice exists: return saved (resume without re-supplying inputs).
+// 4. Else: return full list.
+export async function resolveShardItems({
+ paths,
+ slug,
+ op,
+ fullItems,
+ totalShards,
+ shardIndex,
+ onLog,
+}: ResolveShardOpts): Promise<ResolveShardResult> {
+ const log = onLog ?? ((m: string) => console.log(m));
+ const saved = await loadShardConfig(paths, slug, op);
+ const haveInputs =
+ totalShards !== undefined &&
+ shardIndex !== undefined &&
+ totalShards >= 1 &&
+ shardIndex >= 0 &&
+ shardIndex < totalShards;
+
+ if (
+ haveInputs &&
+ saved &&
+ saved.totalShards === totalShards &&
+ saved.shardIndex === shardIndex
+ ) {
+ log(
+ `Shard ${op}: using saved slice ${saved.shardIndex}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
+ );
+ return { items: saved.items, source: "saved", config: saved };
+ }
+
+ if (haveInputs) {
+ const slice = computeShardSlice(fullItems, totalShards!, shardIndex!);
+ const cfg: ShardConfig = {
+ totalShards: totalShards!,
+ shardIndex: shardIndex!,
+ items: slice,
+ createdAt: new Date().toISOString(),
+ };
+ await saveShardConfig(paths, slug, op, cfg);
+ log(
+ `Shard ${op}: computed slice ${shardIndex}/${totalShards} (${slice.length} of ${fullItems.length} item(s)); persisted.`,
+ );
+ return { items: slice, source: "computed", config: cfg };
+ }
+
+ if (saved) {
+ log(
+ `Shard ${op}: using saved slice ${saved.shardIndex}/${saved.totalShards} (${saved.items.length} item(s), saved ${saved.createdAt}).`,
+ );
+ return { items: saved.items, source: "saved", config: saved };
+ }
+
+ return { items: fullItems, source: "full", config: null };
+}
diff --git a/common/controller/transcodeFailures.ts b/common/controller/transcodeFailures.ts
@@ -23,7 +23,8 @@ function pickSourceAudio(
e.startsWith("audio.") &&
e !== target &&
!e.includes(".tmp-") &&
- !e.endsWith(".info.json"),
+ !e.endsWith(".info.json") &&
+ !e.endsWith(".part"),
);
if (candidates.length === 0) return null;
for (const preferred of AUDIO_PREFERENCE) {
diff --git a/common/controller/transcribeOne.ts b/common/controller/transcribeOne.ts
@@ -23,7 +23,9 @@ async function resolveAudioFile(
if (await pathExists(path.join(videoDir, requested))) return requested;
if (strict) return null;
const entries = await readdir(videoDir);
- const candidates = entries.filter((e) => e.startsWith("audio."));
+ const candidates = entries.filter(
+ (e) => e.startsWith("audio.") && !e.endsWith(".part"),
+ );
if (candidates.length === 0) return null;
for (const preferred of AUDIO_PREFERENCE) {
if (candidates.includes(preferred)) return preferred;
diff --git a/common/controller/whisperBatch.ts b/common/controller/whisperBatch.ts
@@ -4,21 +4,20 @@ import pLimit from "p-limit";
import type { Paths } from "../lib/paths";
import type { AudioFormat } from "../lib/channelConfig";
import { transcribeOneVideo } from "./transcribeOne";
-import { pruneFailedTranscriptions } from "./failedTranscriptions";
+import { resolveShardItems } from "./shard";
const { pathExists, readdir, appendFile, readFile, ensureFile } = fs;
-export type WhisperBatchMode = "all" | "retry-failures";
-
export type WhisperBatchOptions = {
channelSlug: string;
paths: Paths;
- mode: WhisperBatchMode;
concurrency?: number;
audioFilename?: string;
audioFormat?: AudioFormat;
strictAudioFormat?: boolean;
reverse?: boolean;
+ shardTotal?: number;
+ shardIndex?: number;
onLog?: (msg: string) => void;
signal?: AbortSignal;
};
@@ -33,12 +32,13 @@ export type WhisperBatchResult = {
export async function runWhisperBatch({
channelSlug,
paths,
- mode,
concurrency,
audioFilename,
audioFormat,
strictAudioFormat = false,
reverse = false,
+ shardTotal,
+ shardIndex,
onLog,
signal,
}: WhisperBatchOptions): Promise<WhisperBatchResult> {
@@ -57,16 +57,20 @@ export async function runWhisperBatch({
.split("\n")
.filter(Boolean);
- let videoDirs: string[];
- if (mode === "retry-failures") {
- videoDirs = [...originalFailures];
- } else {
- videoDirs = await readdir(dataDir);
- }
+ const allDirs = await readdir(dataDir);
+ const { items: shardItems } = await resolveShardItems({
+ paths,
+ slug: channelSlug,
+ op: "transcribe-missing",
+ fullItems: allDirs,
+ totalShards: shardTotal,
+ shardIndex: shardIndex,
+ onLog: log,
+ });
+ const videoDirs = [...shardItems];
if (reverse) videoDirs.reverse();
const failedSet = new Set<string>(originalFailures);
- const succeeded = new Set<string>();
let attempted = 0;
let succeededCount = 0;
@@ -81,24 +85,14 @@ export async function runWhisperBatch({
return;
}
const videoPath = path.join(dataDir, videoDir);
- if (mode === "all" && failedSet.has(videoDir)) {
+ if (failedSet.has(videoDir)) {
log(`Skipping previously failed transcription for ${videoDir}`);
skipped++;
return;
}
- if (mode === "retry-failures" && !(await pathExists(videoPath))) {
- log(`Skipping ${videoDir}: directory no longer exists`);
- skipped++;
- succeeded.add(videoDir);
- return;
- }
if (await pathExists(path.join(videoPath, "transcript.json"))) {
log(`Transcription for ${videoDir} already exists`);
skipped++;
- if (mode === "retry-failures") {
- // already-good entry — drop it from the failure list too
- succeeded.add(videoDir);
- }
return;
}
if (
@@ -122,7 +116,6 @@ export async function runWhisperBatch({
onLog: log,
signal,
});
- succeeded.add(videoDir);
succeededCount++;
} catch (err) {
if (signal?.aborted) {
@@ -131,24 +124,12 @@ export async function runWhisperBatch({
return;
}
log(`FAILED TO TRANSCRIBE ${videoDir}: ${String(err)}`);
- if (mode === "all") {
- await appendFile(failureListFile, `${videoDir}\n`);
- }
+ await appendFile(failureListFile, `${videoDir}\n`);
failed++;
}
}),
),
);
- if (mode === "retry-failures") {
- const { remaining, pruned } = await pruneFailedTranscriptions(
- failureListFile,
- succeeded,
- );
- log(
- `Pruned ${pruned} from failed-transcriptions; ${remaining} remain.`,
- );
- }
-
return { attempted, succeeded: succeededCount, failed, skipped };
}
diff --git a/common/lib/platform.ts b/common/lib/platform.ts
@@ -26,3 +26,7 @@ export function platformQueueKey(
): string {
return `platform:${platform ?? "unknown"}`;
}
+
+// System-wide queue for resource-bound local jobs (whisper, ffmpeg).
+// Channels share this queue so two heavy local jobs never run in parallel.
+export const TRANSCRIPTION_QUEUE = "transcription";
diff --git a/common/lib/videoStatus.ts b/common/lib/videoStatus.ts
@@ -38,7 +38,10 @@ export async function readVideoFiles(
(e) =>
e.startsWith("audio.") &&
!e.includes(".tmp-") &&
- !e.endsWith(".info.json"),
+ !e.endsWith(".info.json") &&
+ // yt-dlp writes audio.<ext>.part while downloading; treat those as
+ // incomplete so prefilters re-invoke yt-dlp (which resumes via -c).
+ !e.endsWith(".part"),
);
let isUntranscribable = false;
if (hasWhisper && opts.checkUntranscribable) {
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -16,6 +16,8 @@ import {
type ChannelHandling,
} from "../lib/channelConfig";
import type { Paths } from "../lib/paths";
+import { backfillAvailabilityFromMetadata } from "../controller/backfillAvailability";
+import { resolveShardItems } from "../controller/shard";
export type YtdlpMode =
| "store-playlist"
@@ -38,6 +40,10 @@ export type RunYtdlpOpts = {
// --abort-on-error so yt-dlp continues past per-video failures. Defaults
// to true to preserve historical behavior.
abortOnError?: boolean;
+ // download-missing only: split the playlist N ways and only fetch the Mth
+ // shard. Saved/resumed via channels/<slug>/shard-download-missing.json.
+ shardTotal?: number;
+ shardIndex?: number;
// download-one-audio only: full webpage URL of the single video to fetch.
singleVideoUrl?: string;
// download-one-audio only: override channelConfig.audioFormat for this run.
@@ -227,6 +233,7 @@ async function downloadFromPlaylist(opts: RunYtdlpOpts): Promise<void> {
await rm(toFetchPath, { force: true });
await touchLastFullDownload(opts);
+ await safeBackfillAvailability(opts);
}
async function downloadMissing(opts: RunYtdlpOpts): Promise<void> {
@@ -284,14 +291,24 @@ async function downloadMissing(opts: RunYtdlpOpts): Promise<void> {
}.\n`,
);
- if (tofetch.length === 0) {
+ const { items: shardedTofetch } = await resolveShardItems({
+ paths: opts.paths,
+ slug: opts.channelSlug,
+ op: "download-missing",
+ fullItems: tofetch,
+ totalShards: opts.shardTotal,
+ shardIndex: opts.shardIndex,
+ onLog: (m) => opts.onLog(`${m}\n`),
+ });
+
+ if (shardedTofetch.length === 0) {
await rm(toFetchPath, { force: true });
opts.onLog("Nothing to fetch.\n");
await touchLastFullDownload(opts);
return;
}
- await writeFile(toFetchPath, tofetch.join("\n") + "\n");
+ await writeFile(toFetchPath, shardedTofetch.join("\n") + "\n");
if (opts.ignoreArchive) {
opts.onLog(
@@ -316,6 +333,7 @@ async function downloadMissing(opts: RunYtdlpOpts): Promise<void> {
await rm(toFetchPath, { force: true });
await touchLastFullDownload(opts);
+ await safeBackfillAvailability(opts);
}
async function downloadOneAudio(opts: RunYtdlpOpts): Promise<void> {
@@ -343,6 +361,7 @@ async function downloadOneAudio(opts: RunYtdlpOpts): Promise<void> {
opts.singleVideoUrl,
];
await runChildAndStream(opts, root, args);
+ await safeBackfillAvailability(opts);
}
export async function destinationExists(
@@ -363,7 +382,9 @@ export async function destinationExists(
// re-fetch it before whisper runs. YouTube channels expect a .vtt; an audio
// file alone shouldn't suppress the next sync.
if (handling === "transcribe") {
- return entries.some((e) => e.startsWith("audio."));
+ return entries.some(
+ (e) => e.startsWith("audio.") && !e.endsWith(".part"),
+ );
}
return false;
}
@@ -388,6 +409,23 @@ async function sync(opts: RunYtdlpOpts): Promise<void> {
];
await runChildAndStream(opts, root, args);
await touchLastSync(opts);
+ await safeBackfillAvailability(opts);
+}
+
+async function safeBackfillAvailability(opts: RunYtdlpOpts): Promise<void> {
+ // Best-effort: write availability.json for any new video dir from this
+ // download whose metadata.info.json has an availability field. Don't fail
+ // the download if the sweep errors.
+ try {
+ await backfillAvailabilityFromMetadata({
+ channelSlug: opts.channelSlug,
+ paths: opts.paths,
+ onLog: opts.onLog,
+ signal: opts.signal,
+ });
+ } catch (err) {
+ opts.onLog(`Availability backfill skipped: ${(err as Error).message}\n`);
+ }
}
async function runChildAndStream(
diff --git a/editor/app/channels/[slug]/availabilityActions.ts b/editor/app/channels/[slug]/availabilityActions.ts
@@ -3,6 +3,11 @@
import { revalidatePath } from "next/cache";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
+ detectPlatform,
+ platformQueueKey,
+} from "yt-dlp-transcript-common/lib/platform";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import {
CHECK_AVAILABILITY_MODES,
runAvailabilityCheck,
type CheckAvailabilityMode,
@@ -12,10 +17,6 @@ import {
type StreamActionResult,
} from "yt-dlp-transcript-common/jobs/streamCommand";
-function defaultQueueKey(slug: string): string {
- return `channel:${slug}`;
-}
-
function sanitizeConcurrency(c: number | undefined): number | undefined {
if (c === undefined) return undefined;
if (!Number.isFinite(c) || c < 1) return undefined;
@@ -27,15 +28,23 @@ export async function checkAvailabilityAction(
mode: CheckAvailabilityMode,
queueKey?: string,
concurrency?: number,
+ shardTotal?: number,
+ shardIndex?: number,
): Promise<StreamActionResult> {
if (!(CHECK_AVAILABILITY_MODES as string[]).includes(mode)) {
return { ok: false, error: `Unsupported mode: ${mode}` };
}
const paths = getPaths();
+ const config = await readChannelConfig(paths, slug);
+ // Default to the platform queue so availability checks share the per-source
+ // rate-limit budget with downloads instead of fighting them.
+ const defaultQueue = platformQueueKey(
+ config?.platform ?? detectPlatform(config?.url),
+ );
const limit = sanitizeConcurrency(concurrency);
return runManagedFunction({
kind: "check-availability",
- queueKey: queueKey === undefined ? defaultQueueKey(slug) : queueKey.trim(),
+ queueKey: queueKey === undefined ? defaultQueue : queueKey.trim(),
paths,
channelSlug: slug,
fn: async (onLog, signal) => {
@@ -44,6 +53,8 @@ export async function checkAvailabilityAction(
paths,
mode,
concurrency: limit,
+ shardTotal,
+ shardIndex,
onLog,
signal,
});
diff --git a/editor/app/channels/[slug]/components/stages/DiagnosticsStage.tsx b/editor/app/channels/[slug]/components/stages/DiagnosticsStage.tsx
@@ -11,8 +11,22 @@ import { verifyAction, type VerifyResult } from "../../whisperActions";
import { checkAvailabilityAction } from "../../availabilityActions";
import { cancelJobAction } from "../../../../jobs/actions";
import { gotoVideoAction } from "../../../actions";
+import { ConcurrencyControl } from "../../../../components/ConcurrencyControl";
+import {
+ ShardControl,
+ parseShardField,
+ type ShardConfigSummary,
+} from "../../../../components/ShardControl";
import { VideoIdList } from "../VideoIdList";
+function parseConcurrency(s: string): number | undefined {
+ const trimmed = s.trim();
+ if (trimmed === "") return undefined;
+ const n = Number(trimmed);
+ if (!Number.isFinite(n) || n < 1) return undefined;
+ return Math.floor(n);
+}
+
type DiagnosticBucket = {
ids: string[];
label: string;
@@ -28,6 +42,7 @@ type Props = {
duplicateDirIds: string[];
totals: { videos: number; transcribed: number; downloaded: number };
availability: AvailabilitySnapshot;
+ availabilityShard: ShardConfigSummary | null;
};
export function DiagnosticsStage({
@@ -38,6 +53,7 @@ export function DiagnosticsStage({
duplicateDirIds,
totals,
availability,
+ availabilityShard,
}: Props) {
const buckets: DiagnosticBucket[] = [
{
@@ -117,7 +133,7 @@ export function DiagnosticsStage({
title="Availability checks"
desc="Probe yt-dlp to detect videos that have been deleted, made private, or restricted by the uploader. Results write to channels/<slug>/data/<id>/availability.json."
/>
- <AvailabilityButtons slug={slug} />
+ <AvailabilityButtons slug={slug} shard={availabilityShard} />
<AvailabilitySummary slug={slug} availability={availability} />
</div>
<div className="flex flex-col gap-2" aria-label="channel health">
@@ -246,29 +262,91 @@ function Heading({ title, desc }: { title: string; desc: string }) {
);
}
-function AvailabilityButtons({ slug }: { slug: string }) {
+function AvailabilityButtons({
+ slug,
+ shard,
+}: {
+ slug: string;
+ shard: ShardConfigSummary | null;
+}) {
+ // One concurrency + shard config shared across the three modes — they all
+ // iterate the same data dir. Default concurrency 1 to stay polite to the
+ // platform.
+ const [concurrency, setConcurrency] = useState("");
+ const [shardTotal, setShardTotal] = useState(
+ shard ? String(shard.totalShards) : "",
+ );
+ const [shardIndex, setShardIndex] = useState(
+ shard ? String(shard.shardIndex) : "",
+ );
+ const extra = (
+ <>
+ <ConcurrencyControl
+ value={concurrency}
+ onChange={setConcurrency}
+ defaultLimit={1}
+ actionLabel="availability checks"
+ />
+ <ShardControl
+ slug={slug}
+ op="availability"
+ actionLabel="availability checks"
+ existing={shard}
+ total={shardTotal}
+ onTotalChange={setShardTotal}
+ index={shardIndex}
+ onIndexChange={setShardIndex}
+ />
+ </>
+ );
+ const limit = parseConcurrency(concurrency);
+ const total = parseShardField(shardTotal);
+ const index = parseShardField(shardIndex);
return (
<div className="flex flex-col gap-3 sm:flex-row sm:flex-wrap sm:gap-4">
<StreamActionLog
- trigger={() => checkAvailabilityAction(slug, "resume")}
+ trigger={() =>
+ checkAvailabilityAction(slug, "resume", undefined, limit, total, index)
+ }
cancelAction={cancelJobAction}
buttonLabel="Resume checks"
runningLabel="Checking…"
label="Resume availability checks"
+ extraControls={extra}
/>
<StreamActionLog
- trigger={() => checkAvailabilityAction(slug, "recheck-non-deleted")}
+ trigger={() =>
+ checkAvailabilityAction(
+ slug,
+ "recheck-non-deleted",
+ undefined,
+ limit,
+ total,
+ index,
+ )
+ }
cancelAction={cancelJobAction}
buttonLabel="Recheck non-deleted"
runningLabel="Rechecking…"
label="Recheck non-deleted availability"
+ extraControls={extra}
/>
<StreamActionLog
- trigger={() => checkAvailabilityAction(slug, "recheck-all")}
+ trigger={() =>
+ checkAvailabilityAction(
+ slug,
+ "recheck-all",
+ undefined,
+ limit,
+ total,
+ index,
+ )
+ }
cancelAction={cancelJobAction}
buttonLabel="Recheck all"
runningLabel="Rechecking…"
label="Recheck all availability"
+ extraControls={extra}
/>
</div>
);
diff --git a/editor/app/channels/[slug]/components/stages/DownloadStage.tsx b/editor/app/channels/[slug]/components/stages/DownloadStage.tsx
@@ -3,6 +3,11 @@
import { useState } from "react";
import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
import { QueueControl } from "../../../../components/QueueControl";
+import {
+ ShardControl,
+ parseShardField,
+ type ShardConfigSummary,
+} from "../../../../components/ShardControl";
import { cancelJobAction } from "../../../../jobs/actions";
import {
downloadAction,
@@ -17,6 +22,7 @@ type Props = {
existingQueues: string[];
undownloadedIds: string[];
noTranscriptIds: string[];
+ missingShard: ShardConfigSummary | null;
};
export function DownloadStage({
@@ -26,12 +32,19 @@ export function DownloadStage({
existingQueues,
undownloadedIds,
noTranscriptIds,
+ missingShard,
}: Props) {
const [downloadQueue, setDownloadQueue] = useState(defaultQueueKey);
const [missingQueue, setMissingQueue] = useState(defaultQueueKey);
const [ignoreArchive, setIgnoreArchive] = useState(false);
const [downloadAbortOnError, setDownloadAbortOnError] = useState(true);
const [missingAbortOnError, setMissingAbortOnError] = useState(true);
+ const [missingShardTotal, setMissingShardTotal] = useState(
+ missingShard ? String(missingShard.totalShards) : "",
+ );
+ const [missingShardIndex, setMissingShardIndex] = useState(
+ missingShard ? String(missingShard.shardIndex) : "",
+ );
if (!hasUrl) {
return (
@@ -113,19 +126,33 @@ export function DownloadStage({
missingQueue,
ignoreArchive,
missingAbortOnError,
+ parseShardField(missingShardTotal),
+ parseShardField(missingShardIndex),
)
}
cancelAction={cancelJobAction}
buttonLabel="Download videos"
runningLabel="Downloading missing…"
extraControls={
- <QueueControl
- value={missingQueue}
- onChange={setMissingQueue}
- defaultQueueKey={defaultQueueKey}
- existingQueues={existingQueues}
- actionLabel="Download videos"
- />
+ <>
+ <QueueControl
+ value={missingQueue}
+ onChange={setMissingQueue}
+ defaultQueueKey={defaultQueueKey}
+ existingQueues={existingQueues}
+ actionLabel="Download videos"
+ />
+ <ShardControl
+ slug={slug}
+ op="download-missing"
+ actionLabel="Download videos"
+ existing={missingShard}
+ total={missingShardTotal}
+ onTotalChange={setMissingShardTotal}
+ index={missingShardIndex}
+ onIndexChange={setMissingShardIndex}
+ />
+ </>
}
/>
</div>
diff --git a/editor/app/channels/[slug]/components/stages/TranscodeStage.tsx b/editor/app/channels/[slug]/components/stages/TranscodeStage.tsx
@@ -19,6 +19,7 @@ type Props = {
existingQueues: string[];
failedTranscodingIds: string[];
untranscodedIds: string[];
+ transcodeDefaultQueueKey: string;
};
export function TranscodeStage({
@@ -26,10 +27,13 @@ export function TranscodeStage({
existingQueues,
failedTranscodingIds,
untranscodedIds,
+ transcodeDefaultQueueKey,
}: Props) {
- const defaultQueueKey = `channel:${slug}`;
+ // Filesystem-only ops (remove) stay on the channel queue; transcoding itself
+ // is local-resource-bound and shares the transcription queue.
+ const removeDefaultQueueKey = `channel:${slug}`;
const [transcodeFormat, setTranscodeFormat] = useState<AudioFormat>("mp3");
- const [transcodeQueue, setTranscodeQueue] = useState(defaultQueueKey);
+ const [transcodeQueue, setTranscodeQueue] = useState(transcodeDefaultQueueKey);
return (
<div className="flex flex-col gap-6">
<div className="flex flex-col gap-2">
@@ -66,7 +70,7 @@ export function TranscodeStage({
<QueueControl
value={transcodeQueue}
onChange={setTranscodeQueue}
- defaultQueueKey={defaultQueueKey}
+ defaultQueueKey={transcodeDefaultQueueKey}
existingQueues={existingQueues}
actionLabel="Transcode failures"
/>
@@ -78,7 +82,7 @@ export function TranscodeStage({
slug={slug}
ids={failedTranscodingIds}
existingQueues={existingQueues}
- defaultQueueKey={defaultQueueKey}
+ defaultQueueKey={removeDefaultQueueKey}
/>
</div>
);
diff --git a/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx b/editor/app/channels/[slug]/components/stages/TranscribeStage.tsx
@@ -8,9 +8,14 @@ import {
} from "yt-dlp-transcript-common/lib/channelConfig";
import { ConcurrencyControl } from "../../../../components/ConcurrencyControl";
import { QueueControl } from "../../../../components/QueueControl";
+import {
+ ShardControl,
+ parseShardField,
+ type ShardConfigSummary,
+} from "../../../../components/ShardControl";
import { cancelJobAction } from "../../../../jobs/actions";
import {
- retryFailuresAction,
+ clearFailedTranscriptionsAction,
transcribeMissingAction,
} from "../../whisperActions";
import { VideoIdList } from "../VideoIdList";
@@ -31,6 +36,8 @@ type Props = {
failedVideoIds: string[];
downloadedNoTranscriptIds: string[];
defaultConcurrency: number;
+ defaultQueueKey: string;
+ missingShard: ShardConfigSummary | null;
};
export function TranscribeStage({
@@ -39,26 +46,29 @@ export function TranscribeStage({
failedVideoIds,
downloadedNoTranscriptIds,
defaultConcurrency,
+ defaultQueueKey,
+ missingShard,
}: Props) {
- const defaultQueueKey = `channel:${slug}`;
+ const channelQueueKey = `channel:${slug}`;
const [missingQueue, setMissingQueue] = useState(defaultQueueKey);
const [missingConcurrency, setMissingConcurrency] = useState("");
const [missingReverse, setMissingReverse] = useState(false);
const [missingAudioFormat, setMissingAudioFormat] =
useState<AudioFormatChoice>("any");
const [missingStrictFormat, setMissingStrictFormat] = useState(false);
- const [retryQueue, setRetryQueue] = useState(defaultQueueKey);
- const [retryConcurrency, setRetryConcurrency] = useState("");
- const [retryAudioFormat, setRetryAudioFormat] =
- useState<AudioFormatChoice>("any");
- const [retryStrictFormat, setRetryStrictFormat] = useState(false);
+ const [missingShardTotal, setMissingShardTotal] = useState(
+ missingShard ? String(missingShard.totalShards) : "",
+ );
+ const [missingShardIndex, setMissingShardIndex] = useState(
+ missingShard ? String(missingShard.shardIndex) : "",
+ );
return (
<div className="flex flex-col gap-6">
<div className="flex flex-col gap-2">
<Heading
title="Transcribe missing"
- desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions."
+ desc="Run whisper-cli over every video that has audio but no transcript.json. Failures append to channels/<slug>/failed-transcriptions and are skipped on the next run until you clear the list."
/>
<StreamActionLog
trigger={() =>
@@ -69,6 +79,8 @@ export function TranscribeStage({
missingReverse,
missingAudioFormat === "any" ? undefined : missingAudioFormat,
missingAudioFormat !== "any" && missingStrictFormat,
+ parseShardField(missingShardTotal),
+ parseShardField(missingShardIndex),
)
}
cancelAction={cancelJobAction}
@@ -105,6 +117,16 @@ export function TranscribeStage({
/>
Reverse
</label>
+ <ShardControl
+ slug={slug}
+ op="transcribe-missing"
+ actionLabel="Transcribe missing"
+ existing={missingShard}
+ total={missingShardTotal}
+ onTotalChange={setMissingShardTotal}
+ index={missingShardIndex}
+ onIndexChange={setMissingShardIndex}
+ />
</>
}
/>
@@ -113,58 +135,88 @@ export function TranscribeStage({
ids={downloadedNoTranscriptIds}
/>
</div>
- <div className="flex flex-col gap-2">
- <Heading
- title="Retry failures"
- desc="Re-run whisper for any video listed below; successes will be removed from the list."
- />
- <StreamActionLog
- trigger={() =>
- retryFailuresAction(
- slug,
- retryQueue,
- parseConcurrency(retryConcurrency),
- retryAudioFormat === "any" ? undefined : retryAudioFormat,
- retryAudioFormat !== "any" && retryStrictFormat,
- )
- }
- cancelAction={cancelJobAction}
- buttonLabel="Retry failures"
- runningLabel="Retrying…"
- extraControls={
- <>
+ <FailedTranscriptionsSection
+ slug={slug}
+ ids={failedVideoIds}
+ existingQueues={existingQueues}
+ defaultQueueKey={channelQueueKey}
+ />
+ </div>
+ );
+}
+
+function FailedTranscriptionsSection({
+ slug,
+ ids,
+ existingQueues,
+ defaultQueueKey,
+}: {
+ slug: string;
+ ids: string[];
+ existingQueues: string[];
+ defaultQueueKey: string;
+}) {
+ const [clearQueue, setClearQueue] = useState(defaultQueueKey);
+ const [confirmText, setConfirmText] = useState("");
+ const armed = confirmText === "clear";
+ return (
+ <div
+ aria-label="failed transcriptions section"
+ className="flex flex-col gap-2 rounded border border-zinc-200 dark:border-zinc-800 p-3"
+ >
+ <div>
+ <h4 className="text-sm font-semibold">
+ Failed transcriptions ({ids.length})
+ </h4>
+ <p className="text-xs text-zinc-500">
+ Whisper failed on these videos and they're skipped by
+ “Transcribe missing”. Clearing the list lets the next
+ “Transcribe missing” retry them.
+ </p>
+ </div>
+ <VideoIdList
+ slug={slug}
+ ids={ids}
+ ariaLabel="failed transcriptions list"
+ emptyAriaLabel="failed transcriptions empty"
+ emptyMessage="No failed transcriptions."
+ itemAriaLabel={(id) => `failed transcription ${id}`}
+ />
+ {ids.length > 0 ? (
+ <div className="flex flex-col gap-2">
+ <label className="flex items-center gap-2 text-xs text-zinc-600 dark:text-zinc-400">
+ Type
+ <span className="font-mono">clear</span>
+ to confirm
+ <input
+ type="text"
+ value={confirmText}
+ onChange={(e) => setConfirmText(e.target.value)}
+ aria-label="confirm clear failed transcriptions"
+ className="rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 px-2 py-0.5 font-mono"
+ />
+ </label>
+ <StreamActionLog
+ trigger={() =>
+ clearFailedTranscriptionsAction(slug, clearQueue)
+ }
+ cancelAction={cancelJobAction}
+ buttonLabel={`Clear failure list (${ids.length})`}
+ runningLabel="Clearing…"
+ label="Clear failed transcriptions"
+ disabled={!armed}
+ extraControls={
<QueueControl
- value={retryQueue}
- onChange={setRetryQueue}
+ value={clearQueue}
+ onChange={setClearQueue}
defaultQueueKey={defaultQueueKey}
existingQueues={existingQueues}
- actionLabel="Retry failures"
- />
- <ConcurrencyControl
- value={retryConcurrency}
- onChange={setRetryConcurrency}
- defaultLimit={defaultConcurrency}
- actionLabel="Retry failures"
- />
- <AudioFormatControl
- actionLabel="Retry failures"
- value={retryAudioFormat}
- onChange={setRetryAudioFormat}
- strict={retryStrictFormat}
- onStrictChange={setRetryStrictFormat}
+ actionLabel="Clear failed transcriptions"
/>
- </>
- }
- />
- <VideoIdList
- slug={slug}
- ids={failedVideoIds}
- ariaLabel="failed transcriptions list"
- emptyAriaLabel="failed transcriptions empty"
- emptyMessage="No failed transcriptions."
- itemAriaLabel={(id) => `failed transcription ${id}`}
- />
- </div>
+ }
+ />
+ </div>
+ ) : null}
</div>
);
}
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -9,10 +9,16 @@ import {
} from "yt-dlp-transcript-common/controller/channelSnapshot";
import { loadFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { loadFailedTranscodings } from "yt-dlp-transcript-common/controller/failedTranscodings";
+import {
+ loadShardConfig,
+ type ShardConfig,
+ type ShardOp,
+} from "yt-dlp-transcript-common/controller/shard";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import {
detectPlatform,
platformQueueKey,
+ TRANSCRIPTION_QUEUE,
} from "yt-dlp-transcript-common/lib/platform";
import { getRegistry } from "yt-dlp-transcript-common/jobs/registry";
import { ChannelFormClient } from "../components/ChannelFormClient";
@@ -81,6 +87,22 @@ export default async function ChannelDetailPage({
const failedVideoIds = await loadFailedTranscriptions(paths, slug);
const failedTranscodingIds = await loadFailedTranscodings(paths, slug);
+ const summarize = (c: ShardConfig | null) =>
+ c
+ ? {
+ totalShards: c.totalShards,
+ shardIndex: c.shardIndex,
+ itemCount: c.items.length,
+ createdAt: c.createdAt,
+ }
+ : null;
+ const [downloadMissingShard, transcribeMissingShard, availabilityShard] =
+ await Promise.all(
+ (
+ ["download-missing", "transcribe-missing", "availability"] as ShardOp[]
+ ).map((op) => loadShardConfig(paths, slug, op).then(summarize)),
+ );
+
const buckets = normalizeBuckets(snapshot.buckets);
const undownloadedIds = snapshot.undownloadedIds ?? [];
const availability = normalizeAvailability(snapshot.availability);
@@ -134,6 +156,7 @@ export default async function ChannelDetailPage({
existingQueues={existingQueues}
undownloadedIds={undownloadedIds}
noTranscriptIds={buckets.noTranscript}
+ missingShard={downloadMissingShard}
/>
),
transcribe: (
@@ -143,6 +166,8 @@ export default async function ChannelDetailPage({
failedVideoIds={failedVideoIds}
downloadedNoTranscriptIds={buckets.downloadedNoTranscript}
defaultConcurrency={paths.parallelTranscribeLimit}
+ defaultQueueKey={TRANSCRIPTION_QUEUE}
+ missingShard={transcribeMissingShard}
/>
),
cleanup: (
@@ -162,6 +187,7 @@ export default async function ChannelDetailPage({
duplicateDirIds={buckets.duplicateDirs}
totals={snapshot.totals}
availability={availability}
+ availabilityShard={availabilityShard}
/>
),
danger: (
@@ -183,6 +209,7 @@ export default async function ChannelDetailPage({
existingQueues={existingQueues}
failedTranscodingIds={failedTranscodingIds}
untranscodedIds={buckets.untranscoded}
+ transcodeDefaultQueueKey={TRANSCRIPTION_QUEUE}
/>
);
}
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -23,7 +23,12 @@ async function runPipelineAction(
mode: "store-playlist" | "download-from-playlist" | "sync" | "download-missing",
kind: string,
queueKey?: string,
- options?: { ignoreArchive?: boolean; abortOnError?: boolean },
+ options?: {
+ ignoreArchive?: boolean;
+ abortOnError?: boolean;
+ shardTotal?: number;
+ shardIndex?: number;
+ },
): Promise<StreamActionResult> {
const paths = getPaths();
const channelConfig = await readChannelConfig(paths, slug);
@@ -49,6 +54,8 @@ async function runPipelineAction(
signal,
ignoreArchive: options?.ignoreArchive,
abortOnError: options?.abortOnError,
+ shardTotal: options?.shardTotal,
+ shardIndex: options?.shardIndex,
});
revalidatePath(`/channels/${slug}`);
revalidatePath("/channels");
@@ -82,13 +89,15 @@ export async function downloadMissingAction(
queueKey?: string,
ignoreArchive?: boolean,
abortOnError?: boolean,
+ shardTotal?: number,
+ shardIndex?: number,
): Promise<StreamActionResult> {
return runPipelineAction(
slug,
"download-missing",
"download-missing",
queueKey,
- { ignoreArchive, abortOnError },
+ { ignoreArchive, abortOnError, shardTotal, shardIndex },
);
}
diff --git a/editor/app/channels/[slug]/shardActions.ts b/editor/app/channels/[slug]/shardActions.ts
@@ -0,0 +1,25 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import {
+ SHARD_OPS,
+ clearShardConfig,
+ type ShardOp,
+} from "yt-dlp-transcript-common/controller/shard";
+
+export type ClearShardResult =
+ | { ok: true; cleared: boolean }
+ | { ok: false; error: string };
+
+export async function clearShardConfigAction(
+ slug: string,
+ op: ShardOp,
+): Promise<ClearShardResult> {
+ if (!(SHARD_OPS as readonly string[]).includes(op)) {
+ return { ok: false, error: `Unsupported shard op: ${op}` };
+ }
+ const cleared = await clearShardConfig(getPaths(), slug, op);
+ revalidatePath(`/channels/${slug}`);
+ return { ok: true, cleared };
+}
diff --git a/editor/app/channels/[slug]/whisperActions.ts b/editor/app/channels/[slug]/whisperActions.ts
@@ -6,9 +6,11 @@ import {
AUDIO_FORMAT_VALUES,
type AudioFormat,
} from "yt-dlp-transcript-common/lib/channelConfig";
+import { TRANSCRIPTION_QUEUE } from "yt-dlp-transcript-common/lib/platform";
import { runWhisperBatch } from "yt-dlp-transcript-common/controller/whisperBatch";
import { runTranscodeFailures } from "yt-dlp-transcript-common/controller/transcodeFailures";
import { removeFailedTranscodings } from "yt-dlp-transcript-common/controller/removeFailedTranscodings";
+import { clearFailedTranscriptions } from "yt-dlp-transcript-common/controller/failedTranscriptions";
import { cleanAudioFromTranscribed } from "yt-dlp-transcript-common/controller/cleanAudioFromTranscribed";
import { cleanExtraAudioFormats } from "yt-dlp-transcript-common/controller/cleanExtraAudioFormats";
import { verifyTranscripts } from "yt-dlp-transcript-common/controller/verifyTranscripts";
@@ -39,24 +41,27 @@ export async function transcribeMissingAction(
reverse?: boolean,
audioFormat?: AudioFormat,
strictAudioFormat?: boolean,
+ shardTotal?: number,
+ shardIndex?: number,
): Promise<StreamActionResult> {
const paths = getPaths();
const limit = sanitizeConcurrency(concurrency);
const fmt = sanitizeAudioFormat(audioFormat);
return runManagedFunction({
kind: "whisper-all",
- queueKey: queueKey === undefined ? defaultQueueKey(slug) : queueKey.trim(),
+ queueKey: queueKey === undefined ? TRANSCRIPTION_QUEUE : queueKey.trim(),
paths,
channelSlug: slug,
fn: async (onLog, signal) => {
const result = await runWhisperBatch({
channelSlug: slug,
paths,
- mode: "all",
concurrency: limit,
reverse: reverse === true,
audioFormat: fmt,
strictAudioFormat: fmt !== undefined && strictAudioFormat === true,
+ shardTotal,
+ shardIndex,
onLog,
signal,
});
@@ -68,34 +73,20 @@ export async function transcribeMissingAction(
});
}
-export async function retryFailuresAction(
+export async function clearFailedTranscriptionsAction(
slug: string,
queueKey?: string,
- concurrency?: number,
- audioFormat?: AudioFormat,
- strictAudioFormat?: boolean,
): Promise<StreamActionResult> {
const paths = getPaths();
- const limit = sanitizeConcurrency(concurrency);
- const fmt = sanitizeAudioFormat(audioFormat);
return runManagedFunction({
- kind: "whisper-retry",
+ kind: "clear-failed-transcriptions",
queueKey: queueKey === undefined ? defaultQueueKey(slug) : queueKey.trim(),
paths,
channelSlug: slug,
- fn: async (onLog, signal) => {
- const result = await runWhisperBatch({
- channelSlug: slug,
- paths,
- mode: "retry-failures",
- concurrency: limit,
- audioFormat: fmt,
- strictAudioFormat: fmt !== undefined && strictAudioFormat === true,
- onLog,
- signal,
- });
+ fn: async (onLog) => {
+ const { cleared } = await clearFailedTranscriptions(paths, slug);
onLog(
- `Retry batch: ${result.succeeded} succeeded, ${result.failed} failed, ${result.skipped} skipped, ${result.attempted} attempted.`,
+ `Cleared ${cleared} entr${cleared === 1 ? "y" : "ies"} from failed-transcriptions; they'll be retried next "Transcribe missing".`,
);
revalidatePath(`/channels/${slug}`);
},
@@ -113,7 +104,7 @@ export async function transcodeFailuresAction(
const paths = getPaths();
return runManagedFunction({
kind: "transcode-failures",
- queueKey: queueKey === undefined ? defaultQueueKey(slug) : queueKey.trim(),
+ queueKey: queueKey === undefined ? TRANSCRIPTION_QUEUE : queueKey.trim(),
paths,
channelSlug: slug,
fn: async (onLog, signal) => {
diff --git a/editor/app/channels/components/ChannelAvailabilityButton.tsx b/editor/app/channels/components/ChannelAvailabilityButton.tsx
@@ -0,0 +1,83 @@
+"use client";
+
+import { useState } from "react";
+import { checkAvailabilityAction } from "../[slug]/availabilityActions";
+
+type Status =
+ | { kind: "idle" }
+ | { kind: "running"; jobId?: string }
+ | { kind: "done"; jobId: string }
+ | { kind: "error"; message: string };
+
+export function ChannelAvailabilityButton({
+ slug,
+ disabled,
+}: {
+ slug: string;
+ disabled?: boolean;
+}) {
+ const [status, setStatus] = useState<Status>({ kind: "idle" });
+
+ async function handleClick() {
+ setStatus({ kind: "running" });
+ try {
+ // No queueKey override → server picks platformQueueKey(channel) by
+ // default, matching the channel-detail page's availability buttons and
+ // sharing the rate-limit budget with Sync.
+ const result = await checkAvailabilityAction(slug, "resume");
+ if (!result.ok) {
+ setStatus({ kind: "error", message: result.error });
+ return;
+ }
+ setStatus({ kind: "running", jobId: result.jobId });
+ const reader = result.stream.getReader();
+ while (true) {
+ const { done } = await reader.read();
+ if (done) break;
+ }
+ setStatus({ kind: "done", jobId: result.jobId });
+ } catch (e) {
+ setStatus({ kind: "error", message: (e as Error).message });
+ }
+ }
+
+ const running = status.kind === "running";
+ return (
+ <div className="flex items-center gap-2">
+ <button
+ type="button"
+ onClick={handleClick}
+ disabled={running || disabled}
+ aria-label={`check availability ${slug}`}
+ className="px-2 py-1 rounded-md bg-zinc-200 dark:bg-zinc-800 text-zinc-900 dark:text-zinc-100 text-xs font-medium hover:opacity-90 disabled:opacity-50"
+ >
+ {running ? "Checking…" : "Check availability"}
+ </button>
+ {status.kind === "error" && (
+ <span
+ role="alert"
+ aria-label={`check availability ${slug} error`}
+ className="text-xs text-red-700 dark:text-red-400"
+ >
+ {status.message}
+ </span>
+ )}
+ {status.kind === "running" && status.jobId && (
+ <span
+ aria-label={`check availability ${slug} job`}
+ className="text-xs text-zinc-500 font-mono"
+ >
+ job {status.jobId}
+ </span>
+ )}
+ {status.kind === "done" && (
+ <span
+ aria-label={`check availability ${slug} done`}
+ className="text-xs text-zinc-500"
+ >
+ done (job {status.jobId})
+ </span>
+ )}
+ </div>
+ );
+}
diff --git a/editor/app/channels/page.tsx b/editor/app/channels/page.tsx
@@ -2,6 +2,7 @@ import type { Metadata } from "next";
import Link from "next/link";
import { listChannels } from "yt-dlp-transcript-common/controller/channels";
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { ChannelAvailabilityButton } from "./components/ChannelAvailabilityButton";
import { ChannelSyncButton } from "./components/ChannelSyncButton";
export const dynamic = "force-dynamic";
@@ -80,10 +81,16 @@ export default async function ChannelsPage() {
{c.transcriptCount}
</Td>
<Td className="whitespace-nowrap">
- <ChannelSyncButton
- slug={c.slug}
- disabled={!c.config.url}
- />
+ <div className="flex items-center gap-2">
+ <ChannelSyncButton
+ slug={c.slug}
+ disabled={!c.config.url}
+ />
+ <ChannelAvailabilityButton
+ slug={c.slug}
+ disabled={!c.config.url}
+ />
+ </div>
</Td>
</tr>
))}
diff --git a/editor/app/components/ConcurrencyControl.tsx b/editor/app/components/ConcurrencyControl.tsx
@@ -23,7 +23,7 @@ export function ConcurrencyControl({
inputMode="numeric"
value={value}
placeholder={String(defaultLimit)}
- aria-label={`parallel transcribe limit for ${actionLabel}`}
+ aria-label={`parallel limit for ${actionLabel}`}
onChange={(e) => onChange(e.target.value)}
className="w-16 font-mono px-2 py-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 text-zinc-900 dark:text-zinc-100"
/>
diff --git a/editor/app/components/ShardControl.tsx b/editor/app/components/ShardControl.tsx
@@ -0,0 +1,128 @@
+"use client";
+
+import { useRouter } from "next/navigation";
+import { useState, useTransition } from "react";
+import type { ShardOp } from "yt-dlp-transcript-common/controller/shard";
+import { clearShardConfigAction } from "../channels/[slug]/shardActions";
+
+export type ShardConfigSummary = {
+ totalShards: number;
+ shardIndex: number;
+ itemCount: number;
+ createdAt: string;
+};
+
+type Props = {
+ slug: string;
+ op: ShardOp;
+ actionLabel: string;
+ existing: ShardConfigSummary | null;
+ total: string;
+ onTotalChange: (next: string) => void;
+ index: string;
+ onIndexChange: (next: string) => void;
+};
+
+export function ShardControl({
+ slug,
+ op,
+ actionLabel,
+ existing,
+ total,
+ onTotalChange,
+ index,
+ onIndexChange,
+}: Props) {
+ const router = useRouter();
+ const [clearing, startTransition] = useTransition();
+ const [error, setError] = useState<string | null>(null);
+
+ function handleClear() {
+ setError(null);
+ startTransition(async () => {
+ const r = await clearShardConfigAction(slug, op);
+ if (!r.ok) {
+ setError(r.error);
+ return;
+ }
+ router.refresh();
+ });
+ }
+
+ // Aria-labels intentionally key on `op` (a stable slug) instead of
+ // actionLabel so they don't substring-match the action button name
+ // (e.g. "Transcribe missing") and break role-based locators in tests.
+ return (
+ <span
+ aria-label={`shard ${op} control`}
+ title={`shard config for ${actionLabel}`}
+ className="flex flex-wrap items-center gap-2 text-xs text-zinc-500"
+ >
+ <span>Shard</span>
+ <input
+ type="number"
+ min={1}
+ step={1}
+ inputMode="numeric"
+ value={total}
+ placeholder="N"
+ aria-label={`shard ${op} total`}
+ onChange={(e) => onTotalChange(e.target.value)}
+ className="w-14 font-mono px-2 py-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 text-zinc-900 dark:text-zinc-100"
+ />
+ <span>/</span>
+ <input
+ type="number"
+ min={0}
+ step={1}
+ inputMode="numeric"
+ value={index}
+ placeholder="i"
+ aria-label={`shard ${op} index`}
+ onChange={(e) => onIndexChange(e.target.value)}
+ className="w-14 font-mono px-2 py-1 rounded border border-zinc-300 dark:border-zinc-700 bg-white dark:bg-zinc-900 text-zinc-900 dark:text-zinc-100"
+ />
+ {existing ? (
+ <span
+ aria-label={`shard ${op} saved`}
+ className="text-zinc-600 dark:text-zinc-400"
+ >
+ saved: {existing.shardIndex}/{existing.totalShards} ({existing.itemCount} items)
+ </span>
+ ) : (
+ <span
+ aria-label={`shard ${op} empty`}
+ className="text-zinc-400"
+ >
+ (no saved shard)
+ </span>
+ )}
+ <button
+ type="button"
+ onClick={handleClear}
+ disabled={!existing || clearing}
+ aria-label={`shard ${op} clear`}
+ className="px-2 py-0.5 rounded border border-zinc-300 dark:border-zinc-700 text-zinc-600 dark:text-zinc-400 hover:bg-zinc-100 dark:hover:bg-zinc-800 disabled:opacity-40"
+ >
+ {clearing ? "Clearing…" : "Clear"}
+ </button>
+ {error && (
+ <span
+ role="alert"
+ aria-label={`shard ${op} error`}
+ className="text-red-700 dark:text-red-400"
+ >
+ {error}
+ </span>
+ )}
+ </span>
+ );
+}
+
+export function parseShardField(s: string): number | undefined {
+ const trimmed = s.trim();
+ if (trimmed === "") return undefined;
+ const n = Number(trimmed);
+ if (!Number.isFinite(n) || n < 0) return undefined;
+ return Math.floor(n);
+}
diff --git a/editor/e2e/availability-backfill.spec.ts b/editor/e2e/availability-backfill.spec.ts
@@ -0,0 +1,107 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import { pathExists, readJson, resetData, resolvePath } from "./helpers";
+
+type AvailabilityFile = {
+ availability: string;
+ checkedAt: string;
+ webpageUrl?: string;
+};
+
+const CHANNEL = "test-youtube";
+
+async function seedVideo(
+ id: string,
+ metadata: Record<string, unknown>,
+): Promise<void> {
+ const dir = resolvePath(
+ `test-transcripts/channels/${CHANNEL}/data/${id}`,
+ );
+ await mkdir(dir, { recursive: true });
+ await writeFile(
+ resolvePath(
+ `test-transcripts/channels/${CHANNEL}/data/${id}/metadata.info.json`,
+ ),
+ JSON.stringify({
+ id,
+ webpage_url: `https://www.youtube.com/watch?v=${id}`,
+ ...metadata,
+ }),
+ );
+}
+
+test("download backfills availability.json from metadata", async ({ page }) => {
+ await resetData("one-youtube-channel");
+
+ // Two videos: one with availability=private in metadata, one without
+ // availability set (yt-dlp default = treated as public). Neither has an
+ // availability.json yet.
+ await seedVideo("seededPrivate1", { availability: "private" });
+ await seedVideo("seededPlain1", {});
+
+ await page.goto(`/channels/${CHANNEL}`);
+ // Sync triggers downloadOneAudio path? Actually "Sync" runs the sync mode
+ // (--lazy-playlist). The fake-ytdlp emits one fakeSync0001 video. After
+ // success, runYtdlp calls backfillAvailabilityFromMetadata, which sweeps
+ // ALL existing data dirs.
+ await page.getByRole("button", { name: "Sync" }).click();
+ await expect(page.getByLabel("Sync output")).toContainText("backfill", {
+ timeout: 20_000,
+ });
+
+ // Seeded videos should now have availability.json written from metadata.
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${CHANNEL}/data/seededPrivate1/availability.json`,
+ ),
+ ).toBe(true);
+ const priv = await readJson<AvailabilityFile>(
+ `test-transcripts/channels/${CHANNEL}/data/seededPrivate1/availability.json`,
+ );
+ expect(priv.availability).toBe("private");
+ expect(priv.webpageUrl).toBe(
+ "https://www.youtube.com/watch?v=seededPrivate1",
+ );
+
+ const plain = await readJson<AvailabilityFile>(
+ `test-transcripts/channels/${CHANNEL}/data/seededPlain1/availability.json`,
+ );
+ expect(plain.availability).toBe("public");
+
+ // The newly-fetched fakeSync0001 also gets a sidecar from its metadata.
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${CHANNEL}/data/fakeSync0001/availability.json`,
+ ),
+ ).toBe(true);
+});
+
+test("backfill skips videos that already have an availability sidecar", async ({
+ page,
+}) => {
+ await resetData("one-youtube-channel");
+ await seedVideo("seededPrivate1", { availability: "private" });
+ // Pre-existing sidecar marked as "deleted" — must not be overwritten by
+ // backfill (which would otherwise infer "private" from metadata).
+ await writeFile(
+ resolvePath(
+ `test-transcripts/channels/${CHANNEL}/data/seededPrivate1/availability.json`,
+ ),
+ JSON.stringify({
+ availability: "deleted",
+ checkedAt: "2099-01-01T00:00:00.000Z",
+ }),
+ );
+
+ await page.goto(`/channels/${CHANNEL}`);
+ await page.getByRole("button", { name: "Sync" }).click();
+ await expect(page.getByLabel("Sync output")).toContainText("backfill", {
+ timeout: 20_000,
+ });
+
+ const after = await readJson<AvailabilityFile>(
+ `test-transcripts/channels/${CHANNEL}/data/seededPrivate1/availability.json`,
+ );
+ expect(after.availability).toBe("deleted");
+ expect(after.checkedAt).toBe("2099-01-01T00:00:00.000Z");
+});
diff --git a/editor/e2e/availability.spec.ts b/editor/e2e/availability.spec.ts
@@ -59,6 +59,27 @@ test("resume is idempotent: a second run skips all videos", async ({ page }) =>
});
});
+test("resume log breaks down already-checked vs unchecked counts", async ({
+ page,
+}) => {
+ await resetData(FIXTURE);
+ await page.goto(`/channels/${CHANNEL}`);
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ const log = page.getByLabel("Resume availability checks output");
+ // First run: nothing checked yet.
+ await expect(log).toContainText("0 already checked, 6 unchecked", {
+ timeout: 20_000,
+ });
+ await expect(log).toContainText("concurrency=1");
+ await expect(log).toContainText("attempted=6", { timeout: 20_000 });
+
+ // Second run: all six have sidecars from the first run.
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ await expect(log).toContainText("6 already checked, 0 unchecked", {
+ timeout: 20_000,
+ });
+});
+
test("recheck non-deleted skips deleted but advances other checkedAt", async ({
page,
}) => {
diff --git a/editor/e2e/channels-actions.spec.ts b/editor/e2e/channels-actions.spec.ts
@@ -0,0 +1,57 @@
+import { test, expect } from "@playwright/test";
+import { resetData } from "./helpers";
+
+// The channel index page lets you launch jobs without drilling into the
+// detail page. Today: Sync + Check availability. Both default to the platform
+// queue (so they share the per-source rate-limit budget with each other).
+
+test("channel list shows both Sync and Check availability buttons per row", async ({
+ page,
+}) => {
+ await resetData("one-youtube-channel");
+ await page.goto("/channels");
+
+ await expect(
+ page.getByRole("button", { name: "sync test-youtube" }),
+ ).toBeVisible();
+ await expect(
+ page.getByRole("button", { name: "check availability test-youtube" }),
+ ).toBeVisible();
+});
+
+test("Check availability from the list uses the platform queue", async ({
+ page,
+}) => {
+ await resetData("one-youtube-channel");
+ await page.goto("/channels");
+ await page
+ .getByRole("button", { name: "check availability test-youtube" })
+ .click();
+ // Empty data dir → job finishes essentially instantly. Wait for the done
+ // pill (which surfaces the jobId) instead of trying to catch the running pill.
+ await expect(
+ page.getByLabel("check availability test-youtube done"),
+ ).toBeVisible({ timeout: 15_000 });
+
+ await page.goto("/jobs");
+ const firstRow = page.getByRole("row").nth(1);
+ await expect(firstRow).toContainText("platform:youtube");
+ await expect(firstRow).toContainText("check-availability");
+});
+
+test("Transcribe missing defaults to the system-wide 'transcription' queue", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await page.goto("/channels/test-transcribe");
+ await page.getByRole("button", { name: "Transcribe missing" }).click();
+ await expect(page.getByLabel("Transcribe missing output")).toContainText(
+ "succeeded",
+ { timeout: 30_000 },
+ );
+
+ await page.goto("/jobs");
+ const firstRow = page.getByRole("row").nth(1);
+ await expect(firstRow).toContainText("transcription");
+ await expect(firstRow).toContainText("whisper-all");
+});
diff --git a/editor/e2e/download-part-files.spec.ts b/editor/e2e/download-part-files.spec.ts
@@ -0,0 +1,37 @@
+import { rename } from "node:fs/promises";
+import { test, expect } from "@playwright/test";
+import { resetData, resolvePath } from "./helpers";
+
+// `.part` files are yt-dlp's in-progress download placeholders. They should
+// NOT count as a complete audio download — otherwise the prefilter marks the
+// video as already done and re-running "Download videos" won't let yt-dlp
+// resume the partial.
+
+test("audio.<ext>.part doesn't count as a complete audio file", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ // Turn vidA's only audio file into a .part — should now look undownloaded.
+ await rename(
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/data/vidA/audio.m4a",
+ ),
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/data/vidA/audio.m4a.part",
+ ),
+ );
+
+ await page.goto("/channels/test-transcribe");
+
+ // The "Downloaded but not transcribed" list shows videos with audio on
+ // disk but no transcript. With vidA's audio now hidden behind .part, only
+ // vidB and vidC should be listed.
+ const list = page.getByLabel("downloaded without transcript list");
+ await expect(list).toBeVisible();
+ await expect(
+ list.getByLabel(/^downloaded without transcript vid[A-C]$/),
+ ).toHaveCount(2);
+ await expect(
+ page.getByLabel("downloaded without transcript vidA"),
+ ).toHaveCount(0);
+});
diff --git a/editor/e2e/shard.spec.ts b/editor/e2e/shard.spec.ts
@@ -0,0 +1,157 @@
+import { test, expect } from "@playwright/test";
+import { pathExists, readJson, resetData } from "./helpers";
+
+const AVAILABILITY_CHANNEL = "availability-test";
+
+type ShardFile = {
+ totalShards: number;
+ shardIndex: number;
+ items: string[];
+ createdAt: string;
+};
+
+test.describe("Availability check sharding", () => {
+ test("shard 2/0 only checks half the videos and persists the slice", async ({
+ page,
+ }) => {
+ await resetData("availability-baseline");
+ await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
+
+ await page.getByLabel("shard availability total").first().fill("2");
+ await page.getByLabel("shard availability index").first().fill("0");
+
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ const log = page.getByLabel("Resume availability checks output");
+ await expect(log).toContainText("Shard availability: computed slice 0/2", {
+ timeout: 20_000,
+ });
+ await expect(log).toContainText("Availability check done", {
+ timeout: 20_000,
+ });
+
+ // 6 fixture videos sorted lex; shard 0/2 takes 3 of them.
+ const cfg = await readJson<ShardFile>(
+ `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`,
+ );
+ expect(cfg.totalShards).toBe(2);
+ expect(cfg.shardIndex).toBe(0);
+ expect(cfg.items).toHaveLength(3);
+
+ // Only the items in the slice got availability.json (other 3 did not).
+ let written = 0;
+ for (const id of [
+ "viddeleted1",
+ "vidmembers1",
+ "vidneedsauth1",
+ "vidprivate1",
+ "vidpublic1",
+ "vidpublic2",
+ ]) {
+ if (
+ await pathExists(
+ `test-transcripts/channels/${AVAILABILITY_CHANNEL}/data/${id}/availability.json`,
+ )
+ ) {
+ written++;
+ }
+ }
+ expect(written).toBe(3);
+ });
+
+ test("re-running with the same shard inputs reuses the saved slice", async ({
+ page,
+ }) => {
+ await resetData("availability-baseline");
+ await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
+ await page.getByLabel("shard availability total").first().fill("2");
+ await page.getByLabel("shard availability index").first().fill("0");
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ await expect(
+ page.getByLabel("Resume availability checks output"),
+ ).toContainText("computed slice 0/2", { timeout: 20_000 });
+
+ // Re-click — should report "using saved slice" rather than recompute.
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ await expect(
+ page.getByLabel("Resume availability checks output"),
+ ).toContainText("using saved slice 0/2", { timeout: 20_000 });
+ });
+
+ test("Clear button removes the saved shard config", async ({ page }) => {
+ await resetData("availability-baseline");
+ await page.goto(`/channels/${AVAILABILITY_CHANNEL}`);
+ await page.getByLabel("shard availability total").first().fill("2");
+ await page.getByLabel("shard availability index").first().fill("0");
+ await page.getByRole("button", { name: "Resume checks" }).click();
+ await expect(
+ page.getByLabel("Resume availability checks output"),
+ ).toContainText("Availability check done", { timeout: 20_000 });
+
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`,
+ ),
+ ).toBe(true);
+
+ // revalidatePath happens server-side; navigate to pick up the new
+ // existing-shard prop in the React tree so the Clear button becomes enabled.
+ await page.reload();
+ await page.getByLabel("shard availability clear").first().click();
+
+ // After router.refresh inside the control, the saved-shard pill is gone.
+ await expect(
+ page.getByLabel("shard availability empty").first(),
+ ).toBeVisible({ timeout: 10_000 });
+ expect(
+ await pathExists(
+ `test-transcripts/channels/${AVAILABILITY_CHANNEL}/shard-availability.json`,
+ ),
+ ).toBe(false);
+ });
+});
+
+test.describe("Transcribe missing sharding", () => {
+ test("shard 2/1 only transcribes its half and persists the slice", async ({
+ page,
+ }) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await page.goto("/channels/test-transcribe");
+
+ await page
+ .getByLabel("shard transcribe-missing total")
+ .first()
+ .fill("2");
+ await page
+ .getByLabel("shard transcribe-missing index")
+ .first()
+ .fill("1");
+
+ await page.getByRole("button", { name: "Transcribe missing" }).click();
+ const log = page.getByLabel("Transcribe missing output");
+ await expect(log).toContainText("Shard transcribe-missing: computed slice 1/2", {
+ timeout: 30_000,
+ });
+ await expect(log).toContainText("succeeded", { timeout: 30_000 });
+
+ const cfg = await readJson<ShardFile>(
+ "test-transcripts/channels/test-transcribe/shard-transcribe-missing.json",
+ );
+ expect(cfg.totalShards).toBe(2);
+ expect(cfg.shardIndex).toBe(1);
+ // 3 videos (vidA, vidB, vidC) sorted; index 1 of 2 picks 1 of them.
+ expect(cfg.items).toHaveLength(1);
+
+ // Only the sharded id should have a transcript.
+ let transcribed = 0;
+ for (const id of ["vidA", "vidB", "vidC"]) {
+ if (
+ await pathExists(
+ `test-transcripts/channels/test-transcribe/data/${id}/transcript.json`,
+ )
+ ) {
+ transcribed++;
+ }
+ }
+ expect(transcribed).toBe(1);
+ });
+});
diff --git a/editor/e2e/video-page.spec.ts b/editor/e2e/video-page.spec.ts
@@ -43,65 +43,27 @@ test("WhisperPanel renders failure list as links to per-video pages", async ({
await expect(page.getByLabel("failed transcription vidB")).toBeVisible();
});
-test("Retry failures prunes successes and keeps still-failing entries", async ({
+test("failed entry stays in the list across reloads (skip semantics)", async ({
page,
}) => {
await resetData("one-transcribe-channel-with-audio");
- // vidNoAudio: directory exists but has no audio.* file, so transcription
- // genuinely fails and the entry should remain in the failure list.
- const noAudioDir = resolvePath(
- "test-transcripts/channels/test-transcribe/data/vidNoAudio",
- );
- await mkdir(noAudioDir, { recursive: true });
await writeFile(
resolvePath(
"test-transcripts/channels/test-transcribe/failed-transcriptions",
),
- "vidA\nvidB\nvidNoAudio\n",
+ "vidA\n",
);
await page.goto("/channels/test-transcribe");
- await page.getByRole("button", { name: "Retry failures" }).click();
- await expect(page.getByLabel("Retry failures output")).toContainText(
- "Pruned 2 from failed-transcriptions; 1 remain.",
+ await expect(page.getByLabel("failed transcription vidA")).toBeVisible();
+ // Running Transcribe missing should skip the listed failure rather than
+ // re-attempt it (the old retry-failures flow has been replaced by Clear).
+ await page.getByRole("button", { name: "Transcribe missing" }).click();
+ await expect(page.getByLabel("Transcribe missing output")).toContainText(
+ "Skipping previously failed transcription for vidA",
{ timeout: 30_000 },
);
await page.reload();
- await expect(
- page.getByLabel("failed transcription vidNoAudio"),
- ).toBeVisible();
- await expect(page.getByLabel("failed transcription vidA")).toHaveCount(0);
- await expect(page.getByLabel("failed transcription vidB")).toHaveCount(0);
-});
-
-test("Retry failures prunes stale entries (missing dir, already transcribed)", async ({
- page,
-}) => {
- await resetData("one-transcribe-channel-with-audio");
- // vidA already has a transcript.json on disk → should be pruned without retry.
- const vidADir = resolvePath(
- "test-transcripts/channels/test-transcribe/data/vidA",
- );
- await writeFile(
- `${vidADir}/transcript.json`,
- JSON.stringify({ result: { language: "en" }, transcription: [] }),
- );
- // vidGhost is in the failure file but has no directory → should be pruned.
- await writeFile(
- resolvePath(
- "test-transcripts/channels/test-transcribe/failed-transcriptions",
- ),
- "vidA\nvidGhost\n",
- );
- await page.goto("/channels/test-transcribe");
- await page.getByRole("button", { name: "Retry failures" }).click();
- await expect(page.getByLabel("Retry failures output")).toContainText(
- "Pruned 2 from failed-transcriptions; 0 remain.",
- { timeout: 30_000 },
- );
- await page.reload();
- await expect(
- page.getByLabel("failed transcriptions empty"),
- ).toBeVisible();
+ await expect(page.getByLabel("failed transcription vidA")).toBeVisible();
});
test("transcribe-one writes transcript.json for a specific audio file", async ({
diff --git a/editor/e2e/whisper.spec.ts b/editor/e2e/whisper.spec.ts
@@ -1,4 +1,4 @@
-import { writeFile } from "node:fs/promises";
+import { readFile, writeFile } from "node:fs/promises";
import { test, expect } from "@playwright/test";
import { pathExists, resetData, resolvePath } from "./helpers";
@@ -42,12 +42,63 @@ test("verify flags missing transcripts before whisper runs", async ({ page }) =>
});
});
-test("retry-failures runs cleanly with an empty failure list", async ({ page }) => {
+test("Retry failures button is gone; Clear failure list takes its place", async ({
+ page,
+}) => {
await resetData("one-transcribe-channel-with-audio");
+ await writeFile(
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/failed-transcriptions",
+ ),
+ "vidA\nvidB\n",
+ );
await page.goto("/channels/test-transcribe");
- await page.getByRole("button", { name: "Retry failures" }).click();
- await expect(page.getByLabel("Retry failures output")).toContainText(
- "0 succeeded",
+ await expect(page.getByRole("button", { name: "Retry failures" })).toHaveCount(
+ 0,
+ );
+ await expect(
+ page.getByRole("button", { name: /Clear failure list \(2\)/ }),
+ ).toBeVisible();
+});
+
+test("clearing the failure list lets the next Transcribe missing retry them", async ({
+ page,
+}) => {
+ await resetData("one-transcribe-channel-with-audio");
+ await writeFile(
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/failed-transcriptions",
+ ),
+ "vidA\nvidB\n",
+ );
+ await page.goto("/channels/test-transcribe");
+ // Confirm the button is gated behind typing "clear".
+ const clearButton = page.getByRole("button", {
+ name: /Clear failure list \(2\)/,
+ });
+ await expect(clearButton).toBeDisabled();
+ await page
+ .getByLabel("confirm clear failed transcriptions")
+ .fill("clear");
+ await expect(clearButton).toBeEnabled();
+ await clearButton.click();
+ await expect(
+ page.getByLabel("Clear failed transcriptions output"),
+ ).toContainText("Cleared 2 entries", { timeout: 15_000 });
+ expect(
+ await readFile(
+ resolvePath(
+ "test-transcripts/channels/test-transcribe/failed-transcriptions",
+ ),
+ "utf8",
+ ),
+ ).toBe("");
+
+ // Run Transcribe missing — previously-failed vidA/vidB should be attempted.
+ await page.reload();
+ await page.getByRole("button", { name: "Transcribe missing" }).click();
+ await expect(page.getByLabel("Transcribe missing output")).toContainText(
+ "3 succeeded",
{ timeout: 30_000 },
);
});
