commit 245a9908102f9c3afc70d916ceb8af403782465e
parent 174e147e1cf2c36cf0b6f8513fa236cf45592d9d
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Sun, 26 Apr 2026 21:30:00 -0400
feat(editor): per-channel yt-dlp pipeline with fake-binary e2e tests
Adds Store Playlist / Download from Playlist / Sync per channel through
common/ytdlp/runYtdlp.ts with the app-side archive prefilter the user
called for.
- common/ytdlp/runYtdlp.ts:
- store-playlist: yt-dlp --flat-playlist --skip-download --print url,
captures stdout, atomically writes <channel>/playlist.
- download-from-playlist: reads <channel>/playlist, parses
<channel>/archive, writes <channel>/playlist.tofetch with only the
URLs whose ID isn't already in the archive (canonical id extraction
for youtube/youtu.be, rumble, odysee). yt-dlp's --break-on-existing
stays as defense in depth. Touches lastFullDownloadAt on success.
- sync: yt-dlp --lazy-playlist with the channel URL. Touches
lastSyncedAt on success.
- Per-handling args: --write-auto-subs --skip-download for youtube,
-f bestaudio -x --audio-format <fmt> for transcribe (whisper runs
in Phase 7).
- cookiesFromBrowser + ytdlpExtraArgs honored from ChannelConfig.
- editor/app/channels/[slug]/pipelineActions.ts: server actions wrap
runYtdlp via runManagedFunction with mutexKey "channel:<slug>", so two
ops on the same channel can't run concurrently but different channels
can.
- editor/app/channels/[slug]/_components/PipelinePanel.tsx + wired into
the channel detail page. If url is unset, the panel shows a hint
instead of buttons.
- transcripts/.gitignore: covers /*/playlist.tofetch and /.jobs/.
Cypress:
- editor/cypress/fixtures/bin/fake-ytdlp.mjs: stand-in binary that
emits 5 deterministic URLs for store-playlist, materializes
data/<id>/{metadata.info.json,transcript.en.vtt} per URL for
download, and one fakeSync0001 entry for sync. Records invocations
for the prefilter assertion.
- dev:test sets YTDLP_BIN to that script.
- editor/cypress/e2e/pipeline.cy.ts: store-playlist writes 5 URLs,
download-from-playlist materializes all 5 + writes lastFullDownloadAt,
re-running download is fully prefiltered (0 new / 5 archived),
sync writes one new entry + lastSyncedAt, and a no-url channel hides
the pipeline buttons.
Verified: 18/18 cypress specs pass; pnpm --filter export run build still
produces the existing static output.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Diffstat:
8 files changed, 703 insertions(+), 4 deletions(-)
diff --git a/common/ytdlp/runYtdlp.ts b/common/ytdlp/runYtdlp.ts
@@ -0,0 +1,292 @@
+import path from "node:path";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { execa } from "execa";
+import { readArchive } from "../lib/archive";
+import {
+ parseChannelConfig,
+ type ChannelConfig,
+} from "../lib/channelConfig";
+import type { Paths } from "../lib/paths";
+
+export type YtdlpMode =
+ | "store-playlist"
+ | "download-from-playlist"
+ | "sync";
+
+export type RunYtdlpOpts = {
+ channelSlug: string;
+ mode: YtdlpMode;
+ channelConfig: ChannelConfig;
+ paths: Paths;
+ onLog: (s: string) => void;
+ signal: AbortSignal;
+};
+
+export async function runYtdlp(opts: RunYtdlpOpts): Promise<void> {
+ if (!opts.channelConfig.url) {
+ throw new Error("Channel has no `url` configured");
+ }
+ switch (opts.mode) {
+ case "store-playlist":
+ await storePlaylist(opts);
+ return;
+ case "download-from-playlist":
+ await downloadFromPlaylist(opts);
+ return;
+ case "sync":
+ await sync(opts);
+ return;
+ }
+}
+
+function channelRoot(opts: RunYtdlpOpts): string {
+ return path.join(opts.paths.channelsDir, opts.channelSlug);
+}
+
+function configArgs(config: ChannelConfig): string[] {
+ const args: string[] = [];
+ if (config.cookiesFromBrowser) {
+ args.push("--cookies-from-browser", config.cookiesFromBrowser);
+ }
+ if (config.ytdlpExtraArgs?.length) args.push(...config.ytdlpExtraArgs);
+ return args;
+}
+
+function handlingArgs(config: ChannelConfig): string[] {
+ if (config.handling === "youtube") {
+ return ["--write-auto-subs", "--write-info-json", "--skip-download"];
+ }
+ // transcribe: download audio only; whisper runs as a follow-up job (Phase 7).
+ const fmt = config.audioFormat ?? "m4a";
+ return [
+ "--write-info-json",
+ "-f",
+ "bestaudio",
+ "-x",
+ "--audio-format",
+ fmt,
+ ];
+}
+
+const OUTPUT_ARGS: string[] = [
+ "-o",
+ "data/%(id)s/audio.%(ext)s",
+ "-o",
+ "subtitle:data/%(id)s/transcript",
+ "-o",
+ "infojson:data/%(id)s/metadata",
+];
+
+async function storePlaylist(opts: RunYtdlpOpts): Promise<void> {
+ const root = channelRoot(opts);
+ await mkdir(root, { recursive: true });
+ const playlistPath = path.join(root, "playlist");
+ const tmpPath = `${playlistPath}.tmp-${process.pid}`;
+
+ const args: string[] = [
+ "--flat-playlist",
+ "--skip-download",
+ "--print",
+ "url",
+ ...configArgs(opts.channelConfig),
+ opts.channelConfig.url!,
+ ];
+ opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+
+ const child = execa(opts.paths.ytdlpBin, args, {
+ cwd: root,
+ cancelSignal: opts.signal,
+ all: false,
+ buffer: true,
+ reject: false,
+ });
+ child.stderr?.on("data", (c: Buffer) => opts.onLog(c.toString("utf8")));
+ const result = await child;
+
+ if (result.exitCode !== 0) {
+ throw new Error(`yt-dlp exited with code ${result.exitCode}`);
+ }
+
+ const stdout = String(result.stdout ?? "");
+ const urls = stdout
+ .split("\n")
+ .map((s) => s.trim())
+ .filter(Boolean);
+ await writeFile(tmpPath, urls.join("\n") + (urls.length ? "\n" : ""));
+ await rename(tmpPath, playlistPath);
+ opts.onLog(`Wrote ${urls.length} URLs to ${playlistPath}\n`);
+}
+
+async function downloadFromPlaylist(opts: RunYtdlpOpts): Promise<void> {
+ const root = channelRoot(opts);
+ const playlistPath = path.join(root, "playlist");
+ const archivePath = path.join(root, "archive");
+ const toFetchPath = path.join(root, "playlist.tofetch");
+
+ let playlistText: string;
+ try {
+ playlistText = await readFile(playlistPath, "utf8");
+ } catch {
+ throw new Error(
+ `No saved playlist at ${playlistPath}. Run "Store playlist" first.`,
+ );
+ }
+ const urls = playlistText
+ .split("\n")
+ .map((s) => s.trim())
+ .filter(Boolean);
+
+ // App-side prefilter: skip URLs whose ID is already in the archive so
+ // yt-dlp doesn't re-fetch metadata for entries we know are done. The
+ // --break-on-existing flag stays as defense in depth.
+ const archive = await readArchive(archivePath);
+ const tofetch: string[] = [];
+ let skipped = 0;
+ for (const url of urls) {
+ const id = extractVideoId(url);
+ if (id && archive.ids.has(id)) {
+ skipped++;
+ continue;
+ }
+ tofetch.push(url);
+ }
+ opts.onLog(
+ `Prefilter: ${tofetch.length} new, ${skipped} already archived (of ${urls.length} total).\n`,
+ );
+
+ if (tofetch.length === 0) {
+ await rm(toFetchPath, { force: true });
+ opts.onLog("Nothing to fetch.\n");
+ await touchLastFullDownload(opts);
+ return;
+ }
+
+ await writeFile(toFetchPath, tofetch.join("\n") + "\n");
+
+ const args: string[] = [
+ "--ignore-config",
+ "--restrict-filenames",
+ ...OUTPUT_ARGS,
+ ...handlingArgs(opts.channelConfig),
+ "--force-write-archive",
+ "--download-archive",
+ "archive",
+ "--break-on-existing",
+ "--abort-on-error",
+ "-a",
+ "playlist.tofetch",
+ ...configArgs(opts.channelConfig),
+ ];
+ await runChildAndStream(opts, root, args);
+
+ await rm(toFetchPath, { force: true });
+ await touchLastFullDownload(opts);
+}
+
+async function sync(opts: RunYtdlpOpts): Promise<void> {
+ const root = channelRoot(opts);
+ await mkdir(root, { recursive: true });
+
+ const args: string[] = [
+ "--ignore-config",
+ "--restrict-filenames",
+ ...OUTPUT_ARGS,
+ ...handlingArgs(opts.channelConfig),
+ "--force-write-archive",
+ "--download-archive",
+ "archive",
+ "--break-on-existing",
+ "--lazy-playlist",
+ ...configArgs(opts.channelConfig),
+ "--",
+ opts.channelConfig.url!,
+ ];
+ await runChildAndStream(opts, root, args);
+ await touchLastSync(opts);
+}
+
+async function runChildAndStream(
+ opts: RunYtdlpOpts,
+ cwd: string,
+ args: string[],
+): Promise<void> {
+ opts.onLog(`$ ${opts.paths.ytdlpBin} ${args.join(" ")}\n`);
+ const child = execa(opts.paths.ytdlpBin, args, {
+ cwd,
+ cancelSignal: opts.signal,
+ all: true,
+ buffer: false,
+ reject: false,
+ });
+ child.all?.on("data", (c: Buffer) => opts.onLog(c.toString("utf8")));
+ const result = await child;
+ if (result.exitCode !== 0 && !opts.signal.aborted) {
+ throw new Error(`yt-dlp exited with code ${result.exitCode}`);
+ }
+}
+
+async function touchLastSync(opts: RunYtdlpOpts): Promise<void> {
+ const file = path.join(channelRoot(opts), "config.json");
+ await updateConfigField(file, "lastSyncedAt", new Date().toISOString());
+}
+
+async function touchLastFullDownload(opts: RunYtdlpOpts): Promise<void> {
+ const file = path.join(channelRoot(opts), "config.json");
+ await updateConfigField(
+ file,
+ "lastFullDownloadAt",
+ new Date().toISOString(),
+ );
+}
+
+async function updateConfigField(
+ configPath: string,
+ field: "lastSyncedAt" | "lastFullDownloadAt",
+ value: string,
+): Promise<void> {
+ let raw: string;
+ try {
+ raw = await readFile(configPath, "utf8");
+ } catch {
+ return;
+ }
+ const parsed = parseChannelConfig(JSON.parse(raw));
+ if (!parsed) return;
+ parsed[field] = value;
+ const tmp = `${configPath}.tmp-${process.pid}`;
+ await writeFile(tmp, JSON.stringify(parsed, null, 2) + "\n");
+ await rename(tmp, configPath);
+}
+
+export function extractVideoId(url: string): string | null {
+ try {
+ const u = new URL(url);
+ const host = u.hostname.toLowerCase();
+ if (host.endsWith("youtube.com") || host === "youtu.be") {
+ const v = u.searchParams.get("v");
+ if (v) return v;
+ const seg = u.pathname.split("/").filter(Boolean).pop();
+ return seg ?? null;
+ }
+ if (host.endsWith("rumble.com")) {
+ const seg = u.pathname.split("/").filter(Boolean).pop();
+ // Rumble paths often look like /v123abc-some-title.html
+ if (seg) {
+ const trimmed = seg.replace(/\.html?$/i, "");
+ const dashIdx = trimmed.indexOf("-");
+ return dashIdx > 0 ? trimmed.slice(0, dashIdx) : trimmed;
+ }
+ return null;
+ }
+ if (host.endsWith("odysee.com")) {
+ const decoded = decodeURIComponent(u.pathname);
+ const lastColon = decoded.lastIndexOf(":");
+ if (lastColon > 0) return decoded.slice(lastColon + 1);
+ return null;
+ }
+ const seg = u.pathname.split("/").filter(Boolean).pop();
+ return seg ?? null;
+ } catch {
+ return null;
+ }
+}
diff --git a/editor/app/channels/[slug]/_components/PipelinePanel.tsx b/editor/app/channels/[slug]/_components/PipelinePanel.tsx
@@ -0,0 +1,73 @@
+"use client";
+
+import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog";
+import {
+ downloadAction,
+ storePlaylistAction,
+ syncAction,
+} from "../pipelineActions";
+
+type Props = {
+ slug: string;
+ hasUrl: boolean;
+};
+
+export function PipelinePanel({ slug, hasUrl }: Props) {
+ if (!hasUrl) {
+ return (
+ <p className="text-sm text-zinc-500">
+ Set a <code>url</code> in the configuration above to enable the
+ pipeline.
+ </p>
+ );
+ }
+ return (
+ <div className="flex flex-col gap-6">
+ <div className="flex flex-col gap-2">
+ <Heading
+ title="Store playlist"
+ desc="Fetch the channel's full URL list (--flat-playlist --skip-download --print url) and save it for later runs."
+ />
+ <StreamActionLog
+ trigger={() => storePlaylistAction(slug)}
+ buttonLabel="Store playlist"
+ runningLabel="Storing playlist…"
+ testId="pipeline-store"
+ />
+ </div>
+ <div className="flex flex-col gap-2">
+ <Heading
+ title="Download from playlist"
+ desc="Read the saved playlist, prefilter against the archive, then run yt-dlp on whatever's left. Use for first-time imports and resumes."
+ />
+ <StreamActionLog
+ trigger={() => downloadAction(slug)}
+ buttonLabel="Download from playlist"
+ runningLabel="Downloading…"
+ testId="pipeline-download"
+ />
+ </div>
+ <div className="flex flex-col gap-2">
+ <Heading
+ title="Sync"
+ desc="Quick incremental fetch using --lazy-playlist + --break-on-existing. Run regularly to grab the newest videos."
+ />
+ <StreamActionLog
+ trigger={() => syncAction(slug)}
+ buttonLabel="Sync"
+ runningLabel="Syncing…"
+ testId="pipeline-sync"
+ />
+ </div>
+ </div>
+ );
+}
+
+function Heading({ title, desc }: { title: string; desc: string }) {
+ return (
+ <div>
+ <h3 className="text-base font-semibold">{title}</h3>
+ <p className="text-sm text-zinc-500">{desc}</p>
+ </div>
+ );
+}
diff --git a/editor/app/channels/[slug]/page.tsx b/editor/app/channels/[slug]/page.tsx
@@ -4,6 +4,7 @@ import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels"
import { getPaths } from "yt-dlp-transcript-common/lib/paths";
import { ChannelFormClient } from "../_components/ChannelFormClient";
import { DeleteChannelForm } from "../_components/DeleteChannelForm";
+import { PipelinePanel } from "./_components/PipelinePanel";
import {
deleteChannelAction,
updateChannelAction,
@@ -51,9 +52,7 @@ export default async function ChannelDetailPage({
<section className="flex flex-col gap-3 border-t border-zinc-200 dark:border-zinc-800 pt-6">
<h2 className="text-lg font-semibold">Pipeline</h2>
- <p className="text-sm text-zinc-500">
- Store playlist / Download / Sync UI lands in Phase 6.
- </p>
+ <PipelinePanel slug={slug} hasUrl={!!config.url} />
</section>
{config.handling === "transcribe" && (
diff --git a/editor/app/channels/[slug]/pipelineActions.ts b/editor/app/channels/[slug]/pipelineActions.ts
@@ -0,0 +1,65 @@
+"use server";
+
+import { revalidatePath } from "next/cache";
+import { getPaths } from "yt-dlp-transcript-common/lib/paths";
+import { readChannelConfig } from "yt-dlp-transcript-common/controller/channels";
+import { runYtdlp } from "yt-dlp-transcript-common/ytdlp/runYtdlp";
+import {
+ runManagedFunction,
+ type StreamActionResult,
+} from "yt-dlp-transcript-common/jobs/streamCommand";
+
+async function runPipelineAction(
+ slug: string,
+ mode: "store-playlist" | "download-from-playlist" | "sync",
+ kind: string,
+): Promise<StreamActionResult> {
+ const paths = getPaths();
+ const channelConfig = await readChannelConfig(paths, slug);
+ if (!channelConfig) {
+ return { ok: false, error: `Channel "${slug}" not found` };
+ }
+ if (!channelConfig.url) {
+ return { ok: false, error: "Channel has no `url` configured" };
+ }
+ return runManagedFunction({
+ kind,
+ mutexKey: `channel:${slug}`,
+ paths,
+ busyMessage: `Channel "${slug}" already has an operation running`,
+ fn: async (onLog, signal) => {
+ await runYtdlp({
+ channelSlug: slug,
+ mode,
+ channelConfig,
+ paths,
+ onLog,
+ signal,
+ });
+ revalidatePath(`/channels/${slug}`);
+ revalidatePath("/channels");
+ },
+ });
+}
+
+export async function storePlaylistAction(
+ slug: string,
+): Promise<StreamActionResult> {
+ return runPipelineAction(slug, "store-playlist", "store-playlist");
+}
+
+export async function downloadAction(
+ slug: string,
+): Promise<StreamActionResult> {
+ return runPipelineAction(
+ slug,
+ "download-from-playlist",
+ "download-from-playlist",
+ );
+}
+
+export async function syncAction(
+ slug: string,
+): Promise<StreamActionResult> {
+ return runPipelineAction(slug, "sync", "sync");
+}
diff --git a/editor/cypress/e2e/pipeline.cy.ts b/editor/cypress/e2e/pipeline.cy.ts
@@ -0,0 +1,110 @@
+describe("Pipeline", () => {
+ it("store playlist writes a playlist file with 5 URLs", () => {
+ cy.resetData("test-pipeline");
+ cy.visit("/channels/test-pipeline");
+ cy.findByTestId("pipeline-store").find("button").click();
+ cy.findByTestId("pipeline-store-log", { timeout: 20_000 }).should(
+ "contain.text",
+ "Wrote 5 URLs",
+ );
+ cy.task(
+ "readJson",
+ "test-transcripts/channels/test-pipeline/config.json",
+ );
+ cy.task(
+ "pathExists",
+ "test-transcripts/channels/test-pipeline/playlist",
+ ).then((exists) => expect(exists).to.equal(true));
+ });
+
+ it("download from playlist fetches all 5 URLs and writes the archive", () => {
+ cy.resetData("test-pipeline");
+ cy.visit("/channels/test-pipeline");
+ cy.findByTestId("pipeline-store").find("button").click();
+ cy.findByTestId("pipeline-store-log", { timeout: 20_000 }).should(
+ "contain.text",
+ "Wrote 5 URLs",
+ );
+
+ cy.findByTestId("pipeline-download").find("button").click();
+ cy.findByTestId("pipeline-download-log", { timeout: 30_000 }).should(
+ "contain.text",
+ "5 new, 0 already archived",
+ );
+ cy.findByTestId("pipeline-download-log").should(
+ "contain.text",
+ "download complete",
+ );
+
+ for (let i = 1; i <= 5; i++) {
+ cy.task(
+ "pathExists",
+ `test-transcripts/channels/test-pipeline/data/fake0000000${i}/metadata.info.json`,
+ ).then((exists) => expect(exists).to.equal(true));
+ }
+ cy.task("readJson", "test-transcripts/channels/test-pipeline/config.json")
+ .then((c) => {
+ expect((c as { lastFullDownloadAt?: string }).lastFullDownloadAt).to.be
+ .a("string");
+ });
+ });
+
+ it("re-running download skips already-archived entries", () => {
+ cy.resetData("test-pipeline");
+ cy.visit("/channels/test-pipeline");
+ cy.findByTestId("pipeline-store").find("button").click();
+ cy.findByTestId("pipeline-store-log", { timeout: 20_000 }).should(
+ "contain.text",
+ "Wrote 5 URLs",
+ );
+ cy.findByTestId("pipeline-download").find("button").click();
+ cy.findByTestId("pipeline-download-log", { timeout: 30_000 }).should(
+ "contain.text",
+ "download complete",
+ );
+ // Second click — prefilter should skip everything and we should never see
+ // yt-dlp invoked with a non-empty playlist.
+ cy.findByTestId("pipeline-download").find("button").click();
+ cy.findByTestId("pipeline-download-log", { timeout: 20_000 }).should(
+ "contain.text",
+ "0 new, 5 already archived",
+ );
+ cy.findByTestId("pipeline-download-log").should(
+ "contain.text",
+ "Nothing to fetch",
+ );
+ });
+
+ it("sync downloads one new entry and writes lastSyncedAt", () => {
+ cy.resetData("test-pipeline");
+ cy.visit("/channels/test-pipeline");
+ cy.findByTestId("pipeline-sync").find("button").click();
+ cy.findByTestId("pipeline-sync-log", { timeout: 30_000 }).should(
+ "contain.text",
+ "fakeSync0001 done",
+ );
+ cy.task(
+ "pathExists",
+ "test-transcripts/channels/test-pipeline/data/fakeSync0001/metadata.info.json",
+ ).then((exists) => expect(exists).to.equal(true));
+ cy.task("readJson", "test-transcripts/channels/test-pipeline/config.json")
+ .then((c) => {
+ expect((c as { lastSyncedAt?: string }).lastSyncedAt).to.be.a("string");
+ });
+ });
+
+ it("blocks pipeline buttons when channel has no url", () => {
+ cy.resetData("empty");
+ cy.visit("/channels/new");
+ cy.findByLabelText(/^name/i).type("No URL Channel");
+ cy.findByLabelText(/^slug/i).clear().type("no-url");
+ // Leave url blank
+ cy.findByRole("button", { name: /create channel/i }).click();
+ cy.location("pathname", { timeout: 10000 }).should(
+ "eq",
+ "/channels/no-url",
+ );
+ cy.contains(/set a/i);
+ cy.findByTestId("pipeline-store").should("not.exist");
+ });
+});
diff --git a/editor/cypress/fixtures/bin/fake-ytdlp.mjs b/editor/cypress/fixtures/bin/fake-ytdlp.mjs
@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+// Cypress fake yt-dlp. Three modes are recognised by argument shape:
+// 1. --flat-playlist --skip-download --print url <URL>
+// -> emit 5 deterministic URLs to stdout
+// 2. -a playlist.tofetch ...
+// -> for each URL, write data/<id>/{metadata.info.json,transcript.en.vtt},
+// append to ./archive
+// 3. --lazy-playlist <URL> ...
+// -> fetch a single deterministic "newest" entry
+// Per-handling args (--write-auto-subs vs -f bestaudio / -x) are accepted
+// but ignored — both produce a transcript.en.vtt + metadata.info.json so the
+// rest of the editor's controllers stay happy.
+//
+// Cwd is always the channel root (matches the editor's runYtdlp).
+import { mkdir, writeFile, readFile, appendFile } from "node:fs/promises";
+import { existsSync } from "node:fs";
+import path from "node:path";
+
+const argv = process.argv.slice(2);
+
+function has(flag) {
+ return argv.includes(flag);
+}
+
+function arg(flag) {
+ const i = argv.indexOf(flag);
+ if (i < 0) return undefined;
+ return argv[i + 1];
+}
+
+function lastNonFlag() {
+ for (let i = argv.length - 1; i >= 0; i--) {
+ if (!argv[i].startsWith("-")) return argv[i];
+ }
+ return undefined;
+}
+
+function urlIdYouTube(u) {
+ try {
+ const parsed = new URL(u);
+ const v = parsed.searchParams.get("v");
+ if (v) return v;
+ return parsed.pathname.split("/").filter(Boolean).pop();
+ } catch {
+ return u;
+ }
+}
+
+async function ensureDir(p) {
+ await mkdir(p, { recursive: true });
+}
+
+async function writeMetadata(videoDir, id) {
+ const meta = {
+ id,
+ title: `Synthetic ${id}`,
+ channel: "Fake Channel",
+ channel_id: "UCfake",
+ channel_url: "https://www.youtube.com/channel/UCfake",
+ uploader: "Fake Channel",
+ upload_date: "20240101",
+ duration: 60,
+ description: `Synthetic video ${id}`,
+ is_live: false,
+ was_live: false,
+ live_status: "not_live",
+ age_limit: 0,
+ extractor_key: "Youtube",
+ webpage_url: `https://www.youtube.com/watch?v=${id}`,
+ };
+ await writeFile(
+ path.join(videoDir, "metadata.info.json"),
+ JSON.stringify(meta),
+ );
+}
+
+async function writeTranscript(videoDir) {
+ const vtt = `WEBVTT\n\n00:00:00.000 --> 00:00:05.000\nFake transcript line one.\n\n00:00:05.000 --> 00:00:10.000\nFake transcript line two.\n`;
+ await writeFile(path.join(videoDir, "transcript.en.vtt"), vtt);
+}
+
+async function downloadOne(url) {
+ const id = urlIdYouTube(url);
+ if (!id) return false;
+ const videoDir = path.join("data", id);
+ if (existsSync(path.join(videoDir, "metadata.info.json"))) {
+ process.stdout.write(`[download] ${id} already in archive — skipping\n`);
+ return false;
+ }
+ await ensureDir(videoDir);
+ process.stdout.write(`[download] Fetching ${id}\n`);
+ await writeMetadata(videoDir, id);
+ await writeTranscript(videoDir);
+ await appendFile("archive", `youtube ${id}\n`);
+ process.stdout.write(`[download] ${id} done\n`);
+ return true;
+}
+
+async function modeStorePlaylist() {
+ for (let i = 1; i <= 5; i++) {
+ process.stdout.write(`https://www.youtube.com/watch?v=fake0000000${i}\n`);
+ }
+}
+
+async function modeDownloadFromFile(file) {
+ const text = await readFile(file, "utf8");
+ const urls = text.split("\n").map((s) => s.trim()).filter(Boolean);
+ process.stdout.write(`[fake-ytdlp] -a ${file} (${urls.length} urls)\n`);
+ // Mark that yt-dlp was actually invoked so a test can assert prefilter
+ // skipped this step when the playlist was empty.
+ await appendFile("fake-ytdlp.invocations", `download:${urls.length}\n`);
+ for (const url of urls) await downloadOne(url);
+ process.stdout.write(`[fake-ytdlp] download complete\n`);
+}
+
+async function modeSync(url) {
+ process.stdout.write(`[fake-ytdlp] --lazy-playlist sync against ${url}\n`);
+ await appendFile("fake-ytdlp.invocations", "sync:1\n");
+ // Always emit one "newest" entry the first time it's called per channel.
+ await downloadOne("https://www.youtube.com/watch?v=fakeSync0001");
+}
+
+async function main() {
+ if (
+ has("--flat-playlist") &&
+ has("--skip-download") &&
+ has("--print") &&
+ arg("--print") === "url"
+ ) {
+ await modeStorePlaylist();
+ return;
+ }
+
+ const playlistFile = arg("-a");
+ if (playlistFile) {
+ await modeDownloadFromFile(playlistFile);
+ return;
+ }
+
+ if (has("--lazy-playlist")) {
+ const url = lastNonFlag();
+ await modeSync(url ?? "");
+ return;
+ }
+
+ process.stderr.write(
+ `[fake-ytdlp] unknown invocation: ${argv.join(" ")}\n`,
+ );
+ process.exit(2);
+}
+
+main().catch((err) => {
+ process.stderr.write(`[fake-ytdlp] error: ${err}\n`);
+ process.exit(1);
+});
diff --git a/editor/cypress/fixtures/test-transcripts/test-pipeline/channels/test-pipeline/config.json b/editor/cypress/fixtures/test-transcripts/test-pipeline/channels/test-pipeline/config.json
@@ -0,0 +1,5 @@
+{
+ "handling": "youtube",
+ "name": "Test Pipeline Channel",
+ "url": "https://www.youtube.com/@example/videos"
+}
diff --git a/editor/package.json b/editor/package.json
@@ -5,7 +5,7 @@
"type": "module",
"scripts": {
"dev": "next dev --port 3001",
- "dev:test": "TRANSCRIPTS_DIR=$(pwd)/test-transcripts EXPORT_PUBLIC_DIR=$(pwd)/test-transcripts/.export-public next dev --port 3001",
+ "dev:test": "TRANSCRIPTS_DIR=$(pwd)/test-transcripts EXPORT_PUBLIC_DIR=$(pwd)/test-transcripts/.export-public YTDLP_BIN=$(pwd)/cypress/fixtures/bin/fake-ytdlp.mjs next dev --port 3001",
"build": "next build",
"start": "next start --port 3001",
"lint": "eslint",
