// Sourcing clip media for another tool: POST /api/media/fetch-window.
//
// The feature exists so umtool stops running yt-dlp itself, and the thing that
// makes that safe is a rule the fake can't fake around: a window fetch writes
// NO metadata.info.json, because the index keys a video's presence on that
// file. So this spec fetches into a video that has never been downloaded and
// then proves the corpus still says so.
//
// Same token as /api/worker/* (the test server runs with
// WORKER_TOKEN=test-worker-token; see package.json dev:test).

import { mkdir, readdir, readFile, stat, writeFile } from "node:fs/promises";
import { test, expect, type APIRequestContext } from "@playwright/test";
import {
  generateReport,
  readJson,
  resetData,
  resolvePath,
  writeChannelConfig,
  writeSettings,
} from "./helpers";
import { baseUrl } from "./baseUrl";

const TOKEN = "test-worker-token";
const AUTH = { authorization: `Bearer ${TOKEN}` };

// In the playlist, with no data dir: undownloaded, which is the state the
// no-info-json rule has to leave untouched.
const SLUG = "test-youtube";
const VIDEO = "fake00000002";
const rel = (p: string) => `test-transcripts/channels/${SLUG}/${p}`;
const clipRel = (name: string) => rel(`data/${VIDEO}/clips/${name}`);

async function exists(relPath: string): Promise<boolean> {
  try {
    await stat(resolvePath(relPath));
    return true;
  } catch {
    return false;
  }
}

async function invocations(): Promise<string> {
  try {
    return await readFile(resolvePath(rel("fake-ytdlp.invocations")), "utf8");
  } catch {
    return "";
  }
}

async function pollJob(
  request: APIRequestContext,
  jobId: string,
  // A batch sleeps the clip-window gap between two fetches (2 s on the test
  // server, E2E_CLIP_WINDOW_GAP_MS; 30–45 s in production).
  timeout = 30_000,
): Promise<Record<string, unknown>> {
  let last: Record<string, unknown> = {};
  await expect
    .poll(
      async () => {
        const r = await request.get(
          `${baseUrl}/api/media/fetch-window/${jobId}`,
          { headers: AUTH },
        );
        last = (await r.json()) as Record<string, unknown>;
        return last.status as string;
      },
      { timeout },
    )
    .not.toMatch(/^(queued|running)$/);
  return last;
}

test.beforeEach(async () => {
  await resetData("youtube-with-playlist");
  // NAME THE FLOOR. A spec that leaves it at the product default inherits 5 GB
  // and then depends on how much room this host has left — a refusal that
  // arrives as a returned { ok: false } no assertion reads.
  await writeSettings({ minFreeDiskGB: 0 });
});

test("the endpoint is behind the worker token", async ({ request }) => {
  const noAuth = await request.post(`${baseUrl}/api/media/fetch-window`, {
    data: {
      channelSlug: SLUG,
      videoId: VIDEO,
      from: 12,
      to: 42,
      requestedBy: "umtool",
    },
  });
  expect(noAuth.status()).toBe(401);
});

test("a window is refused unless it names who asked and a sane span", async ({
  request,
}) => {
  const cases: Array<[Record<string, unknown>, string]> = [
    [{ channelSlug: "../escape", videoId: VIDEO, from: 1, to: 2, requestedBy: "umtool" }, "traversal in the slug"],
    [{ channelSlug: SLUG, videoId: "a/b", from: 1, to: 2, requestedBy: "umtool" }, "separator in the id"],
    [{ channelSlug: "..", videoId: VIDEO, from: 1, to: 2, requestedBy: "umtool" }, "a bare .. slug"],
    [{ channelSlug: SLUG, videoId: "..", from: 1, to: 2, requestedBy: "umtool" }, "a bare .. id"],
    [{ channelSlug: SLUG, videoId: VIDEO, from: 42, to: 12, requestedBy: "umtool" }, "backwards window"],
    [{ channelSlug: SLUG, videoId: VIDEO, from: 0, to: 1000, requestedBy: "umtool" }, "past the span cap"],
    [{ channelSlug: SLUG, videoId: VIDEO, from: 1, to: 2 }, "anonymous"],
  ];
  for (const [data, why] of cases) {
    const r = await request.post(`${baseUrl}/api/media/fetch-window`, {
      headers: AUTH,
      data,
    });
    expect(r.status(), why).toBe(400);
  }
});

test("a fetched window lands in clips/, carries its provenance, and leaves the video undownloaded", async ({
  page,
  request,
}) => {
  test.setTimeout(90_000);
  const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: VIDEO,
      from: 12,
      to: 42,
      requestedBy: "umtool",
      manifest: "demo-report",
      clipId: "c03",
      reason: "the clip ends mid-sentence",
      pad: 20,
    },
  });
  expect(post.status()).toBe(202);
  const queued = (await post.json()) as { jobId: string; file: string };
  expect(queued.file).toContain(`data/${VIDEO}/clips/12.00-42.00.mp4`);

  const finished = await pollJob(request, queued.jobId);
  expect(finished.status, JSON.stringify(finished)).toBe("done");
  expect(finished.file).toContain("12.00-42.00.mp4");
  expect(Number(finished.bytes)).toBeGreaterThan(0);

  // The bytes, under the name that IS the window.
  expect(await exists(clipRel("12.00-42.00.mp4"))).toBe(true);

  // And the note saying who wanted them. Without this a directory of windows
  // is bytes nobody can account for in six months.
  const sidecar = JSON.parse(
    await readFile(resolvePath(clipRel("12.00-42.00.json")), "utf8"),
  ) as Record<string, unknown>;
  expect(sidecar.requestedBy).toBe("umtool");
  expect(sidecar.manifest).toBe("demo-report");
  expect(sidecar.clipId).toBe("c03");
  expect(sidecar.reason).toBe("the clip ends mid-sentence");
  expect(sidecar.pad).toBe(20);
  expect(Number(sidecar.bytes)).toBeGreaterThan(0);
  expect(typeof sidecar.fetchedAt).toBe("string");

  // yt-dlp was asked for THAT span, with the H.264 pin, and nothing else.
  const inv = await invocations();
  expect(inv).toContain("download-sections:*12.00-42.00");
  expect(inv).toContain("avc1");

  // THE LOAD-BEARING RULE. A window carries no metadata, so the index has no
  // reason to think this video arrived.
  expect(await exists(rel(`data/${VIDEO}/metadata.info.json`))).toBe(false);

  // The page says who asked.
  await page.goto(`/channels/${SLUG}/videos/${VIDEO}`);
  await page.getByLabel("Fetched windows stage summary").click();
  await expect(
    page.getByText(/requested by umtool for demo-report\/c03/),
  ).toBeVisible();
  await expect(
    page.getByText(/Fetched windows do not make this video downloaded/i),
  ).toBeVisible();

  // And the corpus still counts it as undownloaded: four of the playlist's
  // five, exactly as before the fetch.
  await generateReport(page, SLUG);
  await page.goto("/operations/download");
  const row = page
    .getByRole("region", { name: "undownloaded", exact: true })
    .getByLabel(`undownloaded row ${SLUG}`);
  await expect(row).toContainText("4");

  // A SECOND ASK IS FREE. Containing-window reuse is what makes one generous
  // fetch the next tool's cache rather than a second download.
  const before = (await invocations()).length;
  const again = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: VIDEO,
      // INSIDE the fetched window, not equal to it: the point is that a
      // narrower ask is answered by a wider file.
      from: 15,
      to: 30,
      requestedBy: "umtool",
      manifest: "demo-report",
      clipId: "c03",
      reason: "same seconds",
    },
  });
  expect(again.status()).toBe(200);
  const cached = (await again.json()) as {
    cached: boolean;
    file: string;
    from: number;
    to: number;
  };
  expect(cached.cached).toBe(true);
  expect(cached.file).toContain("12.00-42.00.mp4");
  // The FILE's window, not the request's — every cut downstream is expressed
  // relative to it.
  expect(cached.from).toBe(12);
  expect(cached.to).toBe(42);
  expect((await invocations()).length).toBe(before);
});

test("a channel's own yt-dlp args cannot make a window write metadata", async ({
  request,
}) => {
  test.setTimeout(60_000);
  // THE ADVERSARIAL CASE FOR THE LOAD-BEARING RULE. `ytdlpExtraArgs` is free
  // text an operator typed into a form, appended to every invocation. A
  // channel carrying --write-info-json would, unguarded, make a WINDOW fetch
  // write metadata.info.json into a video that has never been downloaded — and
  // the index keys a video's presence on exactly that file.
  await writeChannelConfig(SLUG, {
    url: "https://www.youtube.com/@example/videos",
    ytdlpExtraArgs: [
      "--write-info-json",
      "--write-thumbnail",
      "--write-description",
    ],
  });

  const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: VIDEO,
      from: 50,
      to: 70,
      requestedBy: "umtool",
      manifest: "demo-report",
      clipId: "c09",
      reason: "the channel is configured to write sidecars",
    },
  });
  expect(post.status()).toBe(202);
  const { jobId } = (await post.json()) as { jobId: string };
  const finished = await pollJob(request, jobId);
  expect(finished.status, JSON.stringify(finished)).toBe("done");

  expect(await exists(clipRel("50.00-70.00.mp4"))).toBe(true);
  // STILL no metadata, so still not in the index.
  expect(await exists(rel(`data/${VIDEO}/metadata.info.json`))).toBe(false);

  // yt-dlp takes the LAST occurrence of an option, so the refusals have to come
  // after the operator's args — and the output path after those.
  const inv = await invocations();
  const line = inv.split("\n").find((l) => l.includes("download-sections:*50.00-70.00"));
  expect(line, inv).toBeTruthy();
  const writeAt = line!.indexOf("--write-info-json");
  const refuseAt = line!.indexOf("--no-write-info-json");
  expect(writeAt).toBeGreaterThan(-1);
  expect(refuseAt).toBeGreaterThan(writeAt);
  expect(line).toContain("--no-write-thumbnail");
  expect(line).toContain("--no-write-description");
  expect(line).toContain("--no-download-archive");
});

test("a 429 fails the job and puts the platform in cooldown", async ({
  request,
}) => {
  test.setTimeout(60_000);
  const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: "ratelimitvid1",
      // The fake reads the sentinel out of the URL, so the request names one.
      webpageUrl: "https://www.youtube.com/watch?v=ratelimitvid1",
      from: 5,
      to: 10,
      requestedBy: "umtool",
      manifest: "demo-report",
      clipId: "c01",
      reason: "checking the backoff",
    },
  });
  expect(post.status()).toBe(202);
  const { jobId } = (await post.json()) as { jobId: string };
  const finished = await pollJob(request, jobId);
  expect(finished.status).toBe("failed");
  expect(String(finished.error)).toMatch(/429|Too Many Requests/i);

  // Nothing half-written is left under the window's name.
  expect(
    await exists(`test-transcripts/channels/${SLUG}/data/ratelimitvid1/clips/5.00-10.00.mp4`),
  ).toBe(false);

  // The cooldown the auto-download runner and a clicked sync both honour. The
  // next ask is refused at the door rather than re-storming the source.
  const refused = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: VIDEO,
      from: 12,
      to: 42,
      requestedBy: "umtool",
      manifest: "demo-report",
      clipId: "c03",
      reason: "should be refused",
    },
  });
  expect(refused.status()).toBe(409);
  const body = (await refused.json()) as {
    cooldownMs: number;
    platform: string;
  };
  expect(body.platform).toBe("youtube");
  expect(body.cooldownMs).toBeGreaterThan(0);
});

// THE BATCH: POST /api/ops/fetch-windows, one paced job per platform queue.
// Three windows: one already on disk (no request, no pause), one the source
// refuses with a 403, one that fetches. A single 403 is an item failure — a
// removed Rumble page answers 403 too — so the run carries on past it and the
// platform is NOT backed off; the job still ends failed, naming the window it
// lost. Exactly one pause is owed (between the two network fetches) — the
// clip-window floor of 30–45 s in production, 2 s on the test server
// (E2E_CLIP_WINDOW_GAP_MS, playwright.config.ts) — and the job log names it.
test("a batch skips what is cached, survives one 403, and fetches the rest", async ({
  request,
}) => {
  test.setTimeout(150_000);
  await mkdir(resolvePath(rel(`data/${VIDEO}/clips`)), { recursive: true });
  await writeFile(resolvePath(clipRel("0.00-30.00.mp4")), "already here");

  const post = await request.post(`${baseUrl}/api/ops/fetch-windows`, {
    headers: AUTH,
    data: {
      requestedBy: "umtool",
      manifest: "demo-batch",
      items: [
        { slug: SLUG, id: VIDEO, from: 5, to: 10, clipId: "c01" },
        {
          slug: SLUG,
          id: "win403vid1",
          // The fake reads the sentinel out of the URL.
          webpageUrl: "https://www.youtube.com/watch?v=win403vid1",
          from: 5,
          to: 10,
          clipId: "c02",
        },
        { slug: SLUG, id: VIDEO, from: 50, to: 60, clipId: "c03" },
      ],
    },
  });
  expect(post.status()).toBe(200);
  const body = (await post.json()) as {
    cached: { clipId: string }[];
    jobs: { platform: string; jobId: string; items: number }[];
    jobId: string;
  };
  expect(body.cached.map((c) => c.clipId)).toEqual(["c01"]);
  expect(body.jobs).toHaveLength(1);
  expect(body.jobs[0]).toMatchObject({ platform: "youtube", items: 2 });

  const finished = await pollJob(request, body.jobId, 90_000);
  expect(finished.status).toBe("failed");
  const jobLog = await readFile(
    resolvePath(`test-transcripts/.jobs/${body.jobs[0].jobId}.log`),
    "utf8",
  );
  expect(jobLog.match(/Sleeping \d+s before the next fetch/g)).toEqual([
    "Sleeping 2s before the next fetch",
  ]);
  expect(String(finished.error)).toMatch(/1 window\(s\) failed to fetch/);
  expect(String(finished.error)).toMatch(/1 fetched, 0 cached, 1 failed/);

  expect(await exists(clipRel("50.00-60.00.mp4"))).toBe(true);
  const sidecar = await readJson<{ requestedBy: string; manifest: string; clipId: string }>(
    clipRel("50.00-60.00.json"),
  );
  expect(sidecar).toMatchObject({ requestedBy: "umtool", manifest: "demo-batch", clipId: "c03" });
  expect(
    await exists(`test-transcripts/channels/${SLUG}/data/win403vid1/clips/5.00-10.00.mp4`),
  ).toBe(false);

  // One 403 did not back the platform off: the next ask is not refused.
  const next = await request.post(`${baseUrl}/api/ops/fetch-windows`, {
    headers: AUTH,
    data: {
      requestedBy: "umtool",
      dryRun: true,
      items: [{ slug: SLUG, id: VIDEO, from: 100, to: 110 }],
    },
  });
  expect(next.status()).toBe(200);
  const plan = (await next.json()) as { groups: { platform: string }[] };
  expect(plan.groups.map((g) => g.platform)).toEqual(["youtube"]);
  const single = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: { channelSlug: SLUG, videoId: VIDEO, from: 100, to: 110, requestedBy: "umtool" },
  });
  expect(single.status()).toBe(202);
  await pollJob(request, ((await single.json()) as { jobId: string }).jobId);
});

// THE WHOLE RECORDING, when a window will not do — a tool that needs to re-cut
// freely, or a source whose windows would tile the entire runtime.
//
// It goes to the SAVED-VIDEO STORE, not to clips/: that is where big containers
// already live, with a retention rule that leaves an explicitly-requested one
// alone. `origin` is what records who asked — separate from `keepReason`, which
// stays override/pin so `pruneSavedVideos` never evicts a container somebody
// wanted.
test("a full-source fetch lands in the saved-video store and is cached on a second ask", async ({
  request,
}) => {
  test.setTimeout(90_000);
  // TRANSCRIBE HANDLING, because that is what makes the container persist.
  // `finalizeAppExtraction` — the step that moves `source-media.*` into the
  // saved-video store and writes the pointer — runs only in app-extraction
  // mode, which is `handling: "transcribe"`. The fixture channel is a
  // subtitles-first `youtube` channel, so a full-source fetch there leaves the
  // container in the video dir and never reaches the store. That is not a bug
  // in the fetch; it is the persistence plan doing what it says.
  await writeChannelConfig(SLUG, { handling: "transcribe" });
  const ask = () =>
    request.post(`${baseUrl}/api/media/fetch-window`, {
      headers: AUTH,
      data: {
        channelSlug: SLUG,
        videoId: VIDEO,
        full: true,
        requestedBy: "umtool",
        manifest: "demo-report",
        clipId: "c07",
        reason: "the bench needs to re-cut this one freely",
      },
    });

  const post = await ask();
  expect(post.status(), await post.text()).toBe(202);
  const queued = (await post.json()) as { jobId: string; file: string };
  expect(queued.jobId).toBeTruthy();

  const finished = await pollJob(request, queued.jobId);
  expect(finished.status, JSON.stringify(finished)).toBe("done");

  // THE POINTER IS THE RECORD. It lives in the video dir and names the store
  // directory holding the container — the shape umtool's raw cache reads
  // (`rawCacheOf`'s source-media branch).
  const pointer = JSON.parse(
    await readFile(resolvePath(rel(`data/${VIDEO}/saved-video.json`)), "utf8"),
  ) as Record<string, unknown>;
  expect(typeof pointer.dir).toBe("string");
  expect(String(pointer.file)).toMatch(/^source-media\./);
  expect(Number(pointer.bytes)).toBeGreaterThan(0);
  // WHO ASKED, and it is not `keepReason`: that stays override/pin so the
  // retention prune leaves the container alone, which cannot also carry a
  // requester.
  const origin = pointer.origin as Record<string, unknown>;
  expect(origin?.requestedBy).toBe("umtool");
  expect(origin?.manifest).toBe("demo-report");
  expect(origin?.clipId).toBe("c07");

  // THE SECOND ASK COSTS NOTHING. A cached full source answers 200 with the
  // file rather than queueing a second download of the same container — which
  // is the whole reason a tool routes through here instead of running yt-dlp.
  const invBefore = await invocations();
  const again = await ask();
  expect(again.status()).toBe(200);
  const cached = (await again.json()) as Record<string, unknown>;
  expect(cached.cached).toBe(true);
  expect(String(cached.file)).toContain(String(pointer.file));
  expect(Number(cached.bytes)).toBe(Number(pointer.bytes));
  expect(await invocations()).toBe(invBefore);
});

// THE SAME ASK ON THE CHANNEL AS IT IS (release 10 slice N). The test above
// switches the channel to transcribe handling, because that used to be the only
// way the container reached the store: on this subtitles-first channel the one
// media pass (the no-subs fallback) was skipped for any video with a
// transcript, so the job ended `done` with no file — the silent no-op slice M's
// live proof found. `forceMedia` opens that pass; with a transcript on disk it
// persists the container and extracts nothing.
test("a full-source fetch on a youtube-handling video that already has a transcript downloads anyway", async ({
  request,
}) => {
  test.setTimeout(90_000);
  // The fixture's downloaded video: metadata.info.json + transcript.en.vtt.
  const HAS_TRANSCRIPT = "fake00000001";
  const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: HAS_TRANSCRIPT,
      full: true,
      requestedBy: "mcp",
      manifest: "demo-report",
      reason: "the whole recording, captions or not",
    },
  });
  expect(post.status(), await post.text()).toBe(202);
  const { jobId } = (await post.json()) as { jobId: string };

  const finished = await pollJob(request, jobId);
  expect(finished.status, JSON.stringify(finished)).toBe("done");
  // A FILE, which is the whole difference.
  expect(String(finished.file)).toMatch(/source-media\./);
  expect(Number(finished.bytes)).toBeGreaterThan(0);

  const dir = resolvePath(rel(`data/${HAS_TRANSCRIPT}`));
  const files = await readdir(dir);
  expect(files).toContain("transcript.en.vtt");
  expect(files).toContain("saved-video.json");
  // Nothing extracted beside the transcript.
  expect(files.filter((f) => /^(audio|source-media)\./.test(f))).toEqual([]);

  // AND THE METADATA REWRITE IS ON RECORD, with who asked. The fixture's own
  // info json differs from what the fake's prefetch writes (another title), so
  // this fetch is a rewrite that changed content.
  const history = await readJson<{
    entries: Array<{
      by: string;
      requestedBy?: string;
      changed: Record<string, { from: unknown; to: unknown }>;
    }>;
  }>(rel(`data/${HAS_TRANSCRIPT}/metadata.history.json`));
  expect(history.entries).toHaveLength(1);
  expect(history.entries[0].by).toBe("prefetch");
  expect(history.entries[0].requestedBy).toBe("mcp");
  expect(history.entries[0].changed.title).toEqual({
    from: "Synthetic Test Video 1",
    to: `Synthetic ${HAS_TRANSCRIPT}`,
  });
});

// …AND WHEN THAT DOWNLOAD FAILS, THE JOB SAYS SO (release 11 slice O3). The
// forced media pass failing over a transcript is not a failed download — the
// subtitle pass succeeded, the transcript is fine, so the outcome stays `ok` —
// but the job exists for the source, so the job fails, with yt-dlp's line in
// the log tail the poll returns. Before, it ended `done` with no file, and the
// MCP could only say the job "finished but named no file".
test("a full-source fetch whose media download fails ends failed with yt-dlp's line; the download stays ok", async ({
  request,
}) => {
  test.setTimeout(90_000);
  const HAS_TRANSCRIPT = "fake00000001";
  await writeFile(resolvePath(rel(".fake-ytdlp-media-fail")), "");
  const post = await request.post(`${baseUrl}/api/media/fetch-window`, {
    headers: AUTH,
    data: {
      channelSlug: SLUG,
      videoId: HAS_TRANSCRIPT,
      full: true,
      requestedBy: "mcp",
    },
  });
  expect(post.status(), await post.text()).toBe(202);
  const { jobId } = (await post.json()) as { jobId: string };

  const finished = await pollJob(request, jobId);
  expect(finished.status, JSON.stringify(finished)).toBe("failed");
  expect(finished.file).toBeUndefined();
  expect(String(finished.error)).toContain(
    "The source video was not downloaded: ERROR: [download] Got error: HTTP Error 403: Forbidden",
  );
  expect(String(finished.error)).toContain(
    "forceMedia: the source download failed (yt-dlp exit 1); the transcript on disk is untouched and the download stays ok.",
  );

  const outcome = await readJson<{
    status: string;
    attempts: Array<{ kind: string; ytdlpExitCode: number | null }>;
  }>(rel(`data/${HAS_TRANSCRIPT}/download-outcome.json`));
  expect(outcome.status).toBe("ok");
  expect(outcome.attempts.map((a) => [a.kind, a.ytdlpExitCode])).toEqual([
    ["metadata-prefetch", 0],
    ["primary", 0],
    ["no-subs-fallback", 1],
  ]);
  const files = await readdir(resolvePath(rel(`data/${HAS_TRANSCRIPT}`)));
  expect(files).toContain("transcript.en.vtt");
  expect(files).not.toContain("saved-video.json");
  // The partial is left for a retry to resume, and the log named it.
  expect(files).toContain("source-media.f137.mp4.part");
  expect(String(finished.error)).toContain(
    "Left for a retry to resume: source-media.f137.mp4.part (",
  );
});