#!/usr/bin/env node
// resolve-windows.mjs — widen a manifest's clip windows to whole sentences.
//
// A manifest window starts life as the cue span covering a quote, and a cue
// boundary is a bad place to cut: ASR breaks cues where the caption line wrapped,
// which is routinely mid-sentence and often mid-word. Cutting there drops the
// lead-in that makes a quote make sense, and clips audibly start and stop in the
// middle of speech.
//
// This walks outward from the cue span to the nearest sentence boundary in the
// transcript — a cue whose text ends in . ? or ! — so the clip carries the whole
// thought. Word-level alignment is a separate, audio-side problem: build-video.mjs
// snaps the actual cut to a silence (see --fetch-pad / snapping there).
//
// Expansion is capped so a run-on passage can't drag a clip out to a minute.
//
// In the app: not used. On the CLI:
//   node umtool/report-to-video/resolve-windows.mjs <manifest.json> [--write]
//
// Options:
//   --cut-to-quote  A DIFFERENT PASS: leave the windows alone and derive each
//                   clip's `cutStart`/`cutEnd` -- the tight cut the video
//                   renders -- from where its `quote` actually is inside its
//                   window. See "extent vs cut" in docs/report-video.md.
//                   With --write it also carries the clip's `cite` forward into
//                   the cut it just derived, so the header prints a second the
//                   clip actually plays (and its derived citeUrl follows).
//   --min-match <f> Reject a cut whose quote is less than this well matched
//                   (default 0.6). Below it the clip is reported UNMATCHED and
//                   nothing is written for it.
//   --force-cut     Overwrite cut fields that are already there
//   --write         Rewrite the manifest in place (default: dry run, print a table)
//   --max-lead <s>  Max seconds to expand backwards (default 9)
//   --max-tail <s>  Max seconds to expand forwards  (default 12)
//   --site-origin <url>   Archive to read cues from when there is no local corpus
//                         (defaults to the manifest's provenance.siteOrigin)
//   --resolve-site-ids    On a published-id miss, find the record by scanning the
//                         channel's shards. Slow; see cues.mjs.
//   --cue-source <which>  auto (default) | local | http. The two can disagree
//                         once a corpus moves past its last publish — see cues.mjs.
//
// A clip entry may set `lockStart` / `lockEnd` to pin that edge exactly.
//
// A clip with its own media (`src`) reads its own `cues` file, relative to the
// manifest, in either shape local-media.mjs normalises; one with no `cues`, or
// with no `start`/`end` (it plays the whole file), is left as it is.

import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";

import { createCueSource, siteOriginFromManifest } from "./cues.mjs";
import { clipLabel, createLocalMedia, hasLocalMedia } from "./local-media.mjs";

// THE SENTENCE WIDENING IS COMMON'S (common/lib/cueWiden.mjs): one copy, shared
// with the report converters, which widen a citation's one second into a span.
// This file re-exports it.
import { CUE_EPS as EPS, widen } from "yt-dlp-transcript-common/lib/cueWiden.mjs";

export { widen };

// ---------------------------------------------------------------------------
// THE CUT INSIDE THE EXTENT.
//
// A reviewed window is the clip's USEFUL EXTENT -- the operator widens it until
// the material is worth having, which is a judgement about the recording. The
// cut the video plays is a different question: the sentences the quote is
// actually made of. Deriving the second from the first is what lets somebody
// review once and re-cut later without watching anything again.
//
// The quote is prose a human wrote about speech a machine transcribed, so the
// match is deliberately loose: case and punctuation go, `[bracketed]` editorial
// words and `[Speaker]` markers go, and an ellipsis SPLITS the quote into
// fragments that are located independently -- "A … B" is two places in the
// recording and the cut is the span from the first to the last.
// ---------------------------------------------------------------------------

/** Lowercase, no punctuation, no bracketed editorial, single spaces. */
export const normaliseWords = (s) =>
  String(s ?? "")
    .toLowerCase()
    .replace(/\[[^\]]*\]/g, " ")
    .replace(/[^a-z0-9' ]+/g, " ")
    .replace(/\s+/g, " ")
    .trim();

/** A quote as the fragments an ellipsis divides it into, each a token list. */
export function quoteFragments(quote) {
  return String(quote ?? "")
    .replace(/\[[^\]]*\]/g, " ")
    .split(/\s*(?:\.\.\.|…)\s*/)
    .map((f) => normaliseWords(f).split(" ").filter(Boolean))
    .filter((toks) => toks.length);
}

/** How many of `want`'s tokens appear in `have`, counting repeats once each. */
function overlap(want, have) {
  const pool = new Map();
  for (const t of have) pool.set(t, (pool.get(t) ?? 0) + 1);
  let n = 0;
  for (const t of want) {
    const c = pool.get(t) ?? 0;
    if (c > 0) {
      pool.set(t, c - 1);
      n += 1;
    }
  }
  return n;
}

/** The contiguous run of cues that holds the most of one fragment. */
function bestRun(cueToks, frag) {
  let best = { i: 0, j: -1, hit: 0, len: Infinity };
  const cap = frag.length * 2 + 6;
  for (let i = 0; i < cueToks.length; i += 1) {
    let acc = [];
    for (let j = i; j < cueToks.length; j += 1) {
      acc = acc.concat(cueToks[j]);
      if (acc.length > cap) break;
      const hit = overlap(frag, acc);
      // TIES GO TO THE SHORTER RUN, and that is not a detail: a run that adds
      // the preceding cue without matching another word scores the same and
      // would hand back an edge nobody chose -- the whole lead-in, on every
      // clip whose quote starts at a cue boundary.
      if (hit > best.hit || (hit === best.hit && hit > 0 && acc.length < best.len)) {
        best = { i, j, hit, len: acc.length };
      }
    }
  }
  return best;
}

/**
 * Where inside [start,end] the quote actually is.
 *
 * @returns {{ok:true,cutStart:number,cutEnd:number,score:number,matched:string}
 *          | {ok:false,score:number,why:string,matched:string}}
 */
export function cutToQuote(cues, quote, extent, { minMatch = 0.6, maxLead = 8, maxTail = 12 } = {}) {
  const frags = quoteFragments(quote);
  if (!frags.length) return { ok: false, score: 0, why: "no quote", matched: "" };

  const inside = cues.filter((c) => c.end > extent.start + EPS && c.start < extent.end - EPS);
  if (!inside.length) {
    return { ok: false, score: 0, why: "no cues inside the window", matched: "" };
  }
  const cueToks = inside.map((c) => normaliseWords(c.text).split(" ").filter(Boolean));

  let hits = 0;
  let total = 0;
  let lo = null;
  let hi = null;
  const matched = [];
  for (const frag of frags) {
    total += frag.length;
    const b = bestRun(cueToks, frag);
    if (b.j < b.i) continue;
    hits += b.hit;
    // A fragment counts as LOCATED at half its words. Below that the run is a
    // coincidence of common words and using it as an edge would cut somewhere
    // nobody chose.
    if (b.hit / frag.length < 0.5) continue;
    lo = lo === null ? b.i : Math.min(lo, b.i);
    hi = hi === null ? b.j : Math.max(hi, b.j);
    matched.push(inside.slice(b.i, b.j + 1).map((c) => String(c.text).trim()).join(" "));
  }
  const score = total ? hits / total : 0;
  const text = matched.join(" … ");
  if (lo === null) return { ok: false, score, why: "quote not found in the window", matched: text };
  if (score < minMatch) return { ok: false, score, why: "below --min-match", matched: text };

  // Outward to sentence edges, the same widen() the extent pass uses -- a cut
  // that starts mid-clause is the defect this whole file exists to remove --
  // and then back inside the extent, which is the reviewed judgement and wins.
  const w = widen(cues, inside[lo].start, inside[hi].end, { maxLead, maxTail });
  const cutStart = Math.max(extent.start, Math.min(w.start, inside[lo].start));
  const cutEnd = Math.min(extent.end, Math.max(w.end, inside[hi].end));
  if (cutEnd - cutStart < 0.5) {
    return { ok: false, score, why: "the matched span is under half a second", matched: text };
  }
  return {
    ok: true,
    cutStart: Number(cutStart.toFixed(2)),
    cutEnd: Number(cutEnd.toFixed(2)),
    score: Number(score.toFixed(2)),
    matched: text,
  };
}

/**
 * Where the cite second has to move to once a cut is derived.
 *
 * `cite` is the second the burned-in header PRINTS (attribution.mjs) and the
 * second the card's link points at (render-cards.mjs derives `t=` from it). It
 * starts life as the floor of `start`, which is a second inside the extent --
 * and a cut is a window INSIDE that extent, so the moment the quote's own
 * sentence starts later than `start`, the header names a second the clip no
 * longer plays. build-video.mjs says so on every such clip ("cite N is outside
 * the cut … kept as written"), and it is right to: it cannot move a citation
 * on its own.
 *
 * Deriving the cut is exactly the moment when moving it IS authorised -- this
 * pass is the thing that decided the clip would start later -- so it carries
 * the cite along with the edge it just moved. Clamped to whole seconds inside
 * the cut, because `t=` is a whole second and a header prints h:mm:ss.
 *
 * `citeUrl` is a different promise. An explicit one may deliberately point at
 * ANOTHER recording of the same moment (a mirror that reads better), whose
 * clock is not this one's -- so it is only re-pointed when it names this very
 * channel/video, i.e. when it is the derived URL written down. Anything else is
 * left exactly as the author wrote it.
 *
 * @returns {{cite:number, citeUrl?:string}|null} what to write, or null for
 *          "the cite is already inside the cut" / "no whole second is".
 */
export function citeForCut(entry, cut, { channelSlug } = {}) {
  const lo = Math.ceil(cut.cutStart);
  const hi = Math.floor(cut.cutEnd);
  // A sub-second cut can straddle no whole second at all. Moving the cite to a
  // fraction would be a worse citation than leaving it where it is.
  if (lo > hi) return null;
  const at = entry.cite ?? entry.start;
  if (at >= lo && at <= hi) return null;
  const cite = at < lo ? lo : hi;
  const url = retargetCiteUrl(entry, cite, channelSlug);
  return url ? { cite, citeUrl: url } : { cite };
}

/** The derived-URL test: same recording, so the same clock, so ours to move. */
function retargetCiteUrl(entry, cite, channelSlug) {
  if (!entry.citeUrl) return null;
  let u;
  try {
    u = new URL(entry.citeUrl);
  } catch {
    return null;
  }
  if (u.searchParams.get("v") !== `${entry.channel ?? channelSlug}/${entry.video}`) return null;
  if (!u.searchParams.has("t")) return null;
  // A targeted edit rather than URLSearchParams.set + toString(): re-serialising
  // rewrites the escaping of every other parameter (`/` -> `%2F`, ` ` -> `+`),
  // which would churn the manifest for nothing.
  const next = String(entry.citeUrl).replace(/([?&]t=)[0-9.]+/, `$1${cite}`);
  return next === entry.citeUrl ? null : next;
}

/**
 * The `--cut-to-quote` pass: derive every clip's cut, print it, maybe write it.
 *
 * Deliberately NOT part of the widening run. Widening moves the extent, and the
 * extent is the operator's own judgement about how much of the recording is
 * worth having; a flag that silently did both would make one of those two
 * decisions on their behalf.
 */
/**
 * Why a clip cannot be resolved against cues, or null when it can: a `src`
 * clip with no `cues` file, or one with no window (it plays the whole file).
 */
export function unresolvable(e) {
  if (!hasLocalMedia(e)) return null;
  if (e.cues == null) return "no cues file";
  if (!Number.isFinite(e.start) || !Number.isFinite(e.end)) return "no start/end (the whole file)";
  return null;
}

async function cutPass(manifest, { cuesOf, slug, opts, write, force }) {
  let changed = 0;
  let unmatched = 0;
  let cites = 0;
  for (const e of manifest.timeline) {
    if (e.type !== "clip") continue;
    const id = String(e.id).padEnd(4);
    const label = String(clipLabel(e)).padEnd(12);
    const why = unresolvable(e);
    if (why) {
      console.log(`${id} ${label} ${why} — left as is`);
      continue;
    }
    if (e.lockCut) {
      console.log(`${id} ${label} lockCut — left at ${e.cutStart}–${e.cutEnd}`);
      continue;
    }
    if (!force && (e.cutStart != null || e.cutEnd != null)) {
      console.log(`${id} ${label} already cut ${e.cutStart}–${e.cutEnd} (--force-cut to redo)`);
      // The cut stands, but a cite left behind by an EARLIER run of this pass
      // still names a second the clip does not play. Aligning it needs no
      // cues, no network and no re-derivation, so it is not worth a --force.
      if (Number.isFinite(e.cutStart) && Number.isFinite(e.cutEnd)) {
        const late = citeForCut(e, e, { channelSlug: slug });
        if (late) {
          console.log(`       cite ${e.cite ?? e.start} -> ${late.cite}${late.citeUrl ? " (and citeUrl)" : ""}`);
          cites += 1;
          if (write) {
            e.cite = late.cite;
            if (late.citeUrl) e.citeUrl = late.citeUrl;
          }
        }
      }
      continue;
    }
    const cues = await cuesOf(e);
    const r = cutToQuote(cues, e.quote, { start: e.start, end: e.end }, opts);
    const extent = `${e.start.toFixed(1)}–${e.end.toFixed(1)}`;
    if (!r.ok) {
      unmatched += 1;
      console.log(
        `${id} ${label} ${extent} UNMATCHED (${r.score.toFixed(2)}) — ${r.why}`,
      );
      if (r.matched) console.log(`       best partial: ${r.matched.slice(0, 120)}`);
      continue;
    }
    console.log(
      `${id} ${label} ${extent} -> cut ${r.cutStart.toFixed(1)}–${r.cutEnd.toFixed(1)} ` +
        `(${(r.cutEnd - r.cutStart).toFixed(1)}s, match ${r.score.toFixed(2)})`,
    );
    console.log(`       ${r.matched.slice(0, 120)}`);
    // The cite rides along with the edge that just moved -- see citeForCut().
    const moved = citeForCut(e, r, { channelSlug: slug });
    if (moved) {
      cites += 1;
      console.log(
        `       cite ${e.cite ?? e.start} -> ${moved.cite}` +
          (moved.citeUrl ? " (and citeUrl)" : e.citeUrl ? " (citeUrl left: another recording)" : ""),
      );
    }
    if (write) {
      e.cutStart = r.cutStart;
      e.cutEnd = r.cutEnd;
      if (moved) {
        e.cite = moved.cite;
        if (moved.citeUrl) e.citeUrl = moved.citeUrl;
      }
    }
    changed += 1;
  }
  return { changed, unmatched, cites };
}

async function main() {
  const argv = process.argv.slice(2);
  const manifestPath = argv.find((a) => !a.startsWith("--"));
  if (!manifestPath) {
    console.error("usage: resolve-windows.mjs <manifest.json> [--write]");
    process.exit(2);
  }
  const num = (name, dflt) => {
    const i = argv.indexOf(name);
    return i >= 0 ? Number(argv[i + 1]) : dflt;
  };
  // Lead is where the context lives — it is the run-up that makes a quote make
  // sense. Tail only needs to finish the sentence, so it gets a smaller budget.
  const opts = { maxLead: num("--max-lead", 8), maxTail: num("--max-tail", 12) };

  const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
  const slug = manifest.provenance?.channelSlug;
  const cache = new Map();

  // Cues come from a local corpus when there is one, and from the published
  // archive the manifest was built against when there is not — so this runs in a
  // clone with no `transcripts/` at all. See cues.mjs.
  const flag = (name) => {
    const i = argv.indexOf(name);
    return i >= 0 ? argv[i + 1] : undefined;
  };
  const cues = createCueSource({
    siteOrigin: flag("--site-origin") ?? process.env.SITE_ORIGIN ?? siteOriginFromManifest(manifest),
    resolveSiteIds: argv.includes("--resolve-site-ids"),
    prefer: flag("--cue-source") ?? "auto",
    log: (m) => console.error(`  · ${m}`),
  });
  // A `src` clip's cues are its own file, beside the manifest; every other
  // clip's are its corpus record's.
  const local = createLocalMedia({
    baseDir: path.dirname(path.resolve(manifestPath)),
    allowAbsolute: manifest.render?.allowAbsoluteSrc === true,
  });
  const cuesOf = (e) =>
    hasLocalMedia(e)
      ? local.cues(e)
      : cues
          .load(e.channel ?? slug, e.video, { siteChannel: e.siteChannel, siteVideo: e.siteVideo })
          .then((r) => r.cues);

  // ---- the cut pass, which is a different question ----
  if (argv.includes("--cut-to-quote")) {
    const r = await cutPass(manifest, {
      cuesOf,
      slug,
      opts: { ...opts, minMatch: num("--min-match", 0.6) },
      write: argv.includes("--write"),
      force: argv.includes("--force-cut"),
    });
    if (argv.includes("--write")) {
      await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8");
      console.log(
        `\nwrote ${manifestPath} (${r.changed} cut(s) set, ${r.cites} cite(s) moved into the cut, ` +
          `${r.unmatched} unmatched)`,
      );
    } else {
      console.log(
        `\ndry run — ${r.changed} cut(s) would be set, ${r.cites} cite(s) would move into the cut, ` +
          `${r.unmatched} unmatched; pass --write to apply`,
      );
    }
    return;
  }

  let changed = 0;
  for (const e of manifest.timeline) {
    if (e.type !== "clip") continue;
    // A compilation can span several archived channels (the same streamer's VODs
    // are mirrored across more than one), so a clip may name its own. Key the
    // cache by channel too — the same id under a different slug is a different file.
    // An author can trim a clip to land mid-cue on purpose — a cue often carries
    // a whole paragraph, and cutting a quote short is an editorial decision.
    // Widening would undo exactly that, so `lock` opts the clip out.
    const label = String(clipLabel(e)).padEnd(12);
    const why = unresolvable(e);
    if (why) {
      console.log(`${e.id.padEnd(4)} ${label} ${why} — left as is`);
      continue;
    }
    if (e.lock) {
      console.log(`${e.id.padEnd(4)} ${label} locked, left at ${e.start.toFixed(1)}–${e.end.toFixed(1)}`);
      continue;
    }
    const key = hasLocalMedia(e) ? `src:${e.cues}` : `${e.channel ?? slug}/${e.video}`;
    if (!cache.has(key)) cache.set(key, await cuesOf(e));
    const cues = cache.get(key);

    const before = { start: e.start, end: e.end };
    const w = widen(cues, e.start, e.end, opts);

    // `lockStart` / `lockEnd` pin an edge to exactly what the author wrote. The
    // escape hatch exists because sentence detection is only as good as the ASR's
    // punctuation, and some uploads have none at all — and because an utterance's
    // real trailing pause does not always line up with its last cue's end.
    if (e.lockStart) w.start = before.start;
    if (e.lockEnd) w.end = before.end;
    const dLead = (before.start - w.start).toFixed(1);
    const dTail = (w.end - before.end).toFixed(1);
    const dur = (w.end - w.start).toFixed(1);

    // Ignore sub-frame drift so a re-run on an already-resolved manifest is a
    // genuine no-op rather than a rewrite that nudges every window.
    const moved =
      Math.abs(w.start - before.start) > 0.05 || Math.abs(w.end - before.end) > 0.05;
    if (moved) changed += 1;
    console.log(
      `${e.id.padEnd(4)} ${label} ` +
        `${before.start.toFixed(1)}–${before.end.toFixed(1)} -> ` +
        `${w.start.toFixed(1)}–${w.end.toFixed(1)}  (+${dLead}s lead, +${dTail}s tail, ${dur}s)`,
    );

    if (moved) {
      e.start = Number(w.start.toFixed(2));
      e.end = Number(w.end.toFixed(2));
    }
  }

  // De-overlap clips that come from the SAME video. Widening is per-clip and
  // blind to its neighbours, so a tail that finds no sentence boundary runs to
  // the budget and can swallow the next clip's material — which plays as the
  // same footage twice. (Real case: a 2024 upload whose ASR carries no
  // punctuation at all in that stretch, so nothing stopped the search.)
  // The later clip's start is the deliberate one, so trim the earlier clip's tail.
  const byVideo = new Map();
  for (const e of manifest.timeline) {
    if (e.type !== "clip" || unresolvable(e)) continue;
    // A `src` clip's recording is its file.
    const rec = hasLocalMedia(e) ? `src:${e.src}` : e.video;
    if (!byVideo.has(rec)) byVideo.set(rec, []);
    byVideo.get(rec).push(e);
  }
  for (const [video, list] of byVideo) {
    if (list.length < 2) continue;
    list.sort((a, b) => a.start - b.start);
    for (let i = 0; i < list.length - 1; i += 1) {
      const a = list[i];
      const b = list[i + 1];
      if (a.end <= b.start) continue;
      const overlap = a.end - b.start;
      if (a.lockEnd) {
        console.log(`  ⚠ ${a.id} overlaps ${b.id} by ${overlap.toFixed(1)}s but has lockEnd — not trimmed`);
        continue;
      }
      a.end = Number(b.start.toFixed(2));
      changed += 1;
      console.log(
        `  de-overlap ${video}: ${a.id} trimmed ${overlap.toFixed(1)}s off its tail ` +
          `(it ran into ${b.id})`,
      );
      if (a.end - a.start < 3) {
        console.log(`  ⚠ ${a.id} is now only ${(a.end - a.start).toFixed(1)}s — check it`);
      }
    }
  }

  if (argv.includes("--write")) {
    await writeFile(manifestPath, JSON.stringify(manifest, null, 2) + "\n", "utf8");
    console.log(`\nwrote ${manifestPath} (${changed} window(s) changed)`);
  } else {
    console.log(`\ndry run — ${changed} window(s) would change; pass --write to apply`);
  }
}

if (import.meta.url === `file://${process.argv[1]}`) {
  main().catch((err) => {
    console.error(err);
    process.exit(1);
  });
}