#!/usr/bin/env node
// Decode a paste-back from um-review.html. Handles both modes:
//
//   UMSORT/1 <n> key,verdict;...        verdict 1 good | 2 unclean | 0 not an um
//   UMSALV/1 <n> key,aMs,bMs,keep;...   salvage pass, window included
//
// "unclean" is a holding pile, not a rejection: the clip stays out of the palette
// but is offered again in MODE=salvage when a target needs more material than the
// clean ones cover.
import { readFileSync, writeFileSync, existsSync } from "node:fs";
import path from "node:path";
import { yinFrame, regionPitch } from "./pitch.mjs";
import { REASONS, OTHER_CODE } from "./reasons.mjs";

import { SONG_DATA } from "./paths.mjs";
const DIR = path.resolve(path.dirname(new URL(import.meta.url).pathname));
const SR = 48000;

function readWav(file) {
  const b = readFileSync(file);
  let p = 12, fmt = null, data = null;
  while (p + 8 <= b.length) {
    const id = b.toString("latin1", p, p + 4);
    const sz = b.readUInt32LE(p + 4), body = p + 8;
    if (id === "fmt ") fmt = { ch: b.readUInt16LE(body + 2), sr: b.readUInt32LE(body + 4) };
    if (id === "data") { data = b.subarray(body, body + sz); break; }
    p = body + sz + (sz & 1);
  }
  const n = Math.floor(data.length / 2 / fmt.ch);
  const x = new Float32Array(n);
  for (let i = 0; i < n; i += 1) x[i] = data.readInt16LE(i * fmt.ch * 2) / 32768;
  return { x, sr: fmt.sr };
}
const pct = (a, q) => { const s = [...a].sort((u, v) => u - v); return s[Math.max(0, Math.min(s.length - 1, Math.floor(q * (s.length - 1))))]; };

const raw = readFileSync(process.argv[2], "utf8").trim();
// v2 carries a reason code per verdict; v1 pages are still decodable.
const mSort = raw.match(/UMSORT\/[123]\s+(\d+)\s+(.+)$/s);
const SORT_HAS_WINDOW = /UMSORT\/3/.test(raw);
const mSalv = raw.match(/UMSALV\/[12]\s+(\d+)\s+(.+)$/s);
const mKeep = raw.match(/UMKEEP\/1\s+(\d+)\s+(.+)$/s);
if (!mSort && !mSalv && !mKeep) throw new Error("unrecognised paste-back (expected UMSORT, UMSALV or UMKEEP)");
let reasons = {};
try { reasons = JSON.parse(readFileSync(path.join(DIR, "reasons.json"), "utf8")); } catch {}
const noteReason = (key, verdict, code, textRaw) => {
  if (!code) return;
  if (code === OTHER_CODE) {
    let t = "";
    try { t = decodeURIComponent(textRaw ?? ""); } catch { t = String(textRaw ?? ""); }
    reasons[key] = { verdict, code, why: t ? `other: ${t}` : "other (no text)", text: t };
    return;
  }
  const list = verdict === 2 ? REASONS.dirty : REASONS.bad;
  reasons[key] = { verdict, code, why: list[code - 1] ?? `#${code}` };
};

// the manifest is a growing union, so an older paste-back still resolves
const byKey = new Map();
for (const f of ["um-manifest.json", "slicer-manifest.json", "slicer-manifest-r2.json",
                 "slicer-manifest-r3.json", "slicer-manifest-r4.json", "slicer-manifest-rT1.json"]) {
  try { for (const it of JSON.parse(readFileSync(path.join(DIR, f), "utf8")).items) if (!byKey.has(it.k)) byKey.set(it.k, it); }
  catch { /* not built */ }
}

const acc = JSON.parse(readFileSync(path.join(DIR, "accepted.json"), "utf8"));
const A = new Set(acc.accepted), R = new Set(acc.rejected);
let unclean = new Set();
try { unclean = new Set(JSON.parse(readFileSync(path.join(DIR, "unclean.json"), "utf8"))); } catch {}
let prov = new Set();
try { prov = new Set(JSON.parse(readFileSync(path.join(DIR, "provisional.json"), "utf8"))); } catch {}
// "deliberately dirty": kept in the palette, but the arranger penalises it
// mid-phrase and lets it through free at a phrase end, where a trace of the
// next word is harmless.
let impure = new Set();
try { impure = new Set(JSON.parse(readFileSync(path.join(DIR, "impure.json"), "utf8"))); } catch {}
// Which side a deliberately-kept stray word sits on, decided in the KEEP pass.
let keepside = {};
try { keepside = JSON.parse(readFileSync(path.join(DIR, "keepside.json"), "utf8")); } catch {}
const sideTally = {};
// Every clip that has been through a window-setting pass. A partial export
// followed by a rebuild would otherwise re-offer the ones already decided --
// and re-export them later with whatever the DEFAULT window had become,
// quietly discarding the window the user set by hand.
let decided = {};
try { decided = JSON.parse(readFileSync(path.join(DIR, "decided.json"), "utf8")); } catch {}
const core = existsSync(path.join(DIR, "corepitch.json"))
  ? JSON.parse(readFileSync(path.join(DIR, "corepitch.json"), "utf8")) : {};

const tally = { good: 0, dirty: 0, drop: 0, moved: 0, missing: 0 };
const sortWindows = [];   // windows adjusted during a sort pass, applied below

if (mSort) {
  for (const e of mSort[2].trim().split(";").filter(Boolean)) {
    // v3: key,verdict,reason,aMs,bMs[,text]   v1/v2: key,verdict[,reason[,text]]
    const f = e.split(",");
    const key = f[0].trim(), v = Number(f[1]);
    noteReason(key, v, Number(f[2] || 0), SORT_HAS_WINDOW ? f[5] : f[3]);
    if (!byKey.has(key)) { tally.missing += 1; continue; }
    if (v === 1) {
      // judged clean by ear -- outranks any machine verdict, so drop the
      // provisional flag that would otherwise penalise it in the arranger
      A.add(key); R.delete(key); unclean.delete(key); prov.delete(key);
      // A window adjusted during the fast pass is still a hand-set window and
      // must be honoured, not thrown away because this was the quick mode.
      if (SORT_HAS_WINDOW) sortWindows.push({ key, a: Number(f[3]), b: Number(f[4]) });
      tally.good += 1;
    } else if (v === 2) {
      unclean.add(key); A.delete(key); R.delete(key); prov.delete(key);
      tally.dirty += 1;
    } else {
      R.add(key); A.delete(key); unclean.delete(key); prov.delete(key);
      tally.drop += 1;
    }
  }
} else {
  const KEEP = !!mKeep;
  const M = mKeep ?? mSalv;
  let salvaged = [];
  try { salvaged = JSON.parse(readFileSync(path.join(SONG_DATA, "cand2", "_salvaged.json"), "utf8")).candidates; } catch {}
  const srcCache = new Map();
  for (const e of M[2].trim().split(";").filter(Boolean)) {
    const [k, aRaw, bRaw, keepRaw, rRaw, tRaw] = e.split(",");
    const key = k.trim();
    // In the KEEP pass the fourth field is the SIDE the stray word sits on
    // (1 leading, 2 trailing, 3 both, 0 not a keep), not a keep/dirty verdict.
    const sideRaw = Number(keepRaw);
    const keep = KEEP ? (sideRaw === 0 ? 0 : 2) : sideRaw;
    decided[key] = KEEP ? "keeps" : "salvage";
    if (!KEEP) noteReason(key, keep, Number(rRaw || 0), tRaw);
    const it = byKey.get(key);
    if (!it) { tally.missing += 1; continue; }
    unclean.delete(key);          // the holding pile is ephemeral either way
    if (keep !== 1 && keep !== 2) {
      R.add(key); A.delete(key); impure.delete(key); delete keepside[key];
      tally.drop += 1; continue;
    }

    const start = +(it.clipStart + Number(aRaw) / 1000).toFixed(3);
    const end = +(it.clipStart + Number(bRaw) / 1000).toFixed(3);
    if (end - start < 0.12) { R.add(key); tally.drop += 1; continue; }
    if (!srcCache.has(it.v)) { srcCache.clear(); try { srcCache.set(it.v, readWav(path.join(SONG_DATA, "wav48", `${it.v}.wav`))); } catch { srcCache.set(it.v, null); } }
    const S = srcCache.get(it.v);
    if (!S) { tally.missing += 1; continue; }
    const a = Math.max(0, Math.round(start * SR)), b = Math.min(S.x.length, Math.round(end * SR));
    const seg = Float32Array.prototype.slice.call(S.x, a, b);
    const r = regionPitch(seg, Math.round(0.005 * SR), seg.length - 1, SR, 55, 300);
    if (!r.f0) { R.add(key); tally.drop += 1; continue; }
    const W = 2048, hop = Math.round(SR * 0.01), fs = [];
    for (let i = 0; i + W < seg.length; i += hop) {
      const q = yinFrame(seg, i, W, SR, 55, 300);
      if (q.conf >= 0.5 && q.f0 > 0) fs.push(q.f0);
    }
    const spread = fs.length ? pct(fs.map((v) => Math.abs(1200 * Math.log2(v / r.f0))), 0.8) : 0;
    let s2 = 0;
    for (let i = 0; i < seg.length; i += 1) s2 += seg[i] * seg[i];
    const nk = `${it.v}@${start.toFixed(2)}`;
    if (!salvaged.some((c) => `${c.video}@${c.start.toFixed(2)}` === nk)) {
      salvaged.push({
        video: it.v, token: it.tok, start, end, dur: +(end - start).toFixed(3),
        f0: +r.f0.toFixed(2), spreadCents: +spread.toFixed(1),
        rms: +(20 * Math.log10(Math.sqrt(s2 / seg.length) + 1e-9)).toFixed(1), peak: 0,
        marginBefore: 0.05, marginAfter: 0.05, gapPrev: "", gapNext: "",
        noTrim: true, salvagedFrom: key,
      });
    }
    core[nk] = { f0: +r.f0.toFixed(2), dur: +(end - start).toFixed(3) };
    if (nk !== key) { A.delete(key); R.add(key); impure.delete(key); }
    A.add(nk); R.delete(nk); prov.delete(nk);
    if (keep === 2) { impure.add(nk); tally.dirty += 1; }
    else { impure.delete(nk); tally.good += 1; }
    if (KEEP) {
      if (nk !== key) delete keepside[key];
      keepside[nk] = sideRaw;                     // 1 head | 2 tail | 3 both
      sideTally[sideRaw] = (sideTally[sideRaw] ?? 0) + 1;
    }
    if (nk !== key) tally.moved += 1;
  }
  if (salvaged.length) {
    writeFileSync(path.join(SONG_DATA, "cand2", "_salvaged.json"),
      JSON.stringify({ video: "_salvaged", candidates: salvaged }, null, 1));
  }
}

// A sort-pass window that actually moved becomes a hand-set (noTrim) clip, the
// same as one set in salvage. Unchanged windows are left alone so the default
// keeps improving with the fitted parameters.
if (sortWindows.length) {
  let handset = [];
  try { handset = JSON.parse(readFileSync(path.join(SONG_DATA, "cand2", "_salvaged.json"), "utf8")).candidates; } catch {}
  let moved = 0;
  for (const { key, a, b } of sortWindows) {
    const it = byKey.get(key);
    if (!it) continue;
    if (Math.abs(a / 1000 - it.selA) < 0.005 && Math.abs(b / 1000 - it.selB) < 0.005) continue;
    const start = +(it.clipStart + a / 1000).toFixed(3);
    const end = +(it.clipStart + b / 1000).toFixed(3);
    if (end - start < 0.12) continue;
    const nk = `${it.v}@${start.toFixed(2)}`;
    if (!handset.some((c) => `${c.video}@${c.start.toFixed(2)}` === nk)) {
      handset.push({ video: it.v, token: it.tok, start, end, dur: +(end - start).toFixed(3),
        f0: 0, spreadCents: 0, rms: -20, peak: 0, marginBefore: 0.05, marginAfter: 0.05,
        gapPrev: "", gapNext: "", noTrim: true, sortedFrom: key });
    }
    if (nk !== key) { A.delete(key); R.add(key); }
    A.add(nk); R.delete(nk); prov.delete(nk);
    moved += 1;
  }
  if (moved) {
    writeFileSync(path.join(SONG_DATA, "cand2", "_salvaged.json"),
      JSON.stringify({ video: "_salvaged", candidates: handset }, null, 1));
    console.log(`  windows adjusted during the sort pass: ${moved}`);
  }
}

acc.accepted = [...A];
acc.rejected = [...R].filter((k) => !A.has(k));
writeFileSync(path.join(DIR, "accepted.json"), JSON.stringify(acc, null, 1));
writeFileSync(path.join(DIR, "unclean.json"), JSON.stringify([...unclean], null, 1));
writeFileSync(path.join(DIR, "provisional.json"), JSON.stringify([...prov], null, 1));
writeFileSync(path.join(DIR, "impure.json"), JSON.stringify([...impure], null, 1));
writeFileSync(path.join(DIR, "keepside.json"), JSON.stringify(keepside, null, 1));
writeFileSync(path.join(DIR, "decided.json"), JSON.stringify(decided, null, 1));
writeFileSync(path.join(DIR, "corepitch.json"), JSON.stringify(core, null, 1));
writeFileSync(path.join(DIR, "reasons.json"), JSON.stringify(reasons, null, 1));

console.log(mSort ? "sort pass" : mKeep ? "keep-side pass" : "salvage pass");
if (mKeep) {
  const names = { 1: "leading (\"and um\")", 2: "trailing (\"um and\")", 3: "both sides" };
  for (const [k, n] of Object.entries(sideTally)) console.log(`  ${String(n).padStart(4)}  ${names[k] ?? k}`);
}
console.log(`  ${mSort ? "good" : "cleaned"} (into the palette) ${tally.good}`);
console.log(`  ${mSort ? "unclean (held for salvage)"
  : mKeep ? "kept, side recorded"
  : "kept as deliberately dirty (placed against a rest)"} ${tally.dirty}`);
console.log(`  dropped ${tally.drop}`);
if (tally.moved) console.log(`  re-trimmed windows ${tally.moved}`);
if (tally.missing) console.log(`  not in any manifest ${tally.missing}`);
{
  const tally2 = {};
  for (const v of Object.values(reasons)) tally2[v.why] = (tally2[v.why] ?? 0) + 1;
  const rows = Object.entries(tally2).sort((a, b) => b[1] - a[1]);
  if (rows.length) {
    console.log(`reasons recorded (${Object.keys(reasons).length} tagged in total):`);
    for (const [why, n] of rows) console.log(`  ${String(n).padStart(4)}  ${why}`);
  }
}
console.log(`palette now ${acc.accepted.length} accepted (${prov.size} still machine-graded, ${impure.size} deliberately dirty), ${unclean.size} unclean waiting`);