commit 0c7509c25fe7c3239294274e3aa4618be118299e parent eb5e845e405d3e215af31d12aaad040b50049782 Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st> Date: Sun, 30 Aug 2026 13:04:43 -0400 sites: the pool runs from the family page /build was never about a site — no ?site=, no listSites, eleven corpus-wide actions over the shared channel pool. It is the Pool section of /sites now: the same eight job consoles, moved verbatim, folded under a "Pool jobs" disclosure so the family page does not open on them. RunningJobsList sits ABOVE the disclosure — a running pool job must be visible without a click. The three specs that carried their own six-line "go to /build, press Build index, wait for Done" now share one buildIndex(page) helper that opens the disclosure first, because getByRole ignores what a closed <details> hides. BUILD_KINDS is carried across as-is, with a comment saying what is wrong with it: six kinds, missing the three live-chat kinds the buttons enqueue and including build-export, which none of them does. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Diffstat:
21 files changed, 458 insertions(+), 435 deletions(-)
diff --git a/common/controller/normalizeAll.ts b/common/controller/normalizeAll.ts @@ -13,10 +13,11 @@ import type { Paths } from "../lib/paths"; export type NormalizeAllOptions = { paths: Paths; // Restrict to these channel slugs. Empty/omitted = the whole corpus, which is - // what the /build button has always done. Channel scoping exists because the - // videos that need this are CONCENTRATED — 1,683 of the 1,942 unnormalized - // videos on this corpus are one channel — and clearing one channel should not - // mean walking all ~79,000 dirs from a page nobody opens during digest work. + // what the Pool's button on /sites has always done. Channel scoping exists + // because the videos that need this are CONCENTRATED — 1,683 of the 1,942 + // unnormalized videos on this corpus are one channel — and clearing one + // channel should not mean walking all ~79,000 dirs from a page nobody opens + // during digest work. // // Same shape as diarizeAll's `channelSlugs`, deliberately: that controller // says in its own header that it was modelled on this one, so this is the diff --git a/common/jobs/jobKinds.ts b/common/jobs/jobKinds.ts @@ -131,9 +131,10 @@ const JOB_KINDS: Record<string, JobKindMeta> = { queueKeyStrategy: "parallel", }, // Write the compact transcript.cues.json sidecar next to every raw transcript - // that lacks a current one — corpus-wide from /build, or one channel from the - // digest stage card. It was an UNREGISTERED kind string until now: passed to - // runManagedFunction with no entry here, so /jobs showed the raw machine kind. + // that lacks a current one — corpus-wide from the Pool on /sites, or one + // channel from the digest stage card. It was an UNREGISTERED kind string + // until now: passed to runManagedFunction with no entry here, so /jobs showed + // the raw machine kind. // // Not drainable: the walk honours the cancel signal (which is what stops it) // but has no drain-aware inner loop, and the unit of work is a single file diff --git a/editor/app/build/components/BuildButtons.tsx b/editor/app/build/components/BuildButtons.tsx @@ -1,321 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; -import { - DEFAULT_ARCHIVE_OPTIONS, - type ArchiveBuildOptions, -} from "yt-dlp-transcript-common/lib/archiveOptions"; -import { QueueControl } from "../../components/QueueControl"; -import { cancelJobAction } from "../../jobs/actions"; -import { - archiveCombinedLiveChatAction, - archiveCombinedTranscriptsAction, - archiveLiveChatAction, - archiveTranscriptsAction, - buildIndexAction, - buildStatsAction, - normalizeLiveChatAction, - normalizeTranscriptsAction, -} from "../../sites/lib/buildAction"; -import { ArchiveOptionsControl } from "./ArchiveOptionsControl"; - -type Props = { - existingQueues: string[]; -}; - -const DEFAULT_BUILD_QUEUE = "build"; - -export function BuildButtons({ existingQueues }: Props) { - const [indexQueue, setIndexQueue] = useState(DEFAULT_BUILD_QUEUE); - const [statsQueue, setStatsQueue] = useState(DEFAULT_BUILD_QUEUE); - const [normalizeQueue, setNormalizeQueue] = useState(DEFAULT_BUILD_QUEUE); - const [archiveQueue, setArchiveQueue] = useState(DEFAULT_BUILD_QUEUE); - const [combinedQueue, setCombinedQueue] = useState(DEFAULT_BUILD_QUEUE); - const [liveChatNormalizeQueue, setLiveChatNormalizeQueue] = useState( - DEFAULT_BUILD_QUEUE, - ); - const [liveChatArchiveQueue, setLiveChatArchiveQueue] = useState( - DEFAULT_BUILD_QUEUE, - ); - const [liveChatCombinedQueue, setLiveChatCombinedQueue] = useState( - DEFAULT_BUILD_QUEUE, - ); - const [archiveOptions, setArchiveOptions] = useState<ArchiveBuildOptions>( - DEFAULT_ARCHIVE_OPTIONS, - ); - const [combinedOptions, setCombinedOptions] = useState<ArchiveBuildOptions>( - DEFAULT_ARCHIVE_OPTIONS, - ); - const [liveChatArchiveOptions, setLiveChatArchiveOptions] = - useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS); - const [liveChatCombinedOptions, setLiveChatCombinedOptions] = - useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS); - - return ( - <div className="flex flex-col gap-8"> - <section className="flex flex-col gap-3"> - <div> - <h2 className="text-lg font-semibold">Build index</h2> - <p className="text-sm text-muted-foreground"> - Re-scans <code>transcripts/channels/</code> and rewrites paginated - JSON in <code>export/public/</code>. Cheap when nothing changed - (mtime short-circuit). - </p> - </div> - <StreamActionLog - trigger={() => buildIndexAction(indexQueue)} - cancelAction={cancelJobAction} - buttonLabel="Build index" - runningLabel="Building index…" - extraControls={ - <QueueControl - value={indexQueue} - onChange={setIndexQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build index" - /> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold">Build stats dataset</h2> - <p className="text-sm text-muted-foreground"> - Extracts per-video stats (views, likes, comments, duration, - categories, …) into <code>export/public/stats/</code> for the viewer - charts. Reads engagement metrics from each{" "} - <code>metadata.info.json</code>; cue counts come from the search - index, so run <strong>Build index</strong> first. Incremental - (mtime short-circuit). - </p> - </div> - <StreamActionLog - trigger={() => buildStatsAction(statsQueue)} - cancelAction={cancelJobAction} - buttonLabel="Build stats dataset" - runningLabel="Building stats dataset…" - extraControls={ - <QueueControl - value={statsQueue} - onChange={setStatsQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build stats dataset" - /> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold">Normalize transcripts</h2> - <p className="text-sm text-muted-foreground"> - Walks every video and writes{" "} - <code>transcript.cues.json</code> next to each raw{" "} - <code>.vtt</code>/<code>.json</code>. Idempotent — skips entries - whose cues file is already up to date. - </p> - </div> - <StreamActionLog - trigger={() => normalizeTranscriptsAction(normalizeQueue)} - cancelAction={cancelJobAction} - buttonLabel="Normalize transcripts" - runningLabel="Normalizing transcripts…" - extraControls={ - <QueueControl - value={normalizeQueue} - onChange={setNormalizeQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Normalize transcripts" - /> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold">Build transcript archives</h2> - <p className="text-sm text-muted-foreground"> - Produces one <code><channel>.{archiveOptions.format}</code> per channel - under <code>transcripts/export/archives/</code>, containing each - video's compact <code>transcript.cues.json</code> plus a{" "} - <code>channel.json</code> manifest. Normalizes missing cues on - demand. - </p> - </div> - <StreamActionLog - trigger={() => archiveTranscriptsAction(archiveOptions, archiveQueue)} - cancelAction={cancelJobAction} - buttonLabel="Build archives" - runningLabel="Building archives…" - extraControls={ - <div className="flex flex-wrap items-center gap-3"> - <ArchiveOptionsControl - value={archiveOptions} - onChange={setArchiveOptions} - actionLabel="Build archives" - /> - <QueueControl - value={archiveQueue} - onChange={setArchiveQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build archives" - /> - </div> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold"> - Build combined transcript archive - </h2> - <p className="text-sm text-muted-foreground"> - Produces a single{" "} - <code>all-transcripts.{combinedOptions.format}</code> under{" "} - <code>transcripts/export/archives/</code> containing every - channel plus a top-level <code>manifest.json</code> index. - Normalizes missing cues on demand. - </p> - </div> - <StreamActionLog - trigger={() => - archiveCombinedTranscriptsAction(combinedOptions, combinedQueue) - } - cancelAction={cancelJobAction} - buttonLabel="Build combined archive" - runningLabel="Building combined archive…" - extraControls={ - <div className="flex flex-wrap items-center gap-3"> - <ArchiveOptionsControl - value={combinedOptions} - onChange={setCombinedOptions} - actionLabel="Build combined archive" - /> - <QueueControl - value={combinedQueue} - onChange={setCombinedQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build combined archive" - /> - </div> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold">Normalize live chat</h2> - <p className="text-sm text-muted-foreground"> - Walks every video and writes{" "} - <code>live_chat.cues.json</code> next to each raw{" "} - <code>transcript.live_chat.json</code>. Idempotent — skips - entries whose cues file is already up to date. - </p> - </div> - <StreamActionLog - trigger={() => normalizeLiveChatAction(liveChatNormalizeQueue)} - cancelAction={cancelJobAction} - buttonLabel="Normalize live chat" - runningLabel="Normalizing live chat…" - extraControls={ - <QueueControl - value={liveChatNormalizeQueue} - onChange={setLiveChatNormalizeQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Normalize live chat" - /> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold">Build live chat archives</h2> - <p className="text-sm text-muted-foreground"> - Produces one <code><channel>.live_chat.{liveChatArchiveOptions.format}</code> per - channel under <code>transcripts/export/archives/</code>, - containing each video's parsed{" "} - <code>live_chat.cues.json</code> plus a{" "} - <code>channel.json</code> manifest. Normalizes missing live - chats on demand. - </p> - </div> - <StreamActionLog - trigger={() => - archiveLiveChatAction(liveChatArchiveOptions, liveChatArchiveQueue) - } - cancelAction={cancelJobAction} - buttonLabel="Build live chat archives" - runningLabel="Building live chat archives…" - extraControls={ - <div className="flex flex-wrap items-center gap-3"> - <ArchiveOptionsControl - value={liveChatArchiveOptions} - onChange={setLiveChatArchiveOptions} - actionLabel="Build live chat archives" - /> - <QueueControl - value={liveChatArchiveQueue} - onChange={setLiveChatArchiveQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build live chat archives" - /> - </div> - } - /> - </section> - - <section className="flex flex-col gap-3 border-t border-border pt-6"> - <div> - <h2 className="text-lg font-semibold"> - Build combined live chat archive - </h2> - <p className="text-sm text-muted-foreground"> - Produces a single <code>all-live-chat.{liveChatCombinedOptions.format}</code> under{" "} - <code>transcripts/export/archives/</code> containing every - channel's parsed live chats plus a top-level{" "} - <code>manifest.json</code> index. Normalizes missing live chats - on demand. - </p> - </div> - <StreamActionLog - trigger={() => - archiveCombinedLiveChatAction( - liveChatCombinedOptions, - liveChatCombinedQueue, - ) - } - cancelAction={cancelJobAction} - buttonLabel="Build combined live chat archive" - runningLabel="Building combined live chat archive…" - extraControls={ - <div className="flex flex-wrap items-center gap-3"> - <ArchiveOptionsControl - value={liveChatCombinedOptions} - onChange={setLiveChatCombinedOptions} - actionLabel="Build combined live chat archive" - /> - <QueueControl - value={liveChatCombinedQueue} - onChange={setLiveChatCombinedQueue} - defaultQueueKey={DEFAULT_BUILD_QUEUE} - existingQueues={existingQueues} - actionLabel="Build combined live chat archive" - /> - </div> - } - /> - </section> - </div> - ); -} diff --git a/editor/app/build/page.tsx b/editor/app/build/page.tsx @@ -1,33 +0,0 @@ -import type { Metadata } from "next"; -import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; -import { liveJobRows } from "../jobs/active/buildActiveJobs"; -import { RunningJobsList } from "../jobs/components/RunningJobsList"; -import { BuildButtons } from "./components/BuildButtons"; - -export const dynamic = "force-dynamic"; - -export const metadata: Metadata = { title: "Build" }; - -const BUILD_KINDS = new Set([ - "build-index", - "build-stats", - "build-export", - "normalize-transcripts", - "archive-transcripts", - "archive-combined-transcripts", -]); - -export default async function BuildPage() { - const registry = getRegistry(); - const existingQueues = registry.activeQueueNames(); - // Through the one builder, so this list has the same progress bars /jobs does - // (it used to drop `progress`, `tasks`, `drainable` and the reorder bounds). - const activeJobs = await liveJobRows((j) => BUILD_KINDS.has(j.kind)); - return ( - <div className="flex flex-col gap-4"> - <h1 className="text-2xl font-semibold">Build</h1> - <RunningJobsList jobs={activeJobs} /> - <BuildButtons existingQueues={existingQueues} /> - </div> - ); -} diff --git a/editor/app/channels/[slug]/normalizeActions.ts b/editor/app/channels/[slug]/normalizeActions.ts @@ -7,8 +7,8 @@ // undigestable — transcribeOne is the only automatic caller of // normalizeTranscript, and a `handling: "youtube"` channel fetches subtitles // with --skip-download and never reaches it — and the only fix in the repo was -// a corpus-wide button on /build. So clearing one channel meant walking all -// ~79,000 video dirs from a page nobody opens during digest work. +// a corpus-wide button in the Pool on /sites. So clearing one channel meant +// walking all ~79,000 video dirs from a page nobody opens during digest work. // // ON THE CHANNEL QUEUE, not the digest or backfill one: this is exactly the // "channel-local bookkeeping" queueKeys.ts describes channelQueueKey for, and diff --git a/editor/app/lib/nav.ts b/editor/app/lib/nav.ts @@ -3,7 +3,6 @@ import { ClipboardCheck, Cpu, Globe, - Hammer, House, LayoutDashboard, ListChecks, @@ -82,9 +81,8 @@ export const NAV_GROUPS: NavGroup[] = [ { label: "Sites", links: [ - { href: "/sites", label: "Sites", icon: Globe, keywords: "family charts stats graphs search aliases synonyms suggestions regex search terms deploy publish release cloudflare" }, - // Interim: slice 5 folds these two into the family page (/sites). - { href: "/build", label: "Build", icon: Hammer, keywords: "static export" }, + { href: "/sites", label: "Sites", icon: Globe, keywords: "family charts stats graphs search aliases synonyms suggestions regex search terms deploy publish release cloudflare build static export pool index stats archives normalize live chat" }, + // Interim: slice 5 folds this one into the family page (/sites). { href: "/homepage", label: "Homepage", icon: House, keywords: "hub" }, ], }, diff --git a/editor/app/sites/[siteId]/charts/page.tsx b/editor/app/sites/[siteId]/charts/page.tsx @@ -23,9 +23,9 @@ export default async function SiteChartsPage({ <p className="text-sm text-muted-foreground"> Author the default dashboard for <strong>{siteId}</strong>. Edits save automatically and bake into this site's next export build. Requires - the stats dataset — build it on the{" "} - <Link className="underline" href="/build"> - Build + the stats dataset — build it in the Pool on the{" "} + <Link className="underline" href="/sites"> + Sites </Link>{" "} page. The preview draws the stats dataset as last composed, which may be another site's. diff --git a/editor/app/build/components/ArchiveOptionsControl.tsx b/editor/app/sites/components/ArchiveOptionsControl.tsx diff --git a/editor/app/sites/components/BuildButtons.tsx b/editor/app/sites/components/BuildButtons.tsx @@ -0,0 +1,321 @@ +"use client"; + +import { useState } from "react"; +import { StreamActionLog } from "yt-dlp-transcript-common/components/StreamActionLog"; +import { + DEFAULT_ARCHIVE_OPTIONS, + type ArchiveBuildOptions, +} from "yt-dlp-transcript-common/lib/archiveOptions"; +import { QueueControl } from "../../components/QueueControl"; +import { cancelJobAction } from "../../jobs/actions"; +import { + archiveCombinedLiveChatAction, + archiveCombinedTranscriptsAction, + archiveLiveChatAction, + archiveTranscriptsAction, + buildIndexAction, + buildStatsAction, + normalizeLiveChatAction, + normalizeTranscriptsAction, +} from "../lib/buildAction"; +import { ArchiveOptionsControl } from "./ArchiveOptionsControl"; + +type Props = { + existingQueues: string[]; +}; + +const DEFAULT_BUILD_QUEUE = "build"; + +export function BuildButtons({ existingQueues }: Props) { + const [indexQueue, setIndexQueue] = useState(DEFAULT_BUILD_QUEUE); + const [statsQueue, setStatsQueue] = useState(DEFAULT_BUILD_QUEUE); + const [normalizeQueue, setNormalizeQueue] = useState(DEFAULT_BUILD_QUEUE); + const [archiveQueue, setArchiveQueue] = useState(DEFAULT_BUILD_QUEUE); + const [combinedQueue, setCombinedQueue] = useState(DEFAULT_BUILD_QUEUE); + const [liveChatNormalizeQueue, setLiveChatNormalizeQueue] = useState( + DEFAULT_BUILD_QUEUE, + ); + const [liveChatArchiveQueue, setLiveChatArchiveQueue] = useState( + DEFAULT_BUILD_QUEUE, + ); + const [liveChatCombinedQueue, setLiveChatCombinedQueue] = useState( + DEFAULT_BUILD_QUEUE, + ); + const [archiveOptions, setArchiveOptions] = useState<ArchiveBuildOptions>( + DEFAULT_ARCHIVE_OPTIONS, + ); + const [combinedOptions, setCombinedOptions] = useState<ArchiveBuildOptions>( + DEFAULT_ARCHIVE_OPTIONS, + ); + const [liveChatArchiveOptions, setLiveChatArchiveOptions] = + useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS); + const [liveChatCombinedOptions, setLiveChatCombinedOptions] = + useState<ArchiveBuildOptions>(DEFAULT_ARCHIVE_OPTIONS); + + return ( + <div className="flex flex-col gap-8"> + <section className="flex flex-col gap-3"> + <div> + <h2 className="text-lg font-semibold">Build index</h2> + <p className="text-sm text-muted-foreground"> + Re-scans <code>transcripts/channels/</code> and rewrites paginated + JSON in <code>export/public/</code>. Cheap when nothing changed + (mtime short-circuit). + </p> + </div> + <StreamActionLog + trigger={() => buildIndexAction(indexQueue)} + cancelAction={cancelJobAction} + buttonLabel="Build index" + runningLabel="Building index…" + extraControls={ + <QueueControl + value={indexQueue} + onChange={setIndexQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build index" + /> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Build stats dataset</h2> + <p className="text-sm text-muted-foreground"> + Extracts per-video stats (views, likes, comments, duration, + categories, …) into <code>export/public/stats/</code> for the viewer + charts. Reads engagement metrics from each{" "} + <code>metadata.info.json</code>; cue counts come from the search + index, so run <strong>Build index</strong> first. Incremental + (mtime short-circuit). + </p> + </div> + <StreamActionLog + trigger={() => buildStatsAction(statsQueue)} + cancelAction={cancelJobAction} + buttonLabel="Build stats dataset" + runningLabel="Building stats dataset…" + extraControls={ + <QueueControl + value={statsQueue} + onChange={setStatsQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build stats dataset" + /> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Normalize transcripts</h2> + <p className="text-sm text-muted-foreground"> + Walks every video and writes{" "} + <code>transcript.cues.json</code> next to each raw{" "} + <code>.vtt</code>/<code>.json</code>. Idempotent — skips entries + whose cues file is already up to date. + </p> + </div> + <StreamActionLog + trigger={() => normalizeTranscriptsAction(normalizeQueue)} + cancelAction={cancelJobAction} + buttonLabel="Normalize transcripts" + runningLabel="Normalizing transcripts…" + extraControls={ + <QueueControl + value={normalizeQueue} + onChange={setNormalizeQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Normalize transcripts" + /> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Build transcript archives</h2> + <p className="text-sm text-muted-foreground"> + Produces one <code><channel>.{archiveOptions.format}</code> per channel + under <code>transcripts/export/archives/</code>, containing each + video's compact <code>transcript.cues.json</code> plus a{" "} + <code>channel.json</code> manifest. Normalizes missing cues on + demand. + </p> + </div> + <StreamActionLog + trigger={() => archiveTranscriptsAction(archiveOptions, archiveQueue)} + cancelAction={cancelJobAction} + buttonLabel="Build archives" + runningLabel="Building archives…" + extraControls={ + <div className="flex flex-wrap items-center gap-3"> + <ArchiveOptionsControl + value={archiveOptions} + onChange={setArchiveOptions} + actionLabel="Build archives" + /> + <QueueControl + value={archiveQueue} + onChange={setArchiveQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build archives" + /> + </div> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold"> + Build combined transcript archive + </h2> + <p className="text-sm text-muted-foreground"> + Produces a single{" "} + <code>all-transcripts.{combinedOptions.format}</code> under{" "} + <code>transcripts/export/archives/</code> containing every + channel plus a top-level <code>manifest.json</code> index. + Normalizes missing cues on demand. + </p> + </div> + <StreamActionLog + trigger={() => + archiveCombinedTranscriptsAction(combinedOptions, combinedQueue) + } + cancelAction={cancelJobAction} + buttonLabel="Build combined archive" + runningLabel="Building combined archive…" + extraControls={ + <div className="flex flex-wrap items-center gap-3"> + <ArchiveOptionsControl + value={combinedOptions} + onChange={setCombinedOptions} + actionLabel="Build combined archive" + /> + <QueueControl + value={combinedQueue} + onChange={setCombinedQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build combined archive" + /> + </div> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Normalize live chat</h2> + <p className="text-sm text-muted-foreground"> + Walks every video and writes{" "} + <code>live_chat.cues.json</code> next to each raw{" "} + <code>transcript.live_chat.json</code>. Idempotent — skips + entries whose cues file is already up to date. + </p> + </div> + <StreamActionLog + trigger={() => normalizeLiveChatAction(liveChatNormalizeQueue)} + cancelAction={cancelJobAction} + buttonLabel="Normalize live chat" + runningLabel="Normalizing live chat…" + extraControls={ + <QueueControl + value={liveChatNormalizeQueue} + onChange={setLiveChatNormalizeQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Normalize live chat" + /> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Build live chat archives</h2> + <p className="text-sm text-muted-foreground"> + Produces one <code><channel>.live_chat.{liveChatArchiveOptions.format}</code> per + channel under <code>transcripts/export/archives/</code>, + containing each video's parsed{" "} + <code>live_chat.cues.json</code> plus a{" "} + <code>channel.json</code> manifest. Normalizes missing live + chats on demand. + </p> + </div> + <StreamActionLog + trigger={() => + archiveLiveChatAction(liveChatArchiveOptions, liveChatArchiveQueue) + } + cancelAction={cancelJobAction} + buttonLabel="Build live chat archives" + runningLabel="Building live chat archives…" + extraControls={ + <div className="flex flex-wrap items-center gap-3"> + <ArchiveOptionsControl + value={liveChatArchiveOptions} + onChange={setLiveChatArchiveOptions} + actionLabel="Build live chat archives" + /> + <QueueControl + value={liveChatArchiveQueue} + onChange={setLiveChatArchiveQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build live chat archives" + /> + </div> + } + /> + </section> + + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold"> + Build combined live chat archive + </h2> + <p className="text-sm text-muted-foreground"> + Produces a single <code>all-live-chat.{liveChatCombinedOptions.format}</code> under{" "} + <code>transcripts/export/archives/</code> containing every + channel's parsed live chats plus a top-level{" "} + <code>manifest.json</code> index. Normalizes missing live chats + on demand. + </p> + </div> + <StreamActionLog + trigger={() => + archiveCombinedLiveChatAction( + liveChatCombinedOptions, + liveChatCombinedQueue, + ) + } + cancelAction={cancelJobAction} + buttonLabel="Build combined live chat archive" + runningLabel="Building combined live chat archive…" + extraControls={ + <div className="flex flex-wrap items-center gap-3"> + <ArchiveOptionsControl + value={liveChatCombinedOptions} + onChange={setLiveChatCombinedOptions} + actionLabel="Build combined live chat archive" + /> + <QueueControl + value={liveChatCombinedQueue} + onChange={setLiveChatCombinedQueue} + defaultQueueKey={DEFAULT_BUILD_QUEUE} + existingQueues={existingQueues} + actionLabel="Build combined live chat archive" + /> + </div> + } + /> + </section> + </div> + ); +} diff --git a/editor/app/sites/components/EditorChartsClient.tsx b/editor/app/sites/components/EditorChartsClient.tsx @@ -34,7 +34,7 @@ function Inner({ <p className="text-sm text-destructive"> Failed to load the stats dataset ({error.message}). Run{" "} <strong>Build index</strong> then <strong>Build stats dataset</strong>{" "} - on the Build page first. + under Pool on the Sites page first. </p> ); } diff --git a/editor/app/sites/page.tsx b/editor/app/sites/page.tsx @@ -11,7 +11,11 @@ import { import { getPaths } from "yt-dlp-transcript-common/lib/paths"; import { getSettings } from "yt-dlp-transcript-common/lib/settings"; import { listSites } from "yt-dlp-transcript-common/lib/site"; +import { getRegistry } from "yt-dlp-transcript-common/jobs/registry"; +import { liveJobRows } from "../jobs/active/buildActiveJobs"; +import { RunningJobsList } from "../jobs/components/RunningJobsList"; import { BuildAllSitesButton } from "./components/BuildAllSitesButton"; +import { BuildButtons } from "./components/BuildButtons"; import { BuildModeToggle } from "./components/BuildModeToggle"; import { BuildSitesPanel } from "./components/BuildSitesPanel"; import { CutReleaseForm } from "./components/CutReleaseForm"; @@ -20,6 +24,20 @@ import { DeleteSiteButton, MigrateButton } from "./components/SiteListActions"; export const dynamic = "force-dynamic"; export const metadata: Metadata = { title: "Sites" }; +// Which live jobs the Pool section lists. SIX KINDS, AND THE SET IS WRONG IN +// BOTH DIRECTIONS: the three live-chat kinds the buttons below enqueue are +// missing, and build-export (enqueued from a site's Publish tab, never here) +// is in. Carried as-is from /build — recorded in plans/editor-ia-slice-5.md, +// not fixed by it. +const BUILD_KINDS = new Set([ + "build-index", + "build-stats", + "build-export", + "normalize-transcripts", + "archive-transcripts", + "archive-combined-transcripts", +]); + function loadExportChangelog(): string | null { try { return readFileSync(path.join(getPaths().exportDir, "CHANGELOG.md"), "utf8"); @@ -45,6 +63,10 @@ export default async function SitesPage() { const unreleased = source ? extractUnreleasedSection(source) : ""; const hasUnreleased = unreleased.length > 0; const buildMode = getSettings().buildPipeline.mode; + const existingQueues = getRegistry().activeQueueNames(); + // Through the one builder, so this list has the same progress bars /jobs does + // (it used to drop `progress`, `tasks`, `drainable` and the reorder bounds). + const activeJobs = await liveJobRows((j) => BUILD_KINDS.has(j.kind)); return ( <div className="flex flex-col gap-8"> @@ -150,6 +172,30 @@ export default async function SitesPage() { serial={buildMode === "basic"} /> </section> + + {/* The shared pool: corpus-wide, no site involved. */} + <section className="flex flex-col gap-3 border-t border-border pt-6"> + <div> + <h2 className="text-lg font-semibold">Pool</h2> + <p className="text-sm text-muted-foreground"> + The shared channel pool every site is a selection over: the search + index, the stats dataset, the transcript and live-chat sidecars and + the downloadable archives. Corpus-wide — no site is involved. + </p> + </div> + {/* Above the disclosure: a running pool job must be visible without a + click. */} + <RunningJobsList jobs={activeJobs} /> + {/* Eight job consoles; the family page does not open on them. */} + <details> + <summary className="cursor-pointer text-lg font-semibold select-none"> + Pool jobs + </summary> + <div className="pt-4"> + <BuildButtons existingQueues={existingQueues} /> + </div> + </details> + </section> </div> ); } diff --git a/editor/e2e/availability.spec.ts b/editor/e2e/availability.spec.ts @@ -1,6 +1,7 @@ import { stat } from "node:fs/promises"; import { test, expect } from "@playwright/test"; import { + buildIndex, channelStage, generateReport, pathExists, @@ -187,11 +188,7 @@ test("build index marks deleted videos in the export summaries", async ({ await writeSite("testsite", { channels: [{ slug: CHANNEL, groupId: "default" }], }); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done in", { - timeout: 30_000, - }); + await buildIndex(page); const pageJson = await readJson< Array<{ id: string; isDeleted: boolean }> diff --git a/editor/e2e/build.spec.ts b/editor/e2e/build.spec.ts @@ -1,16 +1,18 @@ import { test, expect } from "@playwright/test"; -import { pathExists, readJson, resetData, writeSite } from "./helpers"; +import { + buildIndex, + pathExists, + readJson, + resetData, + writeSite, +} from "./helpers"; test("build index against an empty site writes a 0-transcript manifest", async ({ page, }) => { await resetData("empty"); await writeSite("testsite", { channels: [] }); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 20_000, - }); + await buildIndex(page); const manifest = await readJson<{ totalCount: number; siteId: string }>( "test-transcripts/.export-index/sites/testsite/summaries/manifest.json", ); @@ -25,11 +27,7 @@ test("build index ingests synthetic video data into shared + per-site output", a await writeSite("testsite", { channels: [{ slug: "test-youtube", groupId: "default" }], }); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 20_000, - }); + await buildIndex(page); const manifest = await readJson<{ totalCount: number; channels: { name: string }[]; @@ -52,11 +50,7 @@ test("two sites over a shared pool get isolated, differently-filtered summaries" channels: [{ slug: "test-youtube", groupId: "default" }], }); await writeSite("without-channel", { channels: [] }); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 20_000, - }); + await buildIndex(page); const withChannel = await readJson<{ totalCount: number }>( "test-transcripts/.export-index/sites/with-channel/summaries/manifest.json", ); @@ -67,16 +61,17 @@ test("two sites over a shared pool get isolated, differently-filtered summaries" expect(without.totalCount).toBe(0); }); -test("build static export lives on the deploy page, not the build page", async ({ +test("build static export lives on the site's Publish tab, not the family page", async ({ page, }) => { await resetData("empty"); - // A configured site is the active build target; without one the export button - // is (correctly) disabled, so add one to keep the enabled check meaningful. + // A configured site is the build target; the Publish tab needs one to exist + // at all, and the enabled check below is about that site. await writeSite("testsite", {}); - // No longer on the build page. - await page.goto("/build"); + // Not in the family page's Pool. + await page.goto("/sites"); + await page.getByText("Pool jobs", { exact: true }).click(); await expect( page.getByRole("heading", { name: "Build static export" }), ).toHaveCount(0); diff --git a/editor/e2e/channel-build-toggle.spec.ts b/editor/e2e/channel-build-toggle.spec.ts @@ -1,5 +1,11 @@ import { test, expect } from "@playwright/test"; -import { pathExists, readJson, resetData, writeSite } from "./helpers"; +import { + buildIndex, + pathExists, + readJson, + resetData, + writeSite, +} from "./helpers"; import type { ChannelConfig } from "yt-dlp-transcript-common/lib/channelConfig"; // The build-inclusion toggle on /channels flips `excludeFromBuild` in the @@ -13,14 +19,6 @@ const MANIFEST_PATH = const CHANNEL_OUT_DIR = "test-transcripts/.export-index/shared/transcripts/test-youtube"; -async function runBuild(page: import("@playwright/test").Page) { - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 20_000, - }); -} - test("toggling a channel off skips it from the build; toggling back restores it", async ({ page, }) => { @@ -29,7 +27,7 @@ test("toggling a channel off skips it from the build; toggling back restores it" channels: [{ slug: "test-youtube", groupId: "default" }], }); - await runBuild(page); + await buildIndex(page); { const manifest = await readJson<{ totalCount: number; @@ -62,7 +60,7 @@ test("toggling a channel off skips it from the build; toggling back restores it" expect(cfg.excludeFromBuild).toBe(true); } - await runBuild(page); + await buildIndex(page); { const manifest = await readJson<{ totalCount: number; @@ -90,7 +88,7 @@ test("toggling a channel off skips it from the build; toggling back restores it" expect(cfg.excludeFromBuild).toBeUndefined(); } - await runBuild(page); + await buildIndex(page); { const manifest = await readJson<{ totalCount: number; diff --git a/editor/e2e/dashboard.spec.ts b/editor/e2e/dashboard.spec.ts @@ -105,7 +105,7 @@ test("pausing downloads persists and blocks a manual sync", async ({ page }) => await expect(row.getByRole("alert")).toContainText("Downloads are paused"); }); -test("sidebar links to Channels, Build, Jobs, Settings", async ({ page }) => { +test("sidebar links to Channels, Sites, Jobs, Settings", async ({ page }) => { await resetData(); await page.goto("/"); const sidebar = page.locator("aside"); @@ -113,9 +113,9 @@ test("sidebar links to Channels, Build, Jobs, Settings", async ({ page }) => { "href", "/channels", ); - await expect(sidebar.getByRole("link", { name: /build/i })).toHaveAttribute( + await expect(sidebar.getByRole("link", { name: /sites/i })).toHaveAttribute( "href", - "/build", + "/sites", ); await expect(sidebar.getByRole("link", { name: /jobs/i })).toHaveAttribute( "href", diff --git a/editor/e2e/duplicate-shorts.spec.ts b/editor/e2e/duplicate-shorts.spec.ts @@ -1,6 +1,12 @@ import { mkdir, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { readJson, resetData, resolvePath, writeSite } from "./helpers"; +import { + buildIndex, + readJson, + resetData, + resolvePath, + writeSite, +} from "./helpers"; // Cross-platform duplicate shorts detection. Seeds four channels with shorts // whose transcripts are near-identical across platforms/channels, plus a unique @@ -175,11 +181,8 @@ async function seed(seeds: Seed[] = SEEDS): Promise<void> { } async function buildData(page: import("@playwright/test").Page): Promise<void> { - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 30_000, - }); + await buildIndex(page); + // Same page, disclosure still open. await page.getByRole("button", { name: "Build stats dataset" }).click(); await expect(page.getByLabel("Build stats dataset output")).toContainText( "Stats built", diff --git a/editor/e2e/helpers.ts b/editor/e2e/helpers.ts @@ -9,7 +9,7 @@ import { utimes, writeFile, } from "node:fs/promises"; -import type { Page } from "@playwright/test"; +import { expect, type Page } from "@playwright/test"; import { baseUrl } from "./baseUrl"; const here = dirname(fileURLToPath(import.meta.url)); @@ -389,3 +389,18 @@ export async function writeChannelConfig( ), ); } + +// Run the corpus-wide Build index job from the family page, the way a user +// does: open the Pool disclosure, press the button, wait for "Done". The +// controls sit under <details> so /sites does not open on eight job consoles, +// and getByRole ignores what a closed disclosure hides — hence the click first. +// Replaces the six-line copy three specs carried inline. goto() resets the +// disclosure, so calling this twice in one test is fine. +export async function buildIndex(page: Page, timeout = 30_000) { + await page.goto("/sites"); + await page.getByText("Pool jobs", { exact: true }).click(); + await page.getByRole("button", { name: "Build index" }).click(); + await expect(page.getByLabel("Build index output")).toContainText("Done", { + timeout, + }); +} diff --git a/editor/e2e/jobs.spec.ts b/editor/e2e/jobs.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from "@playwright/test"; -import { resetData } from "./helpers"; +import { buildIndex, resetData } from "./helpers"; test("empty state when no jobs have run", async ({ page }) => { await resetData("empty"); @@ -18,11 +18,7 @@ test("kicks off a build-index job, lists it, and tails its log", async ({ page, }) => { await resetData("one-youtube-channel-with-data"); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 30_000, - }); + await buildIndex(page); await page.goto("/jobs"); const buildRow = page diff --git a/editor/e2e/navigation.spec.ts b/editor/e2e/navigation.spec.ts @@ -130,7 +130,7 @@ test.describe("navigation", () => { } await writeSite("testsite", {}); - for (const retired of ["/charts", "/aliases", "/deploy"]) { + for (const retired of ["/charts", "/aliases", "/deploy", "/build"]) { await page.goto(retired); await expect(page).toHaveURL(/\/sites$/); await expect( diff --git a/editor/e2e/regional-vtt-fallback.spec.ts b/editor/e2e/regional-vtt-fallback.spec.ts @@ -1,6 +1,12 @@ import { rm, writeFile } from "node:fs/promises"; import { test, expect } from "@playwright/test"; -import { readJson, resetData, resolvePath, writeSite } from "./helpers"; +import { + buildIndex, + readJson, + resetData, + resolvePath, + writeSite, +} from "./helpers"; const CHANNEL = "test-youtube"; const VIDEO_DIR = "20240101_test1234567"; @@ -15,15 +21,14 @@ const VTT = type Detail = { id: string; cues?: Array<unknown> }; -async function buildIndex(page: import("@playwright/test").Page) { +// The site has to exist before the index build runs; buildIndex() from helpers +// is the shared "open the Pool, press the button, wait for Done" (the local name +// would shadow the import, hence seedAndBuild). +async function seedAndBuild(page: import("@playwright/test").Page) { await writeSite("testsite", { channels: [{ slug: CHANNEL, groupId: "default" }], }); - await page.goto("/build"); - await page.getByRole("button", { name: "Build index" }).click(); - await expect(page.getByLabel("Build index output")).toContainText("Done", { - timeout: 30_000, - }); + await buildIndex(page); } // YouTube sometimes serves a video's English captions only under a regional or @@ -38,7 +43,7 @@ test("build index parses a video with only transcript.en-US.vtt", async ({ await rm(resolvePath(`${DATA}/transcript.en.vtt`), { force: true }); await writeFile(resolvePath(`${DATA}/transcript.en-US.vtt`), VTT); - await buildIndex(page); + await seedAndBuild(page); const detailsPage = await readJson<Detail[]>(TRANSCRIPTS_PAGE); const detail = detailsPage.find((d) => d.id === VIDEO_ID); @@ -55,7 +60,7 @@ test("build index ignores a non-English translation track", async ({ page }) => await rm(resolvePath(`${DATA}/transcript.en.vtt`), { force: true }); await writeFile(resolvePath(`${DATA}/transcript.es-en-US.vtt`), VTT); - await buildIndex(page); + await seedAndBuild(page); const detailsPage = await readJson<Detail[]>(TRANSCRIPTS_PAGE); const detail = detailsPage.find((d) => d.id === VIDEO_ID); diff --git a/editor/next.config.ts b/editor/next.config.ts @@ -102,6 +102,7 @@ const nextConfig: NextConfig = { { source: "/aliases", destination: "/sites", permanent: false }, { source: "/deploy", has: [siteQuery], destination: "/sites/:site/publish", permanent: false }, { source: "/deploy", destination: "/sites", permanent: false }, + { source: "/build", destination: "/sites", permanent: false }, ]; }, // Serve the built export artifacts (stats/summaries/transcripts) through a