commit 92841d1f0c314c32befd885c6093c96d1e1dba48
parent 933a03b1449d1fd046aeea223335536ce6f3e63a
Author: I Mean I'm Just Saying <imeanimjustsaying@kiwifarms.st>
Date: Fri, 25 Sep 2026 16:12:35 -0400
common: homepage summary v5 — monthly back catalogue by upload month, per-site numbers, official totals
`monthly` counts transcribed public-universe recordings by the month the
creator uploaded them, stacked by primary public site, full history and
zero-filled; `monthlyUnplaced` counts the ones with no usable upload date.
Each site gains channels / recordings / hoursArchived / gone, and `official`
sums them over the public sites (the instance-wide `totals` are unchanged).
All additive and optional on the type, so a v4 summary still renders.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Diffstat:
2 files changed, 242 insertions(+), 1 deletion(-)
diff --git a/common/lib/homepageSummary.test.ts b/common/lib/homepageSummary.test.ts
@@ -0,0 +1,130 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { buildHomepageSummary, HOMEPAGE_SUMMARY_VERSION } from "./homepageSummary";
+import type { VideoStat } from "./stats";
+import type { Site } from "./site";
+
+// The v5 additions: `monthly` (transcripts by UPLOAD month, stacked by primary
+// public site, full history, zero-filled), per-site numbers, and `official`
+// (the sum over the public sites, not the instance-wide totals).
+//
+// Limits, pinned here rather than discovered: `monthly` is NOT capped — it runs
+// from the earliest upload month to the build month, however old. A transcript
+// whose upload date is missing, malformed or after the build month is not
+// placed in any month; `monthlyUnplaced` counts it. Channels exposed by no
+// public site (pool-only, or a site without a siteUrl) are outside `monthly`,
+// the site numbers and `official`, but still inside `totals`.
+
+const NOW = new Date("2026-03-15T12:00:00Z");
+
+function stat(p: Partial<VideoStat> & { channelSlug: string; id: string }): VideoStat {
+ return {
+ slug: `${p.channelSlug}/${p.id}`,
+ channel: p.channelSlug.toUpperCase(),
+ title: p.id,
+ platform: "youtube",
+ uploadDate: "20260101",
+ downloadedDate: "20260301",
+ transcribedDate: "20260302",
+ timestamp: null,
+ duration: 1800,
+ viewCount: null,
+ likeCount: null,
+ commentCount: null,
+ channelFollowerCount: null,
+ categories: [],
+ tags: [],
+ language: null,
+ isLivestream: false,
+ mediaType: "video",
+ status: "available",
+ hasTranscript: true,
+ cueCount: null,
+ coverage: null,
+ ...p,
+ } as VideoStat;
+}
+
+function site(siteId: string, slugs: string[], siteUrl?: string): Site {
+ return {
+ siteId,
+ siteTitle: siteId.toUpperCase(),
+ siteDescription: `${siteId} desc`,
+ channels: slugs.map((slug) => ({ slug })),
+ ...(siteUrl ? { siteUrl } : {}),
+ } as unknown as Site;
+}
+
+const SITES = [
+ site("alpha", ["a1", "a2"], "https://alpha.example"),
+ site("beta", ["b1"], "https://beta.example"),
+ site("private", ["p1"]), // no siteUrl: not an official instance
+];
+const CHANNEL_SITES = { a1: ["alpha"], a2: ["alpha"], b1: ["beta"], p1: ["private"] };
+
+const STATS: VideoStat[] = [
+ stat({ channelSlug: "a1", id: "x1", uploadDate: "20251101", duration: 3600 }),
+ stat({ channelSlug: "a1", id: "x2", uploadDate: "20251101", duration: 3600, status: "deleted" }),
+ stat({ channelSlug: "a2", id: "x3", uploadDate: "20260203", duration: 1800 }),
+ // downloaded, not transcribed: a recording, not a transcript, not placed
+ stat({ channelSlug: "a2", id: "x4", hasTranscript: false, transcribedDate: null }),
+ stat({ channelSlug: "b1", id: "y1", uploadDate: "20260115", duration: 7200 }),
+ // unplaceable upload dates: missing, malformed, future
+ stat({ channelSlug: "b1", id: "y2", uploadDate: "" }),
+ stat({ channelSlug: "b1", id: "y3", uploadDate: "2026-01" }),
+ stat({ channelSlug: "b1", id: "y4", uploadDate: "20260501" }),
+ // pool-only (no public site): counted in totals, nowhere else
+ stat({ channelSlug: "p1", id: "z1", uploadDate: "20200101", status: "deleted" }),
+];
+
+test("monthly: upload months, zero-filled from first upload to the build month, every site keyed", () => {
+ const s = buildHomepageSummary(STATS, CHANNEL_SITES, SITES, NOW);
+ assert.equal(s.version, HOMEPAGE_SUMMARY_VERSION);
+ assert.deepEqual(
+ s.monthly!.map((m) => m.month),
+ ["2025-11", "2025-12", "2026-01", "2026-02", "2026-03"],
+ );
+ assert.deepEqual(s.monthly![0].bySite, { alpha: 2, beta: 0 });
+ assert.deepEqual(s.monthly![1].bySite, { alpha: 0, beta: 0 });
+ assert.deepEqual(s.monthly![2].bySite, { alpha: 0, beta: 1 });
+ assert.deepEqual(s.monthly![3].bySite, { alpha: 1, beta: 0 });
+ assert.deepEqual(s.monthly![4].bySite, { alpha: 0, beta: 0 });
+ // Placed + unplaced = every transcribed public-universe record.
+ const placed = s.monthly!.reduce((a, m) => a + m.bySite.alpha + m.bySite.beta, 0);
+ assert.equal(s.monthlyUnplaced, 3);
+ assert.equal(placed + s.monthlyUnplaced!, s.official!.transcripts);
+});
+
+test("per-site numbers and official totals sum over the public sites only", () => {
+ const s = buildHomepageSummary(STATS, CHANNEL_SITES, SITES, NOW);
+ const alpha = s.sites.find((x) => x.siteId === "alpha")!;
+ const beta = s.sites.find((x) => x.siteId === "beta")!;
+ assert.equal(s.sites.length, 2);
+ assert.deepEqual(
+ [alpha.channels, alpha.recordings, alpha.transcribed.total, alpha.hoursArchived, alpha.gone],
+ [2, 4, 3, 3, 1], // 3600 + 3600 + 1800 s = 2.5 h -> 3
+ );
+ assert.deepEqual(
+ [beta.channels, beta.recordings, beta.transcribed.total, beta.hoursArchived, beta.gone],
+ [1, 4, 4, 4, 0], // 7200 + 3 x 1800 s = 3.5 h -> 4
+ );
+ // Hours are rounded once over the summed seconds, not per site.
+ assert.deepEqual(s.official, {
+ sites: 2,
+ channels: 3,
+ recordings: 8,
+ transcripts: 7,
+ hoursArchived: 6, // 21600 s
+ gone: 1,
+ });
+ // The instance-wide totals still count the pool-only channel.
+ assert.equal(s.totals.transcripts, 8);
+ assert.equal(s.availability!.byState.deleted, 2);
+});
+
+test("no transcribed public content: monthly is empty, not a run of zeroes", () => {
+ const s = buildHomepageSummary([STATS[8]], CHANNEL_SITES, SITES, NOW);
+ assert.deepEqual(s.monthly, []);
+ assert.equal(s.monthlyUnplaced, 0);
+ assert.equal(s.official!.sites, 0);
+});
diff --git a/common/lib/homepageSummary.ts b/common/lib/homepageSummary.ts
@@ -22,7 +22,11 @@ import { VIDEO_STATES, type VideoState } from "./availability";
// on the type, because nothing gates on this number — it is a provenance marker,
// not a compatibility check — and a summary written by an older build must keep
// deserializing. Readers must treat their absence as "unknown", never as zero.
-export const HOMEPAGE_SUMMARY_VERSION = 4;
+//
+// v5 adds `monthly`, `official` and the per-site `channels` / `recordings` /
+// `hoursArchived` / `gone` numbers — all additive, all optional on the type for
+// the same reason: a v4 summary on disk must still render (the numbers hide).
+export const HOMEPAGE_SUMMARY_VERSION = 5;
// Day buckets are capped to this many trailing days so the embedded summary stays
// small regardless of archive age (daily detail is only useful recently).
@@ -64,6 +68,13 @@ export type HomepageSummarySite = {
siteUrl: string; // public sites only — always present
transcribed: SiteMetricStat;
downloaded: SiteMetricStat;
+ // v5 per-site numbers, over the channels this site is the PRIMARY public site
+ // for (the same attribution as `transcribed`/`downloaded`, so the cards sum to
+ // `official`). Optional: absent from pre-v5 summaries.
+ channels?: number; // channels with at least one transcript
+ recordings?: number; // records with a download date (matches totals.downloads)
+ hoursArchived?: number; // transcribed-video durations, rounded to the hour
+ gone?: number; // records re-checked and found deleted at the source
// The site's own brand accent ("#rrggbb"), when it defines one. Absent for a
// site that hasn't set one — callers fall back to the chart palette rather
// than inventing a brand colour. Added in v4.
@@ -105,6 +116,30 @@ export type HomepageAvailability = {
counted: number;
};
+// One calendar month of the archive's BACK CATALOGUE: transcribed recordings in
+// the public universe, keyed by the month the creator UPLOADED them (not the
+// month we transcribed them — that history is only as old as the tool, while
+// upload month shows how far back each archive reaches). Values are keyed by
+// siteId and attributed to the primary public site, so a month's values sum
+// to that month's transcripts with no double counting.
+export type HomepageMonth = {
+ month: string; // "YYYY-MM"
+ bySite: Record<string, number>;
+};
+
+// The family's own numbers: the sum over `sites` (public sites with activity),
+// NOT the instance-wide `totals`, which also count pool-only channels and
+// sites without a public URL. Copy that says "across the official instances"
+// reads this.
+export type HomepageOfficialTotals = {
+ sites: number;
+ channels: number;
+ recordings: number;
+ transcripts: number;
+ hoursArchived: number;
+ gone: number;
+};
+
export type HomepageSummary = {
version: number;
generatedAt: string; // ISO timestamp
@@ -128,6 +163,15 @@ export type HomepageSummary = {
recent: HomepageRecentItem[];
// Instance-wide state census. Optional — absent from pre-v4 summaries.
availability?: HomepageAvailability;
+ // v5. Full history, one entry per month from the earliest upload month in the
+ // public universe to the build month, zero-filled between. Every site in
+ // `sites` has a key in every entry (zero included). Empty when there is no
+ // transcribed public content. Transcripts whose upload date is missing,
+ // malformed or after the build month are not placed; `monthlyUnplaced`
+ // counts them so nothing is dropped silently.
+ monthly?: HomepageMonth[];
+ monthlyUnplaced?: number;
+ official?: HomepageOfficialTotals;
};
const RECENT_LIMIT = 24;
@@ -139,6 +183,15 @@ function monthOf(date: string | null): string | null {
return `${date.slice(0, 4)}-${date.slice(4, 6)}`;
}
+// An upload date "YYYYMMDD" -> "YYYY-MM", or null when it is not a plausible
+// date (missing, short, non-numeric, month outside 1..12).
+function uploadMonthOf(date: string | null | undefined): string | null {
+ if (!date || !/^\d{8}$/.test(date)) return null;
+ const m = +date.slice(4, 6);
+ if (m < 1 || m > 12 || +date.slice(0, 4) < 1900) return null;
+ return `${date.slice(0, 4)}-${date.slice(4, 6)}`;
+}
+
function monthRange(start: string, end: string): string[] {
const out: string[] = [];
let [y, m] = start.split("-").map(Number);
@@ -284,6 +337,17 @@ export function buildHomepageSummary(
let transcribedThisMonth = 0;
let downloadedThisMonth = 0;
const channelSet = new Set<string>();
+ // Per public site: channels with a transcript, recordings, seconds, gone.
+ type SiteAcc = { channels: Set<string>; recordings: number; seconds: number; gone: number };
+ const siteAcc = new Map<string, SiteAcc>();
+ const accOf = (id: string): SiteAcc => {
+ let a = siteAcc.get(id);
+ if (!a) siteAcc.set(id, (a = { channels: new Set(), recordings: 0, seconds: 0, gone: 0 }));
+ return a;
+ };
+ // Upload-month placement of transcribed public-universe records.
+ const uploadItems: { month: string; siteId: string }[] = [];
+ let monthlyUnplaced = 0;
for (const s of stats) {
const hasTx = s.hasTranscript && !!s.transcribedDate;
@@ -300,7 +364,15 @@ export function buildHomepageSummary(
const siteId = primarySiteOf.get(s.channelSlug);
if (!siteId) continue; // not on any public site
channelName.set(s.channelSlug, s.channel);
+ const acc = accOf(siteId);
+ if (s.downloadedDate) acc.recordings += 1;
+ if (s.status === "deleted") acc.gone += 1;
if (hasTx) {
+ acc.channels.add(s.channelSlug);
+ acc.seconds += s.duration > 0 ? s.duration : 0;
+ const um = uploadMonthOf(s.uploadDate);
+ if (um && um <= nowMonth) uploadItems.push({ month: um, siteId });
+ else monthlyUnplaced += 1;
transcribedItems.push({ date: s.transcribedDate as string, channelSlug: s.channelSlug, siteId });
}
if (s.downloadedDate) {
@@ -326,6 +398,10 @@ export function buildHomepageSummary(
siteUrl: s.siteUrl,
transcribed: siteStatFrom(series.transcribed.month, s.siteId, nowMonth),
downloaded: siteStatFrom(series.downloaded.month, s.siteId, nowMonth),
+ channels: siteAcc.get(s.siteId)?.channels.size ?? 0,
+ recordings: siteAcc.get(s.siteId)?.recordings ?? 0,
+ hoursArchived: Math.round((siteAcc.get(s.siteId)?.seconds ?? 0) / 3600),
+ gone: siteAcc.get(s.siteId)?.gone ?? 0,
...(s.accent ? { accent: s.accent } : {}),
}))
// Keep a public site only if it has any activity in either metric.
@@ -379,6 +455,38 @@ export function buildHomepageSummary(
counted: stats.length,
};
+ // Monthly back-catalogue series, keyed by the sites that survived the
+ // activity filter (a site with no activity has no transcripts to place).
+ const keptIds = summarySites.map((s) => s.siteId);
+ let firstMonth: string | null = null;
+ for (const it of uploadItems) {
+ if (firstMonth === null || it.month < firstMonth) firstMonth = it.month;
+ }
+ const monthly: HomepageMonth[] = firstMonth
+ ? monthRange(firstMonth, nowMonth).map((month) => ({
+ month,
+ bySite: Object.fromEntries(keptIds.map((id) => [id, 0])),
+ }))
+ : [];
+ const monthIndex = new Map(monthly.map((m, i) => [m.month, i]));
+ for (const it of uploadItems) {
+ const i = monthIndex.get(it.month);
+ if (i === undefined) continue; // unreachable: every item's month is in range
+ monthly[i].bySite[it.siteId] = (monthly[i].bySite[it.siteId] ?? 0) + 1;
+ }
+
+ // The family's own totals — hours summed in seconds, then rounded once.
+ let officialSeconds = 0;
+ for (const id of keptIds) officialSeconds += siteAcc.get(id)?.seconds ?? 0;
+ const official: HomepageOfficialTotals = {
+ sites: summarySites.length,
+ channels: summarySites.reduce((a, s) => a + (s.channels ?? 0), 0),
+ recordings: summarySites.reduce((a, s) => a + (s.recordings ?? 0), 0),
+ transcripts: summarySites.reduce((a, s) => a + s.transcribed.total, 0),
+ hoursArchived: Math.round(officialSeconds / 3600),
+ gone: summarySites.reduce((a, s) => a + (s.gone ?? 0), 0),
+ };
+
return {
version: HOMEPAGE_SUMMARY_VERSION,
generatedAt: now.toISOString(),
@@ -396,5 +504,8 @@ export function buildHomepageSummary(
sites: summarySites,
recent,
availability,
+ monthly,
+ monthlyUnplaced,
+ official,
};
}
