// AN ODYSEE OR BITCHUTE CHANNEL'S REMOTE LISTING, DIFFED AGAINST WHAT IS HELD // (release 19 A6, `pnpm ops get remote-listing `). // // One flat-playlist enumeration of the channel's own URL — the same one-spawn // read the quick availability check and a sync's full sweep make, with the // platform's paced args (`configArgs`) — on the platform's own queue, so it is // ONE STREAM with every other request to that platform. Nothing is written: no // roster merge, no maybe-missing, no download. The answer is the diff: // // notHeld listed upstream, no data// here — what a download would get // heldNotListed held here, absent from the listing (deleted, unlisted, or a // record imported from elsewhere) // // POLITE: the editor refuses it while the platform is held or cooling down, // waits out the platform's import floor before the read (jobs/platformGap.ts), // and a rate-limited read backs the platform off (the caller records it). // Ids are the URL-derived canonical ids, which ARE the data-dir names on both // platforms (lib/videoId.ts). import path from "node:path"; import type { ChannelConfig } from "../lib/channelConfig"; import type { Paths } from "../lib/paths"; import { assertChannelTextReadable } from "../lib/channelMedia"; import { extractVideoId } from "../lib/videoId"; import { fetchFlatPlaylistUrls } from "../ytdlp/runYtdlp"; import { heldVideoIds } from "./videoCues"; export { heldVideoIds }; export const REMOTE_LISTING_PLATFORMS = ["odysee", "bitchute"] as const; export type RemoteListingPlatform = (typeof REMOTE_LISTING_PLATFORMS)[number]; export function isRemoteListingPlatform(p: string | null | undefined): p is RemoteListingPlatform { return (REMOTE_LISTING_PLATFORMS as readonly string[]).includes(p ?? ""); } // The log line a remote-listing job ends with: this marker, then the result as // compact JSON (`pnpm ops get remote-listing` prints it on stdout). The same // string is REMOTE_LISTING_RESULT_MARKER in scripts/archilyzer-ops.mjs. export const REMOTE_LISTING_RESULT_MARKER = "@@remote-listing "; export type RemoteListingResult = { slug: string; platform: RemoteListingPlatform; url: string; listedAt: string; // Entries the enumeration printed, and how many named no video id. listed: number; unparsed: number; held: number; notHeld: { id: string; url: string }[]; heldNotListed: string[]; }; // The pure half: a listing's URLs against the held ids. Listing order is kept // (newest first, as the platform lists); duplicates collapse. export function diffRemoteListing( urls: string[], heldIds: ReadonlySet, ): Pick { const seen = new Set(); const notHeld: { id: string; url: string }[] = []; let unparsed = 0; let held = 0; for (const url of urls) { const id = extractVideoId(url); if (!id) { unparsed++; continue; } if (seen.has(id)) continue; seen.add(id); if (heldIds.has(id)) held++; else notHeld.push({ id, url }); } const heldNotListed = [...heldIds].filter((id) => !seen.has(id)).sort(); return { listed: urls.length, unparsed, held, notHeld, heldNotListed }; } export type RemoteListingDeps = { list?: typeof fetchFlatPlaylistUrls; now?: () => Date; }; export async function runRemoteListing(opts: { paths: Paths; slug: string; channelConfig: ChannelConfig; platform: RemoteListingPlatform; onLog: (line: string) => void; signal: AbortSignal; deps?: RemoteListingDeps; }): Promise { const list = opts.deps?.list ?? fetchFlatPlaylistUrls; const now = opts.deps?.now ?? (() => new Date()); const url = opts.channelConfig.url; if (!url) throw new Error(`Channel "${opts.slug}" has no URL configured`); // The text guard: an unmounted or moving drive is not "nothing held". await assertChannelTextReadable(opts.paths, opts.slug, opts.channelConfig); const urls = await list({ channelConfig: opts.channelConfig, paths: opts.paths, channelSlug: opts.slug, onLog: opts.onLog, signal: opts.signal, }); const held = await heldVideoIds(path.join(opts.paths.channelsDir, opts.slug, "data")); const diff = diffRemoteListing(urls, held); const result: RemoteListingResult = { slug: opts.slug, platform: opts.platform, url, listedAt: now().toISOString(), ...diff, }; opts.onLog( `Remote listing of ${opts.slug} (${opts.platform}): ${diff.listed} listed, ${diff.held} held, ` + `${diff.notHeld.length} not held, ${diff.heldNotListed.length} held but not listed` + `${diff.unparsed ? `, ${diff.unparsed} entries with no video id` : ""}.\n`, ); return result; }