import { test, after } from "node:test"; import assert from "node:assert/strict"; import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { gzipSync } from "node:zlib"; import { execFileSync } from "node:child_process"; import { SourceRefusal, auditBare, auditFiles, auditObjects, formatAuditReport, literalLabel, maskLiterals, objectField, parseDenylist, runGitleaks, scanBuffer, treeEntryNames, type Literal, } from "./sourceAudit"; // Run with: // pnpm --filter yt-dlp-transcript-common test // // The source mirror's gate (sourceAudit.ts): the literal list, the byte scan, // the redaction, the object walk over a real temp repo, the staged-file sweep // and gitleaks' absence. Every repo is built in the OS temp dir; the git // variables a hook or a wrapper might export are cleared first so nothing here // can reach the real repo. for (const key of ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_PREFIX"]) { delete process.env[key]; } const TMP = mkdtempSync(path.join(os.tmpdir(), "source-audit-")); after(() => rmSync(TMP, { recursive: true, force: true })); // The planted literal every test hunts for. Never a real name. const PLANTED = "plantedhome"; const lits = (...xs: string[]): Literal[] => xs.map((x, i) => ({ bytes: Buffer.from(x), ci: false, from: `denylist line ${i + 1}` })); let n = 0; function repo(): string { const dir = path.join(TMP, `r${n++}`); mkdirSync(dir); const git = (...args: string[]) => execFileSync("git", args, { cwd: dir, stdio: "pipe" }); git("init", "-q", "-b", "main"); git("config", "user.name", "audit test"); git("config", "user.email", "audit@example.invalid"); git("config", "commit.gpgsign", "false"); return dir; } function commit(dir: string, files: Record, message: string, author?: string): void { for (const [f, text] of Object.entries(files)) { mkdirSync(path.dirname(path.join(dir, f)), { recursive: true }); writeFileSync(path.join(dir, f), text); } const git = (...args: string[]) => execFileSync("git", args, { cwd: dir, stdio: "pipe" }); git("add", "-A"); git("commit", "-q", "-m", message, ...(author ? ["--author", author] : [])); } test("the denylist: one literal a line, i: for any case, comments and blanks skipped, duplicates once", () => { const list = parseDenylist(`# a comment\n\n${PLANTED}\ni:MiXeD\n ${PLANTED} \r\ni:\n`); assert.equal(list.length, 2); assert.deepEqual(list[0], { bytes: Buffer.from(PLANTED), ci: false, from: "denylist line 3" }); assert.deepEqual(list[1], { bytes: Buffer.from("mixed"), ci: true, from: "denylist line 4" }); // Named by where it was written and its length — none of its characters. assert.equal(literalLabel(list, 0), `denylist line 3 (len ${PLANTED.length})`); assert.equal(literalLabel(list, 1), "denylist line 4 (len 5, any case)"); }); test("a byte-order mark and CRLF endings never become part of a literal", () => { const list = parseDenylist(`\uFEFF${PLANTED}\r\ni:Other\r\n`); assert.deepEqual(list.map((l) => l.bytes.toString()), [PLANTED, "other"]); assert.equal(list[1].from, "denylist line 2"); }); test("scanBuffer finds every occurrence; an i: literal matches any ASCII case", () => { const buf = Buffer.from(`a ${PLANTED} b PlantedHome c mixed MIXED`); const hits = scanBuffer(buf, [...lits(PLANTED), ...parseDenylist("i:plantedHOME\ni:mixed")]); assert.deepEqual( hits.map((h) => [h.lit, h.offset]), [[0, 2], [1, 2], [1, 16], [2, 30], [2, 36]], ); }); test("maskLiterals merges overlapping occurrences into one mark and leaves the rest", () => { const list = lits(PLANTED, "homeplanted"); assert.equal(maskLiterals(`/srv/${PLANTED}/x ${PLANTED}`, list), "/srv/[REDACTED]/x [REDACTED]"); assert.equal(maskLiterals(`a plantedhomeplanted b`, list), "a [REDACTED] b"); assert.equal(maskLiterals("nothing here", list), "nothing here"); }); test("objectField names a commit's header field or its message, never its bytes", () => { const commit = Buffer.from( "tree 1234\nparent 5678\nauthor A 1 +0000\ncommitter C 1 +0000\ngpgsig -----BEGIN\n sig line\n -----END\n\nthe message\n", ); const at = (s: string) => commit.indexOf(s); assert.equal(objectField(commit, at("a@x")), "author"); assert.equal(objectField(commit, at("c@x")), "committer"); assert.equal(objectField(commit, at("sig line")), "gpgsig"); assert.equal(objectField(commit, at("message")), "message"); assert.equal(objectField(commit, at("5678")), "parent"); }); test("a tree's entry names are what is scanned, not its binary ids", () => { const id = Buffer.alloc(20, 0x70); // 'p' x20: would match "ppp" if ids were read const tree = Buffer.concat([ Buffer.from("100644 a.txt\0"), id, Buffer.from(`40000 ${PLANTED}\0`), id, ]); assert.equal(treeEntryNames(tree, 20).toString("latin1"), `a.txt\0${PLANTED}\0`); assert.equal(scanBuffer(treeEntryNames(tree, 20), lits("ppp")).length, 0); }); test("the object walk finds a literal in a blob, a commit message, an author line and a tree entry name", async () => { const dir = repo(); commit(dir, { "README.md": "clean\n" }, "first"); commit(dir, { "notes.txt": `path /srv/${PLANTED}/data\n` }, "a blob carries it"); commit(dir, { "README.md": "clean 2\n" }, `the message says /srv/${PLANTED}`); commit(dir, { "README.md": "clean 3\n" }, "an identity", `Planted <${PLANTED}@example.invalid>`); commit(dir, { [`dir-${PLANTED}/x.txt`]: "x\n" }, "a name"); const result = await auditObjects(path.join(dir, ".git"), lits(PLANTED)); const kinds = result.hits.map((h) => `${h.kind}:${h.field ?? ""}`).sort(); // The message and the author line are one commit object each. // The tree's hit is its second entry: README.md, dir-…, notes.txt. assert.deepEqual(kinds, ["blob:", "commit:author", "commit:message", "tree:entry 2"]); assert.equal(result.commits, 5); const objects = execFileSync("git", ["count-objects", "-v"], { cwd: dir }).toString(); assert.equal(result.objects, Number(/^count: (\d+)/m.exec(objects)![1]), "every object was read"); // The report names the literal by where it was written, and prints no // byte of any object: not the literal, not what sits beside it. const report = formatAuditReport(result, lits(PLANTED), { scrubFile: path.join(os.homedir(), ".config", "archilyzer", "source-scrub.txt"), }).join("\n"); assert.ok(!report.includes(PLANTED), report); for (const beside of ["/srv/", "/data", "example.invalid", "Planted <", "the message says"]) { assert.ok(!report.includes(beside), `the report carries "${beside}" from an object`); } assert.match(report, /AUDIT REFUSED: 4 hits/); assert.match(report, /denylist line 1 \(len 11\): 1 in blob, 2 in commits, 1 in tree/); assert.match(report, /commit [0-9a-f]{12} \(author, byte \d+\): denylist line 1 \(len 11\)/); assert.match(report, /commit [0-9a-f]{12} \(message, byte \d+\): denylist line 1 \(len 11\)/); assert.match(report, /blob [0-9a-f]{12} \(byte 10\): denylist line 1 \(len 11\)/); assert.match(report, /tree [0-9a-f]{12} \(entry 2\): denylist line 1 \(len 11\)/); assert.match(report, /add a rule to ~\/\.config\/archilyzer\/source-scrub\.txt or drop the file from history, then re-run\.$/); }); test("a clean repo audits clean, with gitleaks skipped when it is not on PATH", async () => { const dir = repo(); commit(dir, { "a.txt": "nothing to see\n" }, "clean"); const logs: string[] = []; const result = await auditBare(path.join(dir, ".git"), lits(PLANTED), { scratch: TMP, onLog: (l) => logs.push(l), signal: new AbortController().signal, gitleaks: "gitleaks-not-installed-here", }); assert.equal(result.hits.length, 0); assert.equal(result.gitleaks, "skipped"); assert.match(logs.join("\n"), /WARNING: gitleaks-not-installed-here is not on PATH — the secret scan is skipped/); assert.match(formatAuditReport(result, lits(PLANTED), { scrubFile: "/x" })[0], /^\[source\] audit clean: \d+ objects \(1 commit\)/); const skipped = await runGitleaks(path.join(dir, ".git"), { bin: "gitleaks", scratch: TMP, literals: [], onLog: () => {}, signal: new AbortController().signal, env: { PATH: path.join(TMP, "empty-path") }, }); assert.equal(skipped.status, "skipped"); }); test("the staged-file sweep reads contents, names and gzip'd bytes decompressed, skips packs, refuses a symlink", async () => { const dir = path.join(TMP, "stage"); mkdirSync(path.join(dir, "tree", `d-${PLANTED}`), { recursive: true }); writeFileSync(path.join(dir, "tree", "ok.txt"), "fine\n"); writeFileSync(path.join(dir, "tree", "bad.txt"), `x ${PLANTED} y\n`); writeFileSync(path.join(dir, "tree", `d-${PLANTED}`, "f.txt"), "fine\n"); writeFileSync(path.join(dir, "t.tar.gz"), gzipSync(Buffer.from(`inside ${PLANTED}`))); writeFileSync(path.join(dir, "pack-1.pack"), PLANTED); // the object walk's job const result = await auditFiles(dir, lits(PLANTED)); assert.deepEqual(result.hits.map((h) => `${h.where} ${h.field}`).sort(), [ "t.tar.gz decompressed", "tree/bad.txt contents", "tree/d-[REDACTED] path", "tree/d-[REDACTED]/f.txt path", ]); assert.equal(result.files, 5); symlinkSync("/etc/hostname", path.join(dir, "tree", "link")); await assert.rejects(auditFiles(dir, lits(PLANTED)), (err) => err instanceof SourceRefusal && /symlink/.test(err.message)); });