Danijel Martinek
1b6e7189c7
Walks every approved/pre-shipped trace, re-runs its verification-commands, classifies soft/hard divergence, and manages GitHub issues via the gh CLI: - hard drift (non-zero exit or CVE/abandoned keywords) → per-dep library-policy/re-evaluation issue; duplicate-issue guard prevents spam - soft drift (dormant/warning/deprecated keywords at exit 0) → rolling library-policy/dashboard issue (create or update) - clean + lastRevalidated set → close any stale re-evaluation issue - rejected traces skipped entirely ghRunner and commandRunner are injectable for hermetic integration tests; 12 fixture-based tests cover all six story scenarios plus edge cases. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
514 lines
14 KiB
JavaScript
514 lines
14 KiB
JavaScript
import { test, describe } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import fs from "node:fs";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { revalidate } from "./revalidate.mjs";
|
|
|
|
// ---- Fixture helpers ----
|
|
|
|
function makeTmpDir() {
|
|
return fs.mkdtempSync(path.join(os.tmpdir(), "revalidate-"));
|
|
}
|
|
|
|
function writeTrace(dir, pkg, opts = {}) {
|
|
const {
|
|
decision = "approved",
|
|
lastRevalidated = null,
|
|
commands = ["echo ok"],
|
|
socketRisk = "clean",
|
|
} = opts;
|
|
|
|
const lr = lastRevalidated == null ? "null" : lastRevalidated;
|
|
const cmdLines = commands.map((c) => ` - ${c}`).join("\n");
|
|
|
|
const content = `---
|
|
package: ${pkg}
|
|
version: "^1.0.0"
|
|
tier: feature
|
|
decision: ${decision}
|
|
date: 2026-05-14
|
|
deciders: [alice]
|
|
adr: null
|
|
lastRevalidated: ${lr}
|
|
filter-results:
|
|
license: MIT
|
|
types: native
|
|
maintenance: active
|
|
boundary-fit: pass
|
|
shadow-check: pass
|
|
eu-residency: ok
|
|
cve-scan: clean
|
|
named-consumer: pass
|
|
socketRisk: ${socketRisk}
|
|
verification-commands:
|
|
${cmdLines}
|
|
---
|
|
|
|
## Body
|
|
`;
|
|
|
|
const traceDir = path.join(dir, "docs", "library-decisions");
|
|
fs.mkdirSync(traceDir, { recursive: true });
|
|
fs.writeFileSync(path.join(traceDir, `2026-05-14-${pkg}.md`), content);
|
|
}
|
|
|
|
/**
|
|
* Build a mock command runner that maps exact command strings to results.
|
|
* Unrecognised commands return { exitCode: 0, output: "" } by default.
|
|
*/
|
|
function makeCommandMock(responses = {}) {
|
|
const calls = [];
|
|
function commandRunner(cmd) {
|
|
calls.push(cmd);
|
|
const r = responses[cmd];
|
|
return r ?? { exitCode: 0, output: "" };
|
|
}
|
|
return { commandRunner, calls };
|
|
}
|
|
|
|
/**
|
|
* Build a mock gh CLI runner. Accepts an initial set of open issues keyed by
|
|
* label. Tracks all calls; `gh issue create` appends to the label bucket.
|
|
*/
|
|
function makeGhMock(initialIssuesByLabel = {}) {
|
|
const calls = [];
|
|
const issuesByLabel = JSON.parse(JSON.stringify(initialIssuesByLabel));
|
|
let nextNumber = 1000;
|
|
|
|
function ghRunner(args) {
|
|
calls.push([...args]);
|
|
|
|
if (args[0] === "issue" && args[1] === "list") {
|
|
const labelIdx = args.indexOf("--label");
|
|
const label = labelIdx >= 0 ? args[labelIdx + 1] : "";
|
|
return {
|
|
exitCode: 0,
|
|
output: JSON.stringify(issuesByLabel[label] ?? []),
|
|
};
|
|
}
|
|
|
|
if (args[0] === "issue" && args[1] === "create") {
|
|
const titleIdx = args.indexOf("--title");
|
|
const labelIdx = args.indexOf("--label");
|
|
const title = titleIdx >= 0 ? args[titleIdx + 1] : "Untitled";
|
|
const label = labelIdx >= 0 ? args[labelIdx + 1] : "";
|
|
if (label) {
|
|
issuesByLabel[label] = issuesByLabel[label] ?? [];
|
|
issuesByLabel[label].push({ number: ++nextNumber, title, body: "" });
|
|
}
|
|
return { exitCode: 0, output: "" };
|
|
}
|
|
|
|
if (args[0] === "issue" && args[1] === "edit") {
|
|
return { exitCode: 0, output: "" };
|
|
}
|
|
|
|
if (args[0] === "issue" && args[1] === "close") {
|
|
const num = parseInt(args[2], 10);
|
|
for (const label of Object.keys(issuesByLabel)) {
|
|
issuesByLabel[label] = issuesByLabel[label].filter(
|
|
(i) => i.number !== num,
|
|
);
|
|
}
|
|
return { exitCode: 0, output: "" };
|
|
}
|
|
|
|
return { exitCode: 0, output: "" };
|
|
}
|
|
|
|
return { ghRunner, calls, issuesByLabel };
|
|
}
|
|
|
|
function createCalls(calls) {
|
|
return calls.filter((a) => a[0] === "issue" && a[1] === "create");
|
|
}
|
|
|
|
function closeCalls(calls) {
|
|
return calls.filter((a) => a[0] === "issue" && a[1] === "close");
|
|
}
|
|
|
|
// ---- Tests ----
|
|
|
|
describe("revalidate", () => {
|
|
test("no-drift trace → no issue created or closed", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "clean-lib", {
|
|
commands: ["echo all-good"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"echo all-good": { exitCode: 0, output: "all-good" },
|
|
});
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.deepEqual(result.hard, []);
|
|
assert.deepEqual(result.soft, []);
|
|
assert.equal(createCalls(calls).length, 0);
|
|
assert.equal(closeCalls(calls).length, 0);
|
|
});
|
|
|
|
test("soft-drift trace → dashboard issue created", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "drifting-lib", {
|
|
commands: ["npm view drifting-lib version"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"npm view drifting-lib version": {
|
|
exitCode: 0,
|
|
output: "package is dormant — no recent releases",
|
|
},
|
|
});
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(result.soft.length, 1);
|
|
assert.equal(result.soft[0].pkg, "drifting-lib");
|
|
|
|
const created = createCalls(calls);
|
|
assert.equal(created.length, 1);
|
|
|
|
const titleIdx = created[0].indexOf("--title");
|
|
const labelIdx = created[0].indexOf("--label");
|
|
const title = created[0][titleIdx + 1];
|
|
const label = created[0][labelIdx + 1];
|
|
|
|
assert.ok(
|
|
title.startsWith("Library trace drift dashboard"),
|
|
`title: ${title}`,
|
|
);
|
|
assert.equal(label, "library-policy/dashboard");
|
|
});
|
|
|
|
test("soft-drift with existing dashboard issue → issue updated, not duplicated", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "drifting-lib", {
|
|
commands: ["echo outdated package"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"echo outdated package": {
|
|
exitCode: 0,
|
|
output: "outdated package detected",
|
|
},
|
|
});
|
|
const existingIssue = {
|
|
number: 55,
|
|
title: "Library trace drift dashboard — 2026-05-07",
|
|
body: "",
|
|
};
|
|
const { ghRunner, calls } = makeGhMock({
|
|
"library-policy/dashboard": [existingIssue],
|
|
});
|
|
|
|
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
|
|
|
assert.equal(
|
|
createCalls(calls).length,
|
|
0,
|
|
"should not create a new dashboard issue",
|
|
);
|
|
const editCalls = calls.filter((a) => a[0] === "issue" && a[1] === "edit");
|
|
assert.equal(editCalls.length, 1);
|
|
assert.equal(editCalls[0][2], "55");
|
|
});
|
|
|
|
test("hard-drift trace → per-dep re-evaluation issue created with correct labels and title", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "risky-lib", {
|
|
commands: ["pnpm audit --audit-level=moderate"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"pnpm audit --audit-level=moderate": {
|
|
exitCode: 1,
|
|
output: "high severity vulnerability in risky-lib",
|
|
},
|
|
});
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(result.hard.length, 1);
|
|
assert.equal(result.hard[0].pkg, "risky-lib");
|
|
|
|
const created = createCalls(calls);
|
|
assert.equal(created.length, 1);
|
|
|
|
const titleIdx = created[0].indexOf("--title");
|
|
const labelIdx = created[0].indexOf("--label");
|
|
const title = created[0][titleIdx + 1];
|
|
const label = created[0][labelIdx + 1];
|
|
|
|
assert.ok(
|
|
title.startsWith("re-evaluate: risky-lib@"),
|
|
`title should start with "re-evaluate: risky-lib@", got: ${title}`,
|
|
);
|
|
assert.ok(
|
|
title.includes(" — "),
|
|
`title should contain em-dash separator, got: ${title}`,
|
|
);
|
|
assert.equal(label, "library-policy/re-evaluation");
|
|
});
|
|
|
|
test("hard-drift with CVE in output → issue title includes CVE reference", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "cve-lib", {
|
|
commands: ["socket scan cve-lib"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"socket scan cve-lib": {
|
|
exitCode: 0,
|
|
output: "CVE-2024-12345 found in transitive dependency",
|
|
},
|
|
});
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(result.hard.length, 1);
|
|
|
|
const created = createCalls(calls);
|
|
assert.equal(created.length, 1);
|
|
const titleIdx = created[0].indexOf("--title");
|
|
assert.ok(
|
|
created[0][titleIdx + 1].includes("CVE-2024-12345"),
|
|
`title should reference the CVE`,
|
|
);
|
|
});
|
|
|
|
test("duplicate-issue guard → no second issue opened when open issue already exists", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "already-flagged", {
|
|
commands: ["pnpm audit"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"pnpm audit": {
|
|
exitCode: 1,
|
|
output: "critical severity vulnerability",
|
|
},
|
|
});
|
|
const existingIssue = {
|
|
number: 77,
|
|
title: "re-evaluate: already-flagged@^1.0.0 — previous finding",
|
|
body: "",
|
|
};
|
|
const { ghRunner, calls } = makeGhMock({
|
|
"library-policy/re-evaluation": [existingIssue],
|
|
});
|
|
|
|
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
|
|
|
assert.equal(
|
|
createCalls(calls).length,
|
|
0,
|
|
"should not open a duplicate re-evaluation issue",
|
|
);
|
|
});
|
|
|
|
test("stale-issue close on refreshed lastRevalidated → open issue closed with comment", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "refreshed-lib", {
|
|
lastRevalidated: "2026-05-14",
|
|
commands: ["npm view refreshed-lib license"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"npm view refreshed-lib license": { exitCode: 0, output: "MIT" },
|
|
});
|
|
const openIssue = {
|
|
number: 42,
|
|
title: "re-evaluate: refreshed-lib@^1.0.0 — old finding from last week",
|
|
body: "",
|
|
};
|
|
const { ghRunner, calls } = makeGhMock({
|
|
"library-policy/re-evaluation": [openIssue],
|
|
});
|
|
|
|
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
|
|
|
const closed = closeCalls(calls);
|
|
assert.equal(closed.length, 1, "should close the stale issue");
|
|
assert.equal(closed[0][2], "42", "should close issue number 42");
|
|
|
|
const commentIdx = closed[0].indexOf("--comment");
|
|
assert.ok(commentIdx >= 0, "close call should include --comment flag");
|
|
assert.ok(
|
|
closed[0][commentIdx + 1].includes("2026-05-14"),
|
|
"comment should reference the revalidation date",
|
|
);
|
|
});
|
|
|
|
test("clean trace with null lastRevalidated does not close open issue", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "unrevalidated-lib", {
|
|
lastRevalidated: null,
|
|
commands: ["echo ok"],
|
|
});
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"echo ok": { exitCode: 0, output: "ok" },
|
|
});
|
|
const openIssue = {
|
|
number: 33,
|
|
title: "re-evaluate: unrevalidated-lib@^1.0.0 — earlier finding",
|
|
body: "",
|
|
};
|
|
const { ghRunner, calls } = makeGhMock({
|
|
"library-policy/re-evaluation": [openIssue],
|
|
});
|
|
|
|
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
|
|
|
assert.equal(
|
|
closeCalls(calls).length,
|
|
0,
|
|
"should not close issue when lastRevalidated is null",
|
|
);
|
|
});
|
|
|
|
test("rejected-trace skip → no commands run, no issues created", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "rejected-lib", {
|
|
decision: "rejected",
|
|
commands: ["pnpm audit"],
|
|
});
|
|
|
|
let commandsCalled = 0;
|
|
const commandRunner = () => {
|
|
commandsCalled++;
|
|
return { exitCode: 0, output: "" };
|
|
};
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(
|
|
commandsCalled,
|
|
0,
|
|
"should not run commands for rejected traces",
|
|
);
|
|
assert.equal(createCalls(calls).length, 0, "should not create any issues");
|
|
assert.deepEqual(result.hard, []);
|
|
assert.deepEqual(result.soft, []);
|
|
});
|
|
|
|
test("pre-shipped trace is processed like approved", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "preshipped-lib", {
|
|
decision: "pre-shipped",
|
|
commands: ["echo clean"],
|
|
});
|
|
|
|
const { commandRunner, calls: cmdCalls } = makeCommandMock({
|
|
"echo clean": { exitCode: 0, output: "clean" },
|
|
});
|
|
const { ghRunner } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(
|
|
cmdCalls.length,
|
|
1,
|
|
"should run commands for pre-shipped traces",
|
|
);
|
|
assert.deepEqual(result.hard, []);
|
|
assert.deepEqual(result.soft, []);
|
|
});
|
|
|
|
test("multiple traces: independent classification per trace", () => {
|
|
const dir = makeTmpDir();
|
|
writeTrace(dir, "clean-pkg", { commands: ["echo ok"] });
|
|
writeTrace(dir, "soft-pkg", { commands: ["echo package is deprecated"] });
|
|
writeTrace(dir, "hard-pkg", { commands: ["pnpm audit"] });
|
|
|
|
const { commandRunner } = makeCommandMock({
|
|
"echo ok": { exitCode: 0, output: "ok" },
|
|
"echo package is deprecated": {
|
|
exitCode: 0,
|
|
output: "package is deprecated",
|
|
},
|
|
"pnpm audit": { exitCode: 1, output: "vulnerability found" },
|
|
});
|
|
const { ghRunner, calls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(result.hard.length, 1);
|
|
assert.equal(result.hard[0].pkg, "hard-pkg");
|
|
assert.equal(result.soft.length, 1);
|
|
assert.equal(result.soft[0].pkg, "soft-pkg");
|
|
|
|
// one re-eval issue for hard, one dashboard issue for soft
|
|
const created = createCalls(calls);
|
|
assert.equal(created.length, 2);
|
|
|
|
const labels = created.map((c) => {
|
|
const idx = c.indexOf("--label");
|
|
return c[idx + 1];
|
|
});
|
|
assert.ok(labels.includes("library-policy/re-evaluation"));
|
|
assert.ok(labels.includes("library-policy/dashboard"));
|
|
});
|
|
|
|
test("trace with empty verification-commands → treated as ok (no drift)", () => {
|
|
const dir = makeTmpDir();
|
|
// writeTrace with commands:[] produces an empty block sequence which
|
|
// parseFrontmatter returns as {} (object, not array). revalidate.mjs
|
|
// must handle this gracefully.
|
|
writeTrace(dir, "no-cmds-lib", { commands: [] });
|
|
|
|
const { commandRunner, calls: cmdCalls } = makeCommandMock();
|
|
const { ghRunner, calls: ghCalls } = makeGhMock();
|
|
|
|
const result = revalidate(dir, {
|
|
commandRunner,
|
|
ghRunner,
|
|
today: "2026-05-14",
|
|
});
|
|
|
|
assert.equal(
|
|
cmdCalls.length,
|
|
0,
|
|
"no commands should run for empty commands list",
|
|
);
|
|
assert.deepEqual(result.hard, []);
|
|
assert.deepEqual(result.soft, []);
|
|
assert.equal(createCalls(ghCalls).length, 0);
|
|
});
|
|
});
|