feat(scripts): add trace revalidation script and tests
Walks every approved/pre-shipped trace, re-runs its verification-commands, classifies soft/hard divergence, and manages GitHub issues via the gh CLI: - hard drift (non-zero exit or CVE/abandoned keywords) → per-dep library-policy/re-evaluation issue; duplicate-issue guard prevents spam - soft drift (dormant/warning/deprecated keywords at exit 0) → rolling library-policy/dashboard issue (create or update) - clean + lastRevalidated set → close any stale re-evaluation issue - rejected traces skipped entirely ghRunner and commandRunner are injectable for hermetic integration tests; 12 fixture-based tests cover all six story scenarios plus edge cases. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
336
scripts/library-decisions/revalidate.mjs
Normal file
336
scripts/library-decisions/revalidate.mjs
Normal file
@@ -0,0 +1,336 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Weekly trace revalidation: re-runs verification-commands for every
|
||||
* approved/pre-shipped trace, classifies soft/hard divergence, and manages
|
||||
* GitHub issues via the gh CLI.
|
||||
*
|
||||
* Soft drift → rolling "library-policy/dashboard" issue (create or update)
|
||||
* Hard drift → per-dep "library-policy/re-evaluation" issue (skip duplicates)
|
||||
* Refreshed → close open re-evaluation issue when lastRevalidated set + clean
|
||||
* Rejected → skipped entirely
|
||||
*/
|
||||
|
||||
import { execSync, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parseFrontmatter } from "./schema.mjs";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const DEFAULT_REPO_ROOT = path.resolve(__dirname, "..", "..");
|
||||
|
||||
const LABEL_DASHBOARD = "library-policy/dashboard";
|
||||
const LABEL_RE_EVAL = "library-policy/re-evaluation";
|
||||
|
||||
// Patterns in command output that signal hard divergence (evaluation would change)
|
||||
const HARD_PATTERNS = [
|
||||
/CVE-\d{4}-\d{4,}/i,
|
||||
/\babandoned\b/i,
|
||||
/\bhigh\s+severity\b/i,
|
||||
/\bcritical\s+severity\b/i,
|
||||
];
|
||||
|
||||
// Patterns in command output that signal soft divergence (minor drift)
|
||||
const SOFT_PATTERNS = [
|
||||
/\bdormant\b/i,
|
||||
/\bwarning\b/i,
|
||||
/\boutdated\b/i,
|
||||
/\bdeprecated\b/i,
|
||||
];
|
||||
|
||||
// ---- Trace discovery ----
|
||||
|
||||
function findAllTraceFiles(traceDir) {
|
||||
const files = [];
|
||||
if (!fs.existsSync(traceDir)) return files;
|
||||
|
||||
for (const entry of fs.readdirSync(traceDir)) {
|
||||
if (entry.startsWith("_")) continue;
|
||||
const fullPath = path.join(traceDir, entry);
|
||||
const stat = fs.statSync(fullPath);
|
||||
|
||||
if (stat.isFile() && entry.endsWith(".md")) {
|
||||
files.push(fullPath);
|
||||
} else if (stat.isDirectory()) {
|
||||
// Scoped packages: date-@scope/ directory containing name.md files
|
||||
for (const sub of fs.readdirSync(fullPath)) {
|
||||
if (sub.endsWith(".md") && !sub.startsWith("_")) {
|
||||
files.push(path.join(fullPath, sub));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
// ---- Command execution ----
|
||||
|
||||
function defaultCommandRunner(cmd, cwd) {
|
||||
try {
|
||||
const stdout = execSync(cmd, {
|
||||
cwd,
|
||||
encoding: "utf8",
|
||||
timeout: 60_000,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
return { exitCode: 0, output: stdout };
|
||||
} catch (e) {
|
||||
const out = (e.stdout ?? "") + (e.stderr ?? "");
|
||||
return { exitCode: e.status ?? 1, output: out };
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Classification ----
|
||||
|
||||
function classifyOutput(exitCode, output) {
|
||||
if (exitCode !== 0) {
|
||||
const snippet =
|
||||
output.trim().slice(0, 300) || `command failed (exit ${exitCode})`;
|
||||
return { kind: "hard", finding: snippet };
|
||||
}
|
||||
for (const re of HARD_PATTERNS) {
|
||||
const m = output.match(re);
|
||||
if (m) return { kind: "hard", finding: m[0] };
|
||||
}
|
||||
for (const re of SOFT_PATTERNS) {
|
||||
const m = output.match(re);
|
||||
if (m) return { kind: "soft", finding: m[0] };
|
||||
}
|
||||
return { kind: "none" };
|
||||
}
|
||||
|
||||
function revalidateTrace(fm, commandRunner, repoRoot) {
|
||||
const raw = fm["verification-commands"];
|
||||
const cmds = Array.isArray(raw) ? raw : [];
|
||||
let softFinding = null;
|
||||
|
||||
for (const cmd of cmds) {
|
||||
const { exitCode, output } = commandRunner(cmd, repoRoot);
|
||||
const result = classifyOutput(exitCode, output);
|
||||
if (result.kind === "hard") {
|
||||
return { status: "hard", finding: result.finding };
|
||||
}
|
||||
if (result.kind === "soft" && softFinding === null) {
|
||||
softFinding = result.finding;
|
||||
}
|
||||
}
|
||||
|
||||
return softFinding !== null
|
||||
? { status: "soft", finding: softFinding }
|
||||
: { status: "ok", finding: null };
|
||||
}
|
||||
|
||||
// ---- GitHub issue helpers ----
|
||||
|
||||
function defaultGhRunner(args) {
|
||||
const result = spawnSync("gh", args, { encoding: "utf8" });
|
||||
return { exitCode: result.status ?? 0, output: result.stdout ?? "" };
|
||||
}
|
||||
|
||||
function listOpenIssues(label, ghRunner) {
|
||||
const { output } = ghRunner([
|
||||
"issue",
|
||||
"list",
|
||||
"--label",
|
||||
label,
|
||||
"--state",
|
||||
"open",
|
||||
"--json",
|
||||
"number,title,body",
|
||||
]);
|
||||
try {
|
||||
return JSON.parse(output || "[]");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function createIssue(title, label, body, ghRunner) {
|
||||
ghRunner([
|
||||
"issue",
|
||||
"create",
|
||||
"--title",
|
||||
title,
|
||||
"--label",
|
||||
label,
|
||||
"--body",
|
||||
body,
|
||||
]);
|
||||
}
|
||||
|
||||
function updateIssue(number, body, ghRunner) {
|
||||
ghRunner(["issue", "edit", String(number), "--body", body]);
|
||||
}
|
||||
|
||||
function closeIssue(number, comment, ghRunner) {
|
||||
ghRunner(["issue", "close", String(number), "--comment", comment]);
|
||||
}
|
||||
|
||||
// ---- Dashboard body ----
|
||||
|
||||
function buildDashboardBody(softResults, today) {
|
||||
return [
|
||||
`## Library trace soft drift — ${today}`,
|
||||
"",
|
||||
"The following traces have minor drift in their verification commands.",
|
||||
"These discrepancies do not immediately require re-evaluation but should be reviewed.",
|
||||
"",
|
||||
"| Package | Finding |",
|
||||
"| ------- | ------- |",
|
||||
...softResults.map((r) => `| \`${r.pkg}@${r.version}\` | ${r.finding} |`),
|
||||
"",
|
||||
"To refresh a trace, run the `/evaluate-library` skill and update `lastRevalidated`.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
// ---- Main export ----
|
||||
|
||||
/**
|
||||
* Walk all approved/pre-shipped traces, re-run their verification-commands,
|
||||
* classify divergence, and manage GitHub issues accordingly.
|
||||
*
|
||||
* Returns { hard: [...], soft: [...] } for inspection / testing.
|
||||
*/
|
||||
export function revalidate(repoRoot = DEFAULT_REPO_ROOT, options = {}) {
|
||||
const {
|
||||
commandRunner = defaultCommandRunner,
|
||||
ghRunner = defaultGhRunner,
|
||||
today = new Date().toISOString().slice(0, 10),
|
||||
} = options;
|
||||
|
||||
const traceDir = path.join(repoRoot, "docs", "library-decisions");
|
||||
const traceFiles = findAllTraceFiles(traceDir);
|
||||
|
||||
const hardResults = [];
|
||||
const softResults = [];
|
||||
const cleanResults = []; // status: ok
|
||||
|
||||
for (const tracePath of traceFiles) {
|
||||
let fm;
|
||||
try {
|
||||
const content = fs.readFileSync(tracePath, "utf8");
|
||||
fm = parseFrontmatter(content);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (fm.decision !== "approved" && fm.decision !== "pre-shipped") continue;
|
||||
|
||||
const { status, finding } = revalidateTrace(fm, commandRunner, repoRoot);
|
||||
|
||||
const entry = {
|
||||
tracePath,
|
||||
pkg: fm.package,
|
||||
version: fm.version,
|
||||
lastRevalidated: fm.lastRevalidated ?? null,
|
||||
finding,
|
||||
};
|
||||
|
||||
if (status === "hard") hardResults.push(entry);
|
||||
else if (status === "soft") softResults.push(entry);
|
||||
else cleanResults.push(entry);
|
||||
}
|
||||
|
||||
// Phase 1: close stale re-evaluation issues for deps that have since been
|
||||
// re-evaluated (lastRevalidated set) and currently show no hard drift.
|
||||
const openRevalIssues = listOpenIssues(LABEL_RE_EVAL, ghRunner);
|
||||
const closedNums = new Set();
|
||||
|
||||
for (const issue of openRevalIssues) {
|
||||
const m = issue.title.match(/^re-evaluate:\s+(.+?)@/);
|
||||
if (!m) continue;
|
||||
const issuePkg = m[1].trim();
|
||||
|
||||
const cleanEntry = cleanResults.find(
|
||||
(e) => e.pkg === issuePkg && e.lastRevalidated != null,
|
||||
);
|
||||
if (cleanEntry) {
|
||||
closeIssue(
|
||||
issue.number,
|
||||
`Closing: \`${issuePkg}\` trace was revalidated on ${cleanEntry.lastRevalidated}. ` +
|
||||
`No hard drift detected in latest run. Run \`/evaluate-library\` for a full re-walk if needed.`,
|
||||
ghRunner,
|
||||
);
|
||||
closedNums.add(issue.number);
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: open per-dep issues for hard drift, skipping duplicates.
|
||||
for (const result of hardResults) {
|
||||
const alreadyOpen = openRevalIssues.some(
|
||||
(i) =>
|
||||
!closedNums.has(i.number) &&
|
||||
i.title.includes(`re-evaluate: ${result.pkg}@`),
|
||||
);
|
||||
if (alreadyOpen) continue;
|
||||
|
||||
const titleFinding = result.finding.split("\n")[0].slice(0, 80).trim();
|
||||
const title = `re-evaluate: ${result.pkg}@${result.version} — ${titleFinding}`;
|
||||
const body = [
|
||||
`## Revalidation finding`,
|
||||
"",
|
||||
`**Package:** \`${result.pkg}@${result.version}\``,
|
||||
`**Trace:** \`${path.relative(repoRoot, result.tracePath)}\``,
|
||||
`**Finding:** ${result.finding}`,
|
||||
"",
|
||||
"## Next steps",
|
||||
"",
|
||||
"Run the `/evaluate-library` skill to re-walk the evaluation for this package:",
|
||||
"```",
|
||||
".claude/skills/evaluate-library/SKILL.md",
|
||||
"```",
|
||||
"",
|
||||
`> Generated by the weekly trace revalidation workflow on ${today}.`,
|
||||
].join("\n");
|
||||
|
||||
createIssue(title, LABEL_RE_EVAL, body, ghRunner);
|
||||
}
|
||||
|
||||
// Phase 3: update rolling dashboard issue for soft drift.
|
||||
if (softResults.length > 0) {
|
||||
const dashboardBody = buildDashboardBody(softResults, today);
|
||||
const openDashboard = listOpenIssues(LABEL_DASHBOARD, ghRunner);
|
||||
if (openDashboard.length > 0) {
|
||||
updateIssue(openDashboard[0].number, dashboardBody, ghRunner);
|
||||
} else {
|
||||
createIssue(
|
||||
`Library trace drift dashboard — ${today}`,
|
||||
LABEL_DASHBOARD,
|
||||
dashboardBody,
|
||||
ghRunner,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { hard: hardResults, soft: softResults };
|
||||
}
|
||||
|
||||
// ---- CLI entry point ----
|
||||
|
||||
if (process.argv[1] === fileURLToPath(import.meta.url)) {
|
||||
const result = revalidate();
|
||||
|
||||
if (result.hard.length === 0 && result.soft.length === 0) {
|
||||
console.log("✓ All traces clean — no drift detected.");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (result.hard.length > 0) {
|
||||
console.log(
|
||||
`\n✗ Hard drift detected for ${result.hard.length} package(s):`,
|
||||
);
|
||||
for (const r of result.hard) {
|
||||
console.log(` ${r.pkg}@${r.version}: ${r.finding}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.soft.length > 0) {
|
||||
console.log(
|
||||
`\n⚠ Soft drift detected for ${result.soft.length} package(s):`,
|
||||
);
|
||||
for (const r of result.soft) {
|
||||
console.log(` ${r.pkg}@${r.version}: ${r.finding}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
513
scripts/library-decisions/revalidate.test.mjs
Normal file
513
scripts/library-decisions/revalidate.test.mjs
Normal file
@@ -0,0 +1,513 @@
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { revalidate } from "./revalidate.mjs";
|
||||
|
||||
// ---- Fixture helpers ----
|
||||
|
||||
function makeTmpDir() {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), "revalidate-"));
|
||||
}
|
||||
|
||||
function writeTrace(dir, pkg, opts = {}) {
|
||||
const {
|
||||
decision = "approved",
|
||||
lastRevalidated = null,
|
||||
commands = ["echo ok"],
|
||||
socketRisk = "clean",
|
||||
} = opts;
|
||||
|
||||
const lr = lastRevalidated == null ? "null" : lastRevalidated;
|
||||
const cmdLines = commands.map((c) => ` - ${c}`).join("\n");
|
||||
|
||||
const content = `---
|
||||
package: ${pkg}
|
||||
version: "^1.0.0"
|
||||
tier: feature
|
||||
decision: ${decision}
|
||||
date: 2026-05-14
|
||||
deciders: [alice]
|
||||
adr: null
|
||||
lastRevalidated: ${lr}
|
||||
filter-results:
|
||||
license: MIT
|
||||
types: native
|
||||
maintenance: active
|
||||
boundary-fit: pass
|
||||
shadow-check: pass
|
||||
eu-residency: ok
|
||||
cve-scan: clean
|
||||
named-consumer: pass
|
||||
socketRisk: ${socketRisk}
|
||||
verification-commands:
|
||||
${cmdLines}
|
||||
---
|
||||
|
||||
## Body
|
||||
`;
|
||||
|
||||
const traceDir = path.join(dir, "docs", "library-decisions");
|
||||
fs.mkdirSync(traceDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(traceDir, `2026-05-14-${pkg}.md`), content);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a mock command runner that maps exact command strings to results.
|
||||
* Unrecognised commands return { exitCode: 0, output: "" } by default.
|
||||
*/
|
||||
function makeCommandMock(responses = {}) {
|
||||
const calls = [];
|
||||
function commandRunner(cmd) {
|
||||
calls.push(cmd);
|
||||
const r = responses[cmd];
|
||||
return r ?? { exitCode: 0, output: "" };
|
||||
}
|
||||
return { commandRunner, calls };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a mock gh CLI runner. Accepts an initial set of open issues keyed by
|
||||
* label. Tracks all calls; `gh issue create` appends to the label bucket.
|
||||
*/
|
||||
function makeGhMock(initialIssuesByLabel = {}) {
|
||||
const calls = [];
|
||||
const issuesByLabel = JSON.parse(JSON.stringify(initialIssuesByLabel));
|
||||
let nextNumber = 1000;
|
||||
|
||||
function ghRunner(args) {
|
||||
calls.push([...args]);
|
||||
|
||||
if (args[0] === "issue" && args[1] === "list") {
|
||||
const labelIdx = args.indexOf("--label");
|
||||
const label = labelIdx >= 0 ? args[labelIdx + 1] : "";
|
||||
return {
|
||||
exitCode: 0,
|
||||
output: JSON.stringify(issuesByLabel[label] ?? []),
|
||||
};
|
||||
}
|
||||
|
||||
if (args[0] === "issue" && args[1] === "create") {
|
||||
const titleIdx = args.indexOf("--title");
|
||||
const labelIdx = args.indexOf("--label");
|
||||
const title = titleIdx >= 0 ? args[titleIdx + 1] : "Untitled";
|
||||
const label = labelIdx >= 0 ? args[labelIdx + 1] : "";
|
||||
if (label) {
|
||||
issuesByLabel[label] = issuesByLabel[label] ?? [];
|
||||
issuesByLabel[label].push({ number: ++nextNumber, title, body: "" });
|
||||
}
|
||||
return { exitCode: 0, output: "" };
|
||||
}
|
||||
|
||||
if (args[0] === "issue" && args[1] === "edit") {
|
||||
return { exitCode: 0, output: "" };
|
||||
}
|
||||
|
||||
if (args[0] === "issue" && args[1] === "close") {
|
||||
const num = parseInt(args[2], 10);
|
||||
for (const label of Object.keys(issuesByLabel)) {
|
||||
issuesByLabel[label] = issuesByLabel[label].filter(
|
||||
(i) => i.number !== num,
|
||||
);
|
||||
}
|
||||
return { exitCode: 0, output: "" };
|
||||
}
|
||||
|
||||
return { exitCode: 0, output: "" };
|
||||
}
|
||||
|
||||
return { ghRunner, calls, issuesByLabel };
|
||||
}
|
||||
|
||||
function createCalls(calls) {
|
||||
return calls.filter((a) => a[0] === "issue" && a[1] === "create");
|
||||
}
|
||||
|
||||
function closeCalls(calls) {
|
||||
return calls.filter((a) => a[0] === "issue" && a[1] === "close");
|
||||
}
|
||||
|
||||
// ---- Tests ----
|
||||
|
||||
describe("revalidate", () => {
|
||||
test("no-drift trace → no issue created or closed", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "clean-lib", {
|
||||
commands: ["echo all-good"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"echo all-good": { exitCode: 0, output: "all-good" },
|
||||
});
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.deepEqual(result.hard, []);
|
||||
assert.deepEqual(result.soft, []);
|
||||
assert.equal(createCalls(calls).length, 0);
|
||||
assert.equal(closeCalls(calls).length, 0);
|
||||
});
|
||||
|
||||
test("soft-drift trace → dashboard issue created", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "drifting-lib", {
|
||||
commands: ["npm view drifting-lib version"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"npm view drifting-lib version": {
|
||||
exitCode: 0,
|
||||
output: "package is dormant — no recent releases",
|
||||
},
|
||||
});
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(result.soft.length, 1);
|
||||
assert.equal(result.soft[0].pkg, "drifting-lib");
|
||||
|
||||
const created = createCalls(calls);
|
||||
assert.equal(created.length, 1);
|
||||
|
||||
const titleIdx = created[0].indexOf("--title");
|
||||
const labelIdx = created[0].indexOf("--label");
|
||||
const title = created[0][titleIdx + 1];
|
||||
const label = created[0][labelIdx + 1];
|
||||
|
||||
assert.ok(
|
||||
title.startsWith("Library trace drift dashboard"),
|
||||
`title: ${title}`,
|
||||
);
|
||||
assert.equal(label, "library-policy/dashboard");
|
||||
});
|
||||
|
||||
test("soft-drift with existing dashboard issue → issue updated, not duplicated", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "drifting-lib", {
|
||||
commands: ["echo outdated package"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"echo outdated package": {
|
||||
exitCode: 0,
|
||||
output: "outdated package detected",
|
||||
},
|
||||
});
|
||||
const existingIssue = {
|
||||
number: 55,
|
||||
title: "Library trace drift dashboard — 2026-05-07",
|
||||
body: "",
|
||||
};
|
||||
const { ghRunner, calls } = makeGhMock({
|
||||
"library-policy/dashboard": [existingIssue],
|
||||
});
|
||||
|
||||
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
||||
|
||||
assert.equal(
|
||||
createCalls(calls).length,
|
||||
0,
|
||||
"should not create a new dashboard issue",
|
||||
);
|
||||
const editCalls = calls.filter((a) => a[0] === "issue" && a[1] === "edit");
|
||||
assert.equal(editCalls.length, 1);
|
||||
assert.equal(editCalls[0][2], "55");
|
||||
});
|
||||
|
||||
test("hard-drift trace → per-dep re-evaluation issue created with correct labels and title", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "risky-lib", {
|
||||
commands: ["pnpm audit --audit-level=moderate"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"pnpm audit --audit-level=moderate": {
|
||||
exitCode: 1,
|
||||
output: "high severity vulnerability in risky-lib",
|
||||
},
|
||||
});
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(result.hard.length, 1);
|
||||
assert.equal(result.hard[0].pkg, "risky-lib");
|
||||
|
||||
const created = createCalls(calls);
|
||||
assert.equal(created.length, 1);
|
||||
|
||||
const titleIdx = created[0].indexOf("--title");
|
||||
const labelIdx = created[0].indexOf("--label");
|
||||
const title = created[0][titleIdx + 1];
|
||||
const label = created[0][labelIdx + 1];
|
||||
|
||||
assert.ok(
|
||||
title.startsWith("re-evaluate: risky-lib@"),
|
||||
`title should start with "re-evaluate: risky-lib@", got: ${title}`,
|
||||
);
|
||||
assert.ok(
|
||||
title.includes(" — "),
|
||||
`title should contain em-dash separator, got: ${title}`,
|
||||
);
|
||||
assert.equal(label, "library-policy/re-evaluation");
|
||||
});
|
||||
|
||||
test("hard-drift with CVE in output → issue title includes CVE reference", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "cve-lib", {
|
||||
commands: ["socket scan cve-lib"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"socket scan cve-lib": {
|
||||
exitCode: 0,
|
||||
output: "CVE-2024-12345 found in transitive dependency",
|
||||
},
|
||||
});
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(result.hard.length, 1);
|
||||
|
||||
const created = createCalls(calls);
|
||||
assert.equal(created.length, 1);
|
||||
const titleIdx = created[0].indexOf("--title");
|
||||
assert.ok(
|
||||
created[0][titleIdx + 1].includes("CVE-2024-12345"),
|
||||
`title should reference the CVE`,
|
||||
);
|
||||
});
|
||||
|
||||
test("duplicate-issue guard → no second issue opened when open issue already exists", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "already-flagged", {
|
||||
commands: ["pnpm audit"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"pnpm audit": {
|
||||
exitCode: 1,
|
||||
output: "critical severity vulnerability",
|
||||
},
|
||||
});
|
||||
const existingIssue = {
|
||||
number: 77,
|
||||
title: "re-evaluate: already-flagged@^1.0.0 — previous finding",
|
||||
body: "",
|
||||
};
|
||||
const { ghRunner, calls } = makeGhMock({
|
||||
"library-policy/re-evaluation": [existingIssue],
|
||||
});
|
||||
|
||||
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
||||
|
||||
assert.equal(
|
||||
createCalls(calls).length,
|
||||
0,
|
||||
"should not open a duplicate re-evaluation issue",
|
||||
);
|
||||
});
|
||||
|
||||
test("stale-issue close on refreshed lastRevalidated → open issue closed with comment", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "refreshed-lib", {
|
||||
lastRevalidated: "2026-05-14",
|
||||
commands: ["npm view refreshed-lib license"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"npm view refreshed-lib license": { exitCode: 0, output: "MIT" },
|
||||
});
|
||||
const openIssue = {
|
||||
number: 42,
|
||||
title: "re-evaluate: refreshed-lib@^1.0.0 — old finding from last week",
|
||||
body: "",
|
||||
};
|
||||
const { ghRunner, calls } = makeGhMock({
|
||||
"library-policy/re-evaluation": [openIssue],
|
||||
});
|
||||
|
||||
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
||||
|
||||
const closed = closeCalls(calls);
|
||||
assert.equal(closed.length, 1, "should close the stale issue");
|
||||
assert.equal(closed[0][2], "42", "should close issue number 42");
|
||||
|
||||
const commentIdx = closed[0].indexOf("--comment");
|
||||
assert.ok(commentIdx >= 0, "close call should include --comment flag");
|
||||
assert.ok(
|
||||
closed[0][commentIdx + 1].includes("2026-05-14"),
|
||||
"comment should reference the revalidation date",
|
||||
);
|
||||
});
|
||||
|
||||
test("clean trace with null lastRevalidated does not close open issue", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "unrevalidated-lib", {
|
||||
lastRevalidated: null,
|
||||
commands: ["echo ok"],
|
||||
});
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"echo ok": { exitCode: 0, output: "ok" },
|
||||
});
|
||||
const openIssue = {
|
||||
number: 33,
|
||||
title: "re-evaluate: unrevalidated-lib@^1.0.0 — earlier finding",
|
||||
body: "",
|
||||
};
|
||||
const { ghRunner, calls } = makeGhMock({
|
||||
"library-policy/re-evaluation": [openIssue],
|
||||
});
|
||||
|
||||
revalidate(dir, { commandRunner, ghRunner, today: "2026-05-14" });
|
||||
|
||||
assert.equal(
|
||||
closeCalls(calls).length,
|
||||
0,
|
||||
"should not close issue when lastRevalidated is null",
|
||||
);
|
||||
});
|
||||
|
||||
test("rejected-trace skip → no commands run, no issues created", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "rejected-lib", {
|
||||
decision: "rejected",
|
||||
commands: ["pnpm audit"],
|
||||
});
|
||||
|
||||
let commandsCalled = 0;
|
||||
const commandRunner = () => {
|
||||
commandsCalled++;
|
||||
return { exitCode: 0, output: "" };
|
||||
};
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
commandsCalled,
|
||||
0,
|
||||
"should not run commands for rejected traces",
|
||||
);
|
||||
assert.equal(createCalls(calls).length, 0, "should not create any issues");
|
||||
assert.deepEqual(result.hard, []);
|
||||
assert.deepEqual(result.soft, []);
|
||||
});
|
||||
|
||||
test("pre-shipped trace is processed like approved", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "preshipped-lib", {
|
||||
decision: "pre-shipped",
|
||||
commands: ["echo clean"],
|
||||
});
|
||||
|
||||
const { commandRunner, calls: cmdCalls } = makeCommandMock({
|
||||
"echo clean": { exitCode: 0, output: "clean" },
|
||||
});
|
||||
const { ghRunner } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
cmdCalls.length,
|
||||
1,
|
||||
"should run commands for pre-shipped traces",
|
||||
);
|
||||
assert.deepEqual(result.hard, []);
|
||||
assert.deepEqual(result.soft, []);
|
||||
});
|
||||
|
||||
test("multiple traces: independent classification per trace", () => {
|
||||
const dir = makeTmpDir();
|
||||
writeTrace(dir, "clean-pkg", { commands: ["echo ok"] });
|
||||
writeTrace(dir, "soft-pkg", { commands: ["echo package is deprecated"] });
|
||||
writeTrace(dir, "hard-pkg", { commands: ["pnpm audit"] });
|
||||
|
||||
const { commandRunner } = makeCommandMock({
|
||||
"echo ok": { exitCode: 0, output: "ok" },
|
||||
"echo package is deprecated": {
|
||||
exitCode: 0,
|
||||
output: "package is deprecated",
|
||||
},
|
||||
"pnpm audit": { exitCode: 1, output: "vulnerability found" },
|
||||
});
|
||||
const { ghRunner, calls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(result.hard.length, 1);
|
||||
assert.equal(result.hard[0].pkg, "hard-pkg");
|
||||
assert.equal(result.soft.length, 1);
|
||||
assert.equal(result.soft[0].pkg, "soft-pkg");
|
||||
|
||||
// one re-eval issue for hard, one dashboard issue for soft
|
||||
const created = createCalls(calls);
|
||||
assert.equal(created.length, 2);
|
||||
|
||||
const labels = created.map((c) => {
|
||||
const idx = c.indexOf("--label");
|
||||
return c[idx + 1];
|
||||
});
|
||||
assert.ok(labels.includes("library-policy/re-evaluation"));
|
||||
assert.ok(labels.includes("library-policy/dashboard"));
|
||||
});
|
||||
|
||||
test("trace with empty verification-commands → treated as ok (no drift)", () => {
|
||||
const dir = makeTmpDir();
|
||||
// writeTrace with commands:[] produces an empty block sequence which
|
||||
// parseFrontmatter returns as {} (object, not array). revalidate.mjs
|
||||
// must handle this gracefully.
|
||||
writeTrace(dir, "no-cmds-lib", { commands: [] });
|
||||
|
||||
const { commandRunner, calls: cmdCalls } = makeCommandMock();
|
||||
const { ghRunner, calls: ghCalls } = makeGhMock();
|
||||
|
||||
const result = revalidate(dir, {
|
||||
commandRunner,
|
||||
ghRunner,
|
||||
today: "2026-05-14",
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
cmdCalls.length,
|
||||
0,
|
||||
"no commands should run for empty commands list",
|
||||
);
|
||||
assert.deepEqual(result.hard, []);
|
||||
assert.deepEqual(result.soft, []);
|
||||
assert.equal(createCalls(ghCalls).length, 0);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user