#!/usr/bin/env node /** * Weekly trace revalidation: re-runs verification-commands for every * approved/pre-shipped trace, classifies soft/hard divergence, and manages * GitHub issues via the gh CLI. * * Soft drift → rolling "library-policy/dashboard" issue (create or update) * Hard drift → per-dep "library-policy/re-evaluation" issue (skip duplicates) * Refreshed → close open re-evaluation issue when lastRevalidated set + clean * Rejected → skipped entirely */ import { execSync, spawnSync } from "node:child_process"; import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { parseFrontmatter } from "./schema.mjs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const DEFAULT_REPO_ROOT = path.resolve(__dirname, "..", ".."); const LABEL_DASHBOARD = "library-policy/dashboard"; const LABEL_RE_EVAL = "library-policy/re-evaluation"; // Patterns in command output that signal hard divergence (evaluation would change) const HARD_PATTERNS = [ /CVE-\d{4}-\d{4,}/i, /\babandoned\b/i, /\bhigh\s+severity\b/i, /\bcritical\s+severity\b/i, ]; // Patterns in command output that signal soft divergence (minor drift) const SOFT_PATTERNS = [ /\bdormant\b/i, /\bwarning\b/i, /\boutdated\b/i, /\bdeprecated\b/i, ]; // ---- Trace discovery ---- function findAllTraceFiles(traceDir) { const files = []; if (!fs.existsSync(traceDir)) return files; for (const entry of fs.readdirSync(traceDir)) { if (entry.startsWith("_")) continue; const fullPath = path.join(traceDir, entry); const stat = fs.statSync(fullPath); if (stat.isFile() && entry.endsWith(".md")) { files.push(fullPath); } else if (stat.isDirectory()) { // Scoped packages: date-@scope/ directory containing name.md files for (const sub of fs.readdirSync(fullPath)) { if (sub.endsWith(".md") && !sub.startsWith("_")) { files.push(path.join(fullPath, sub)); } } } } return files; } // ---- Command execution ---- function defaultCommandRunner(cmd, cwd) { try { const stdout = execSync(cmd, { cwd, encoding: "utf8", timeout: 60_000, stdio: ["ignore", "pipe", "pipe"], }); return { exitCode: 0, output: stdout }; } catch (e) { const out = (e.stdout ?? "") + (e.stderr ?? ""); return { exitCode: e.status ?? 1, output: out }; } } // ---- Classification ---- function classifyOutput(exitCode, output) { if (exitCode !== 0) { const snippet = output.trim().slice(0, 300) || `command failed (exit ${exitCode})`; return { kind: "hard", finding: snippet }; } for (const re of HARD_PATTERNS) { const m = output.match(re); if (m) return { kind: "hard", finding: m[0] }; } for (const re of SOFT_PATTERNS) { const m = output.match(re); if (m) return { kind: "soft", finding: m[0] }; } return { kind: "none" }; } function revalidateTrace(fm, commandRunner, repoRoot) { const raw = fm["verification-commands"]; const cmds = Array.isArray(raw) ? raw : []; let softFinding = null; for (const cmd of cmds) { const { exitCode, output } = commandRunner(cmd, repoRoot); const result = classifyOutput(exitCode, output); if (result.kind === "hard") { return { status: "hard", finding: result.finding }; } if (result.kind === "soft" && softFinding === null) { softFinding = result.finding; } } return softFinding !== null ? { status: "soft", finding: softFinding } : { status: "ok", finding: null }; } // ---- GitHub issue helpers ---- function defaultGhRunner(args) { const result = spawnSync("gh", args, { encoding: "utf8" }); return { exitCode: result.status ?? 0, output: result.stdout ?? "" }; } function listOpenIssues(label, ghRunner) { const { output } = ghRunner([ "issue", "list", "--label", label, "--state", "open", "--json", "number,title,body", ]); try { return JSON.parse(output || "[]"); } catch { return []; } } function createIssue(title, label, body, ghRunner) { ghRunner([ "issue", "create", "--title", title, "--label", label, "--body", body, ]); } function updateIssue(number, body, ghRunner) { ghRunner(["issue", "edit", String(number), "--body", body]); } function closeIssue(number, comment, ghRunner) { ghRunner(["issue", "close", String(number), "--comment", comment]); } // ---- Dashboard body ---- function buildDashboardBody(softResults, today) { return [ `## Library trace soft drift — ${today}`, "", "The following traces have minor drift in their verification commands.", "These discrepancies do not immediately require re-evaluation but should be reviewed.", "", "| Package | Finding |", "| ------- | ------- |", ...softResults.map((r) => `| \`${r.pkg}@${r.version}\` | ${r.finding} |`), "", "To refresh a trace, run the `/evaluate-library` skill and update `lastRevalidated`.", ].join("\n"); } // ---- Main export ---- /** * Walk all approved/pre-shipped traces, re-run their verification-commands, * classify divergence, and manage GitHub issues accordingly. * * Returns { hard: [...], soft: [...] } for inspection / testing. */ export function revalidate(repoRoot = DEFAULT_REPO_ROOT, options = {}) { const { commandRunner = defaultCommandRunner, ghRunner = defaultGhRunner, today = new Date().toISOString().slice(0, 10), } = options; const traceDir = path.join(repoRoot, "docs", "library-decisions"); const traceFiles = findAllTraceFiles(traceDir); const hardResults = []; const softResults = []; const cleanResults = []; // status: ok for (const tracePath of traceFiles) { let fm; try { const content = fs.readFileSync(tracePath, "utf8"); fm = parseFrontmatter(content); } catch { continue; } if (fm.decision !== "approved" && fm.decision !== "pre-shipped") continue; const { status, finding } = revalidateTrace(fm, commandRunner, repoRoot); const entry = { tracePath, pkg: fm.package, version: fm.version, lastRevalidated: fm.lastRevalidated ?? null, finding, }; if (status === "hard") hardResults.push(entry); else if (status === "soft") softResults.push(entry); else cleanResults.push(entry); } // Phase 1: close stale re-evaluation issues for deps that have since been // re-evaluated (lastRevalidated set) and currently show no hard drift. const openRevalIssues = listOpenIssues(LABEL_RE_EVAL, ghRunner); const closedNums = new Set(); for (const issue of openRevalIssues) { const m = issue.title.match(/^re-evaluate:\s+(.+?)@/); if (!m) continue; const issuePkg = m[1].trim(); const cleanEntry = cleanResults.find( (e) => e.pkg === issuePkg && e.lastRevalidated != null, ); if (cleanEntry) { closeIssue( issue.number, `Closing: \`${issuePkg}\` trace was revalidated on ${cleanEntry.lastRevalidated}. ` + `No hard drift detected in latest run. Run \`/evaluate-library\` for a full re-walk if needed.`, ghRunner, ); closedNums.add(issue.number); } } // Phase 2: open per-dep issues for hard drift, skipping duplicates. for (const result of hardResults) { const alreadyOpen = openRevalIssues.some( (i) => !closedNums.has(i.number) && i.title.includes(`re-evaluate: ${result.pkg}@`), ); if (alreadyOpen) continue; const titleFinding = result.finding.split("\n")[0].slice(0, 80).trim(); const title = `re-evaluate: ${result.pkg}@${result.version} — ${titleFinding}`; const body = [ `## Revalidation finding`, "", `**Package:** \`${result.pkg}@${result.version}\``, `**Trace:** \`${path.relative(repoRoot, result.tracePath)}\``, `**Finding:** ${result.finding}`, "", "## Next steps", "", "Run the `/evaluate-library` skill to re-walk the evaluation for this package:", "```", ".claude/skills/evaluate-library/SKILL.md", "```", "", `> Generated by the weekly trace revalidation workflow on ${today}.`, ].join("\n"); createIssue(title, LABEL_RE_EVAL, body, ghRunner); } // Phase 3: update rolling dashboard issue for soft drift. if (softResults.length > 0) { const dashboardBody = buildDashboardBody(softResults, today); const openDashboard = listOpenIssues(LABEL_DASHBOARD, ghRunner); if (openDashboard.length > 0) { updateIssue(openDashboard[0].number, dashboardBody, ghRunner); } else { createIssue( `Library trace drift dashboard — ${today}`, LABEL_DASHBOARD, dashboardBody, ghRunner, ); } } return { hard: hardResults, soft: softResults }; } // ---- CLI entry point ---- if (process.argv[1] === fileURLToPath(import.meta.url)) { const result = revalidate(); if (result.hard.length === 0 && result.soft.length === 0) { console.log("✓ All traces clean — no drift detected."); process.exit(0); } if (result.hard.length > 0) { console.log( `\n✗ Hard drift detected for ${result.hard.length} package(s):`, ); for (const r of result.hard) { console.log(` ${r.pkg}@${r.version}: ${r.finding}`); } } if (result.soft.length > 0) { console.log( `\n⚠ Soft drift detected for ${result.soft.length} package(s):`, ); for (const r of result.soft) { console.log(` ${r.pkg}@${r.version}: ${r.finding}`); } } }