agentic-dev/scripts/library-decisions/revalidate.mjs

#!/usr/bin/env node
/**
 * Weekly trace revalidation: re-runs verification-commands for every
 * approved/pre-shipped trace, classifies soft/hard divergence, and manages
 * GitHub issues via the gh CLI.
 *
 * Soft drift  → rolling "library-policy/dashboard" issue (create or update)
 * Hard drift  → per-dep "library-policy/re-evaluation" issue (skip duplicates)
 * Refreshed   → close open re-evaluation issue when lastRevalidated set + clean
 * Rejected    → skipped entirely
 */

import { execSync, spawnSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { parseFrontmatter } from "./schema.mjs";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const DEFAULT_REPO_ROOT = path.resolve(__dirname, "..", "..");

const LABEL_DASHBOARD = "library-policy/dashboard";
const LABEL_RE_EVAL = "library-policy/re-evaluation";

// Patterns in command output that signal hard divergence (evaluation would change)
const HARD_PATTERNS = [
  /CVE-\d{4}-\d{4,}/i,
  /\babandoned\b/i,
  /\bhigh\s+severity\b/i,
  /\bcritical\s+severity\b/i,
];

// Patterns in command output that signal soft divergence (minor drift)
const SOFT_PATTERNS = [
  /\bdormant\b/i,
  /\bwarning\b/i,
  /\boutdated\b/i,
  /\bdeprecated\b/i,
];

// ---- Trace discovery ----

function findAllTraceFiles(traceDir) {
  const files = [];
  if (!fs.existsSync(traceDir)) return files;

  for (const entry of fs.readdirSync(traceDir)) {
    if (entry.startsWith("_")) continue;
    const fullPath = path.join(traceDir, entry);
    const stat = fs.statSync(fullPath);

    if (stat.isFile() && entry.endsWith(".md")) {
      files.push(fullPath);
    } else if (stat.isDirectory()) {
      // Scoped packages: date-@scope/ directory containing name.md files
      for (const sub of fs.readdirSync(fullPath)) {
        if (sub.endsWith(".md") && !sub.startsWith("_")) {
          files.push(path.join(fullPath, sub));
        }
      }
    }
  }

  return files;
}

// ---- Command execution ----

function defaultCommandRunner(cmd, cwd) {
  try {
    const stdout = execSync(cmd, {
      cwd,
      encoding: "utf8",
      timeout: 60_000,
      stdio: ["ignore", "pipe", "pipe"],
    });
    return { exitCode: 0, output: stdout };
  } catch (e) {
    const out = (e.stdout ?? "") + (e.stderr ?? "");
    return { exitCode: e.status ?? 1, output: out };
  }
}

// ---- Classification ----

function classifyOutput(exitCode, output) {
  if (exitCode !== 0) {
    const snippet =
      output.trim().slice(0, 300) || `command failed (exit ${exitCode})`;
    return { kind: "hard", finding: snippet };
  }
  for (const re of HARD_PATTERNS) {
    const m = output.match(re);
    if (m) return { kind: "hard", finding: m[0] };
  }
  for (const re of SOFT_PATTERNS) {
    const m = output.match(re);
    if (m) return { kind: "soft", finding: m[0] };
  }
  return { kind: "none" };
}

function revalidateTrace(fm, commandRunner, repoRoot) {
  const raw = fm["verification-commands"];
  const cmds = Array.isArray(raw) ? raw : [];
  let softFinding = null;

  for (const cmd of cmds) {
    const { exitCode, output } = commandRunner(cmd, repoRoot);
    const result = classifyOutput(exitCode, output);
    if (result.kind === "hard") {
      return { status: "hard", finding: result.finding };
    }
    if (result.kind === "soft" && softFinding === null) {
      softFinding = result.finding;
    }
  }

  return softFinding !== null
    ? { status: "soft", finding: softFinding }
    : { status: "ok", finding: null };
}

// ---- GitHub issue helpers ----

function defaultGhRunner(args) {
  const result = spawnSync("gh", args, { encoding: "utf8" });
  return { exitCode: result.status ?? 0, output: result.stdout ?? "" };
}

function listOpenIssues(label, ghRunner) {
  const { output } = ghRunner([
    "issue",
    "list",
    "--label",
    label,
    "--state",
    "open",
    "--json",
    "number,title,body",
  ]);
  try {
    return JSON.parse(output || "[]");
  } catch {
    return [];
  }
}

function createIssue(title, label, body, ghRunner) {
  ghRunner([
    "issue",
    "create",
    "--title",
    title,
    "--label",
    label,
    "--body",
    body,
  ]);
}

function updateIssue(number, body, ghRunner) {
  ghRunner(["issue", "edit", String(number), "--body", body]);
}

function closeIssue(number, comment, ghRunner) {
  ghRunner(["issue", "close", String(number), "--comment", comment]);
}

// ---- Dashboard body ----

function buildDashboardBody(softResults, today) {
  return [
    `## Library trace soft drift — ${today}`,
    "",
    "The following traces have minor drift in their verification commands.",
    "These discrepancies do not immediately require re-evaluation but should be reviewed.",
    "",
    "| Package | Finding |",
    "| ------- | ------- |",
    ...softResults.map((r) => `| \`${r.pkg}@${r.version}\` | ${r.finding} |`),
    "",
    "To refresh a trace, run the `/evaluate-library` skill and update `lastRevalidated`.",
  ].join("\n");
}

// ---- Main export ----

/**
 * Walk all approved/pre-shipped traces, re-run their verification-commands,
 * classify divergence, and manage GitHub issues accordingly.
 *
 * Returns { hard: [...], soft: [...] } for inspection / testing.
 */
export function revalidate(repoRoot = DEFAULT_REPO_ROOT, options = {}) {
  const {
    commandRunner = defaultCommandRunner,
    ghRunner = defaultGhRunner,
    today = new Date().toISOString().slice(0, 10),
  } = options;

  const traceDir = path.join(repoRoot, "docs", "library-decisions");
  const traceFiles = findAllTraceFiles(traceDir);

  const hardResults = [];
  const softResults = [];
  const cleanResults = []; // status: ok

  for (const tracePath of traceFiles) {
    let fm;
    try {
      const content = fs.readFileSync(tracePath, "utf8");
      fm = parseFrontmatter(content);
    } catch {
      continue;
    }

    if (fm.decision !== "approved" && fm.decision !== "pre-shipped") continue;

    const { status, finding } = revalidateTrace(fm, commandRunner, repoRoot);

    const entry = {
      tracePath,
      pkg: fm.package,
      version: fm.version,
      lastRevalidated: fm.lastRevalidated ?? null,
      finding,
    };

    if (status === "hard") hardResults.push(entry);
    else if (status === "soft") softResults.push(entry);
    else cleanResults.push(entry);
  }

  // Phase 1: close stale re-evaluation issues for deps that have since been
  // re-evaluated (lastRevalidated set) and currently show no hard drift.
  const openRevalIssues = listOpenIssues(LABEL_RE_EVAL, ghRunner);
  const closedNums = new Set();

  for (const issue of openRevalIssues) {
    const m = issue.title.match(/^re-evaluate:\s+(.+?)@/);
    if (!m) continue;
    const issuePkg = m[1].trim();

    const cleanEntry = cleanResults.find(
      (e) => e.pkg === issuePkg && e.lastRevalidated != null,
    );
    if (cleanEntry) {
      closeIssue(
        issue.number,
        `Closing: \`${issuePkg}\` trace was revalidated on ${cleanEntry.lastRevalidated}. ` +
          `No hard drift detected in latest run. Run \`/evaluate-library\` for a full re-walk if needed.`,
        ghRunner,
      );
      closedNums.add(issue.number);
    }
  }

  // Phase 2: open per-dep issues for hard drift, skipping duplicates.
  for (const result of hardResults) {
    const alreadyOpen = openRevalIssues.some(
      (i) =>
        !closedNums.has(i.number) &&
        i.title.includes(`re-evaluate: ${result.pkg}@`),
    );
    if (alreadyOpen) continue;

    const titleFinding = result.finding.split("\n")[0].slice(0, 80).trim();
    const title = `re-evaluate: ${result.pkg}@${result.version} — ${titleFinding}`;
    const body = [
      `## Revalidation finding`,
      "",
      `**Package:** \`${result.pkg}@${result.version}\``,
      `**Trace:** \`${path.relative(repoRoot, result.tracePath)}\``,
      `**Finding:** ${result.finding}`,
      "",
      "## Next steps",
      "",
      "Run the `/evaluate-library` skill to re-walk the evaluation for this package:",
      "```",
      ".claude/skills/evaluate-library/SKILL.md",
      "```",
      "",
      `> Generated by the weekly trace revalidation workflow on ${today}.`,
    ].join("\n");

    createIssue(title, LABEL_RE_EVAL, body, ghRunner);
  }

  // Phase 3: update rolling dashboard issue for soft drift.
  if (softResults.length > 0) {
    const dashboardBody = buildDashboardBody(softResults, today);
    const openDashboard = listOpenIssues(LABEL_DASHBOARD, ghRunner);
    if (openDashboard.length > 0) {
      updateIssue(openDashboard[0].number, dashboardBody, ghRunner);
    } else {
      createIssue(
        `Library trace drift dashboard — ${today}`,
        LABEL_DASHBOARD,
        dashboardBody,
        ghRunner,
      );
    }
  }

  return { hard: hardResults, soft: softResults };
}

// ---- CLI entry point ----

if (process.argv[1] === fileURLToPath(import.meta.url)) {
  const result = revalidate();

  if (result.hard.length === 0 && result.soft.length === 0) {
    console.log("✓ All traces clean — no drift detected.");
    process.exit(0);
  }

  if (result.hard.length > 0) {
    console.log(
      `\n✗ Hard drift detected for ${result.hard.length} package(s):`,
    );
    for (const r of result.hard) {
      console.log(`  ${r.pkg}@${r.version}: ${r.finding}`);
    }
  }

  if (result.soft.length > 0) {
    console.log(
      `\n⚠ Soft drift detected for ${result.soft.length} package(s):`,
    );
    for (const r of result.soft) {
      console.log(`  ${r.pkg}@${r.version}: ${r.finding}`);
    }
  }
}