feat(scripts): add emit-sub-processors compliance script + tests

Adds scripts/compliance/emit-sub-processors.mjs which walks docs/library-decisions/*.md, filters is-sub-processor: true entries via frontmatter parsing, merges compliance/sub-processors.manual.yml (graceful skip if absent), and emits sorted deterministic YAML to compliance/sub-processors.yml. - parseFrontmatter: extracts top-level scalars, skips nested blocks - parseLibraryTraceSubProcessors: discriminated-union filter on is-sub-processor flag - loadManualEntries / parseSimpleYamlList: flat YAML list parser for manual entries; injects source: manual - buildSubProcessors: merge + sort by package name - renderSubProcessorsYaml: package-first field order, rest alphabetical - --check and --print modes via shared unifiedDiff from emit-data-map - 39 unit tests across all exported functions Wires compliance:sub-processors root package script. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-05-18 20:01:06 +00:00
parent d8dbc99bab
commit 33bac95c41
3 changed files with 852 additions and 0 deletions
--- a/scripts/compliance/emit-sub-processors.test.mjs
+++ b/scripts/compliance/emit-sub-processors.test.mjs
@@ -0,0 +1,512 @@
+import { test, describe } from "node:test";
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  parseFrontmatter,
+  findLibraryDecisionFiles,
+  parseLibraryTraceSubProcessors,
+  parseSimpleYamlList,
+  loadManualEntries,
+  buildSubProcessors,
+  renderSubProcessorsYaml,
+  unifiedDiff,
+  OUTPUT_PATH,
+  MANUAL_PATH,
+} from "./emit-sub-processors.mjs";
+
+// ---- Fixtures ----
+
+const SUB_PROCESSOR_MD = `---
+package: stripe
+version: "^14.0.0"
+tier: feature
+decision: approved
+date: 2026-05-18
+deciders: [Danijel Martinek]
+adr: null
+lastRevalidated: null
+is-sub-processor: true
+processes-pii: true
+data-sent: payment card details and billing address
+region: eu-west-1
+dpa-signed: true
+sccs-required: false
+contact: https://stripe.com/privacy
+filter-results:
+  license: MIT
+  types: native
+  maintenance: active
+  boundary-fit: pass
+  shadow-check: pass
+  eu-residency: ok
+  cve-scan: clean
+  named-consumer: pass
+  socketRisk: clean
+verification-commands:
+  - npm view stripe license
+accepted-cves: []
+---
+
+## Filter: license
+
+MIT.
+`;
+
+const NON_SUB_PROCESSOR_MD = `---
+package: zod
+version: "^3.0.0"
+tier: core
+decision: approved
+date: 2026-05-14
+deciders: [Danijel Martinek]
+adr: null
+lastRevalidated: null
+is-sub-processor: false
+processes-pii: false
+filter-results:
+  license: MIT
+  types: native
+  maintenance: active
+  boundary-fit: pass
+  shadow-check: pass
+  eu-residency: n/a
+  cve-scan: clean
+  named-consumer: pass
+  socketRisk: clean
+verification-commands:
+  - npm view zod license
+accepted-cves: []
+---
+`;
+
+const ANOTHER_SUB_PROCESSOR_MD = `---
+package: sendgrid
+version: "^7.0.0"
+tier: feature
+decision: approved
+date: 2026-05-18
+deciders: [Danijel Martinek]
+adr: null
+lastRevalidated: null
+is-sub-processor: true
+processes-pii: true
+data-sent: email address and name for transactional emails
+region: eu
+dpa-signed: false
+sccs-required: true
+contact: https://sendgrid.com/privacy
+filter-results:
+  license: MIT
+  types: native
+  maintenance: active
+  boundary-fit: pass
+  shadow-check: pass
+  eu-residency: ok
+  cve-scan: clean
+  named-consumer: pass
+  socketRisk: clean
+verification-commands:
+  - npm view @sendgrid/mail license
+accepted-cves: []
+---
+`;
+
+const MANUAL_YAML = `- package: aws-s3
+  data-sent: file uploads and user avatars
+  region: eu-west-1
+  dpa-signed: true
+  sccs-required: false
+  contact: https://aws.amazon.com/compliance/eu-data-privacy/
+  decision: approved
+  version: "^3.0.0"
+`;
+
+// ---- Test helpers ----
+
+function makeLibraryDecisions(files) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-sub-processors-"));
+  const dir = path.join(root, "docs", "library-decisions");
+  fs.mkdirSync(dir, { recursive: true });
+  for (const [name, src] of Object.entries(files)) {
+    fs.writeFileSync(path.join(dir, name), src, "utf8");
+  }
+  return root;
+}
+
+function makeRepoWithManual(libraryFiles, manualYaml) {
+  const root = makeLibraryDecisions(libraryFiles);
+  if (manualYaml !== undefined) {
+    const complianceDir = path.join(root, "compliance");
+    fs.mkdirSync(complianceDir, { recursive: true });
+    fs.writeFileSync(path.join(root, MANUAL_PATH), manualYaml, "utf8");
+  }
+  return root;
+}
+
+// ---- Tests ----
+
+describe("parseFrontmatter", () => {
+  test("parses top-level scalar fields", () => {
+    const meta = parseFrontmatter(SUB_PROCESSOR_MD);
+    assert.equal(meta.package, "stripe");
+    assert.equal(meta["is-sub-processor"], true);
+    assert.equal(meta["dpa-signed"], true);
+    assert.equal(meta["sccs-required"], false);
+    assert.equal(meta.decision, "approved");
+    assert.equal(meta.contact, "https://stripe.com/privacy");
+  });
+
+  test("parses false boolean correctly (is-sub-processor: false)", () => {
+    const meta = parseFrontmatter(NON_SUB_PROCESSOR_MD);
+    assert.equal(meta["is-sub-processor"], false);
+    assert.equal(meta["processes-pii"], false);
+  });
+
+  test("strips quotes from version strings", () => {
+    const meta = parseFrontmatter(SUB_PROCESSOR_MD);
+    assert.equal(meta.version, "^14.0.0");
+  });
+
+  test("skips nested block fields (filter-results)", () => {
+    const meta = parseFrontmatter(SUB_PROCESSOR_MD);
+    assert.ok(
+      !("license" in meta),
+      "should not include nested filter-results.license",
+    );
+    assert.ok(
+      !("filter-results" in meta),
+      "filter-results block header should be skipped",
+    );
+  });
+
+  test("returns null when no frontmatter is present", () => {
+    assert.equal(parseFrontmatter("no frontmatter here"), null);
+  });
+
+  test("returns null for empty string", () => {
+    assert.equal(parseFrontmatter(""), null);
+  });
+});
+
+describe("findLibraryDecisionFiles", () => {
+  test("finds all .md files and excludes _template.md", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-stripe.md": SUB_PROCESSOR_MD,
+      "2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
+      "_template.md": "# template",
+    });
+    const files = findLibraryDecisionFiles(root);
+    assert.equal(files.length, 2);
+    assert.ok(files.every((f) => !path.basename(f).startsWith("_")));
+  });
+
+  test("returns files in sorted order", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-18-stripe.md": SUB_PROCESSOR_MD,
+      "2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
+    });
+    const files = findLibraryDecisionFiles(root);
+    assert.ok(
+      path.basename(files[0]) < path.basename(files[1]),
+      "files should be sorted alphabetically",
+    );
+  });
+
+  test("returns empty array when docs/library-decisions does not exist", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "empty-repo-"));
+    assert.deepEqual(findLibraryDecisionFiles(root), []);
+  });
+});
+
+describe("parseLibraryTraceSubProcessors — discriminated union", () => {
+  test("returns only entries where is-sub-processor: true", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-stripe.md": SUB_PROCESSOR_MD,
+      "2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
+    });
+    const entries = parseLibraryTraceSubProcessors(root);
+    assert.equal(entries.length, 1);
+    assert.equal(entries[0].package, "stripe");
+  });
+
+  test("sets source to library-trace for parsed entries", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-stripe.md": SUB_PROCESSOR_MD,
+    });
+    const [entry] = parseLibraryTraceSubProcessors(root);
+    assert.equal(entry.source, "library-trace");
+  });
+
+  test("extracts all sub-processor fields from frontmatter", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-stripe.md": SUB_PROCESSOR_MD,
+    });
+    const [entry] = parseLibraryTraceSubProcessors(root);
+    assert.equal(entry.package, "stripe");
+    assert.equal(entry.version, "^14.0.0");
+    assert.equal(entry.decision, "approved");
+    assert.equal(
+      entry["data-sent"],
+      "payment card details and billing address",
+    );
+    assert.equal(entry.region, "eu-west-1");
+    assert.equal(entry["dpa-signed"], true);
+    assert.equal(entry["sccs-required"], false);
+    assert.equal(entry.contact, "https://stripe.com/privacy");
+  });
+
+  test("returns empty array when no sub-processors exist", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
+    });
+    assert.deepEqual(parseLibraryTraceSubProcessors(root), []);
+  });
+
+  test("returns empty array when library-decisions dir is absent", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "no-decisions-"));
+    assert.deepEqual(parseLibraryTraceSubProcessors(root), []);
+  });
+
+  test("handles multiple sub-processor files", () => {
+    const root = makeLibraryDecisions({
+      "2026-05-14-stripe.md": SUB_PROCESSOR_MD,
+      "2026-05-14-sendgrid.md": ANOTHER_SUB_PROCESSOR_MD,
+      "2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
+    });
+    const entries = parseLibraryTraceSubProcessors(root);
+    assert.equal(entries.length, 2);
+    const packages = entries.map((e) => e.package).sort();
+    assert.deepEqual(packages, ["sendgrid", "stripe"]);
+  });
+});
+
+describe("parseSimpleYamlList", () => {
+  test("parses a simple YAML list into an array of objects", () => {
+    const entries = parseSimpleYamlList(MANUAL_YAML);
+    assert.equal(entries.length, 1);
+    assert.equal(entries[0].package, "aws-s3");
+    assert.equal(entries[0]["dpa-signed"], true);
+    assert.equal(entries[0]["sccs-required"], false);
+    assert.equal(entries[0].region, "eu-west-1");
+  });
+
+  test("strips quotes from quoted values", () => {
+    const entries = parseSimpleYamlList(MANUAL_YAML);
+    assert.equal(entries[0].version, "^3.0.0");
+  });
+
+  test("parses multiple list items", () => {
+    const src = `- package: alpha
+  region: eu
+- package: beta
+  region: us
+`;
+    const entries = parseSimpleYamlList(src);
+    assert.equal(entries.length, 2);
+    assert.equal(entries[0].package, "alpha");
+    assert.equal(entries[1].package, "beta");
+  });
+
+  test("returns empty array for empty input", () => {
+    assert.deepEqual(parseSimpleYamlList(""), []);
+  });
+
+  test("returns empty array for comment-only input", () => {
+    assert.deepEqual(
+      parseSimpleYamlList("# just a comment\n# another line"),
+      [],
+    );
+  });
+});
+
+describe("loadManualEntries", () => {
+  test("returns empty array when manual file is absent (graceful skip)", () => {
+    const root = fs.mkdtempSync(path.join(os.tmpdir(), "no-manual-"));
+    assert.deepEqual(loadManualEntries(root), []);
+  });
+
+  test("loads manual entries and injects source: manual", () => {
+    const root = makeRepoWithManual({}, MANUAL_YAML);
+    const entries = loadManualEntries(root);
+    assert.equal(entries.length, 1);
+    assert.equal(entries[0].package, "aws-s3");
+    assert.equal(entries[0].source, "manual");
+  });
+
+  test("preserves all fields from manual file", () => {
+    const root = makeRepoWithManual({}, MANUAL_YAML);
+    const [entry] = loadManualEntries(root);
+    assert.equal(entry["data-sent"], "file uploads and user avatars");
+    assert.equal(entry["dpa-signed"], true);
+    assert.equal(entry["sccs-required"], false);
+    assert.equal(entry.decision, "approved");
+  });
+});
+
+describe("buildSubProcessors — merge and sort", () => {
+  test("merges traced and manual entries sorted by package name", () => {
+    const traced = [{ package: "stripe", source: "library-trace" }];
+    const manual = [{ package: "aws-s3", source: "manual" }];
+    const merged = buildSubProcessors(traced, manual);
+    assert.equal(merged.length, 2);
+    assert.equal(merged[0].package, "aws-s3");
+    assert.equal(merged[1].package, "stripe");
+  });
+
+  test("preserves source field for each entry type", () => {
+    const traced = [{ package: "stripe", source: "library-trace" }];
+    const manual = [{ package: "aws-s3", source: "manual" }];
+    const merged = buildSubProcessors(traced, manual);
+    assert.equal(merged[0].source, "manual");
+    assert.equal(merged[1].source, "library-trace");
+  });
+
+  test("returns empty array when both inputs are empty", () => {
+    assert.deepEqual(buildSubProcessors([], []), []);
+  });
+
+  test("returns traced-only when no manual entries", () => {
+    const traced = [{ package: "stripe", source: "library-trace" }];
+    const merged = buildSubProcessors(traced, []);
+    assert.equal(merged.length, 1);
+    assert.equal(merged[0].source, "library-trace");
+  });
+
+  test("returns manual-only when no traced entries", () => {
+    const manual = [{ package: "aws-s3", source: "manual" }];
+    const merged = buildSubProcessors([], manual);
+    assert.equal(merged.length, 1);
+    assert.equal(merged[0].source, "manual");
+  });
+
+  test("sorts multiple entries alphabetically by package", () => {
+    const traced = [
+      { package: "zod", source: "library-trace" },
+      { package: "stripe", source: "library-trace" },
+    ];
+    const merged = buildSubProcessors(traced, []);
+    assert.equal(merged[0].package, "stripe");
+    assert.equal(merged[1].package, "zod");
+  });
+});
+
+describe("renderSubProcessorsYaml", () => {
+  test("renders YAML header comment", () => {
+    const yaml = renderSubProcessorsYaml([]);
+    assert.ok(yaml.startsWith("# compliance/sub-processors.yml"));
+    assert.ok(yaml.includes("emit-sub-processors.mjs"));
+  });
+
+  test("renders empty list as sub-processors: []", () => {
+    const yaml = renderSubProcessorsYaml([]);
+    assert.ok(yaml.includes("sub-processors:"));
+    assert.ok(yaml.includes("  []"));
+  });
+
+  test("renders entries in package-name order", () => {
+    const entries = [
+      { package: "aws-s3", source: "manual", decision: "approved" },
+      { package: "stripe", source: "library-trace", decision: "approved" },
+    ];
+    const yaml = renderSubProcessorsYaml(entries);
+    const awsIdx = yaml.indexOf("package: aws-s3");
+    const stripeIdx = yaml.indexOf("package: stripe");
+    assert.ok(awsIdx < stripeIdx, "aws-s3 should appear before stripe");
+  });
+
+  test("renders all defined fields per entry", () => {
+    const entries = [
+      {
+        package: "stripe",
+        version: "^14.0.0",
+        decision: "approved",
+        "data-sent": "payment card details",
+        region: "eu-west-1",
+        "dpa-signed": true,
+        "sccs-required": false,
+        contact: "https://stripe.com/privacy",
+        source: "library-trace",
+      },
+    ];
+    const yaml = renderSubProcessorsYaml(entries);
+    assert.ok(yaml.includes("package: stripe"));
+    assert.ok(yaml.includes("version: ^14.0.0"));
+    assert.ok(yaml.includes("decision: approved"));
+    assert.ok(yaml.includes("data-sent: payment card details"));
+    assert.ok(yaml.includes("region: eu-west-1"));
+    assert.ok(yaml.includes("dpa-signed: true"));
+    assert.ok(yaml.includes("sccs-required: false"));
+    assert.ok(yaml.includes("contact:"));
+    assert.ok(yaml.includes("source: library-trace"));
+  });
+
+  test("omits undefined fields from entry", () => {
+    const entries = [{ package: "stripe", source: "library-trace" }];
+    const yaml = renderSubProcessorsYaml(entries);
+    assert.ok(
+      !yaml.includes("data-sent:"),
+      "should not render missing data-sent",
+    );
+    assert.ok(!yaml.includes("region:"), "should not render missing region");
+    assert.ok(
+      !yaml.includes("dpa-signed:"),
+      "should not render missing dpa-signed",
+    );
+  });
+
+  test("output is deterministic across multiple calls", () => {
+    const entries = [
+      { package: "stripe", source: "library-trace", decision: "approved" },
+    ];
+    assert.equal(
+      renderSubProcessorsYaml(entries),
+      renderSubProcessorsYaml(entries),
+    );
+  });
+
+  test("package field is the first field in each list item (starts with '- package:')", () => {
+    const entries = [{ package: "stripe", source: "library-trace" }];
+    const yaml = renderSubProcessorsYaml(entries);
+    assert.ok(
+      yaml.includes("  - package: stripe"),
+      "entry should start with '  - package:'",
+    );
+  });
+
+  test("OUTPUT_PATH is compliance/sub-processors.yml", () => {
+    assert.equal(OUTPUT_PATH, "compliance/sub-processors.yml");
+  });
+});
+
+describe("--check mode (integration)", () => {
+  test("passes when committed file matches generated output", () => {
+    const entries = [
+      { package: "stripe", source: "library-trace", decision: "approved" },
+    ];
+    const yaml = renderSubProcessorsYaml(entries);
+    const diff = unifiedDiff(yaml, yaml, OUTPUT_PATH);
+    assert.equal(diff, null, "no diff expected when files match");
+  });
+
+  test("fails with readable diff when committed file is stale", () => {
+    const entries = [
+      { package: "stripe", source: "library-trace", decision: "approved" },
+    ];
+    const yaml = renderSubProcessorsYaml(entries);
+    const stale = "# stale content\nsub-processors: []\n";
+    const diff = unifiedDiff(stale, yaml, OUTPUT_PATH);
+    assert.ok(diff !== null, "diff expected when file is stale");
+    assert.ok(
+      diff.includes(`--- ${OUTPUT_PATH}`),
+      "diff should include filename header",
+    );
+    assert.ok(
+      diff.includes("- # stale content"),
+      "diff should show removed line",
+    );
+    assert.ok(diff.includes("Line"), "diff should include line numbers");
+  });
+});