chore(template): clean-slate template snapshot from bb4a0c7
Curated, product-agnostic snapshot of the post-story-04 tree: demo content deleted, auth-only reference feature, web-next shell, all gates green. Product-specific docs, ADRs 027-029, PRDs/epics/archive, editor library traces, and product naming are curated out; generic template repairs (coverage provider devDeps, root test:coverage script, live lint fixes, root-only release-please) are kept. See TEMPLATE.md for provenance, curation list, and usage. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_016j8z4VHjedXDTjEDNg7qHK
This commit is contained in:
62
scripts/compliance/emit-all.mjs
Normal file
62
scripts/compliance/emit-all.mjs
Normal file
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* emit-all.mjs — Compliance artifact orchestrator.
|
||||
*
|
||||
* Default: regenerates all three compliance artifacts (write mode).
|
||||
* --check: diffs each artifact against the committed file; exits non-zero
|
||||
* if any generator reports a mismatch or validation failure.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compliance/emit-all.mjs # regenerate all artifacts
|
||||
* node scripts/compliance/emit-all.mjs --check # drift check (CI gate)
|
||||
* pnpm compliance:emit-all
|
||||
* pnpm compliance:emit-all --check
|
||||
*/
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import path from "node:path";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const SCRIPTS = [
|
||||
"emit-data-map.mjs",
|
||||
"emit-retention-policy.mjs",
|
||||
"emit-sub-processors.mjs",
|
||||
];
|
||||
|
||||
const checkMode = process.argv.includes("--check");
|
||||
const subArgs = checkMode ? ["--check"] : [];
|
||||
|
||||
let anyFailed = false;
|
||||
|
||||
for (const script of SCRIPTS) {
|
||||
const scriptPath = path.join(__dirname, script);
|
||||
const result = spawnSync(process.execPath, [scriptPath, ...subArgs], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
|
||||
if (result.status !== 0) {
|
||||
anyFailed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (anyFailed) {
|
||||
if (checkMode) {
|
||||
process.stderr.write(
|
||||
"\n✗ compliance:emit-all — one or more artifacts are out of date.\n" +
|
||||
" Run `pnpm compliance:emit-all` to regenerate.\n",
|
||||
);
|
||||
} else {
|
||||
process.stderr.write(
|
||||
"\n✗ compliance:emit-all — one or more artifacts failed to generate.\n",
|
||||
);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (checkMode) {
|
||||
console.log("✓ compliance:emit-all — all artifacts are up to date");
|
||||
} else {
|
||||
console.log("✓ compliance:emit-all — all artifacts regenerated");
|
||||
}
|
||||
412
scripts/compliance/emit-data-map.mjs
Normal file
412
scripts/compliance/emit-data-map.mjs
Normal file
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* emit-data-map.mjs — PII field inventory emitter.
|
||||
*
|
||||
* Walks packages/*\/src/integrations/cms/collections/*.ts, applies
|
||||
* PAYLOAD_AUTH_PII_DEFAULTS + custom.authPii overrides, and emits a
|
||||
* deterministic YAML PII inventory at compliance/data-map.yml.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compliance/emit-data-map.mjs # write compliance/data-map.yml
|
||||
* node scripts/compliance/emit-data-map.mjs --print # write to stdout
|
||||
* node scripts/compliance/emit-data-map.mjs --check # diff vs committed file; exit 1 on mismatch
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parse } from "@typescript-eslint/parser";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const REPO_ROOT = path.resolve(__dirname, "../..");
|
||||
export const OUTPUT_PATH = "compliance/data-map.yml";
|
||||
|
||||
// ---- Constants ----
|
||||
|
||||
/**
|
||||
* Default PII classification for Payload auth fields.
|
||||
* Null means the field is NOT PII and must be excluded from the data map.
|
||||
* Mirrors packages/core-shared/src/payload/pii-types.ts PAYLOAD_AUTH_PII_DEFAULTS.
|
||||
*/
|
||||
export const PAYLOAD_AUTH_PII_DEFAULTS = {
|
||||
email: {
|
||||
category: "contact-email",
|
||||
purpose: ["account-authentication", "transactional-notifications"],
|
||||
exportable: true,
|
||||
restrictable: true,
|
||||
},
|
||||
password: null,
|
||||
salt: null,
|
||||
hash: null,
|
||||
resetPasswordToken: null,
|
||||
resetPasswordExpiration: null,
|
||||
loginAttempts: null,
|
||||
lockUntil: null,
|
||||
apiKey: null,
|
||||
apiKeyIndex: null,
|
||||
};
|
||||
|
||||
// ---- AST parsing helpers ----
|
||||
|
||||
/**
|
||||
* Recursively extract a plain JS value from an AST node.
|
||||
* Returns undefined for unresolvable nodes (identifier references, etc.).
|
||||
*/
|
||||
function extractValue(node) {
|
||||
if (!node) return null;
|
||||
if (node.type === "Literal") return node.value;
|
||||
if (node.type === "TSAsExpression") return extractValue(node.expression);
|
||||
if (node.type === "Identifier") {
|
||||
if (node.name === "true") return true;
|
||||
if (node.name === "false") return false;
|
||||
if (node.name === "null") return null;
|
||||
return undefined;
|
||||
}
|
||||
if (node.type === "ObjectExpression") {
|
||||
const obj = {};
|
||||
for (const prop of node.properties) {
|
||||
if (prop.type !== "Property") continue;
|
||||
const key =
|
||||
prop.key.type === "Identifier"
|
||||
? prop.key.name
|
||||
: prop.key.type === "Literal"
|
||||
? String(prop.key.value)
|
||||
: null;
|
||||
if (!key) continue;
|
||||
const v = extractValue(prop.value);
|
||||
if (v !== undefined) obj[key] = v;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
if (node.type === "ArrayExpression") {
|
||||
return node.elements
|
||||
.map((e) => (e ? extractValue(e) : null))
|
||||
.filter((e) => e !== undefined);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Payload collection TypeScript file.
|
||||
* Returns the first exported variable whose value is an object with a string
|
||||
* `slug` field, or null on parse failure / missing file.
|
||||
*/
|
||||
export function parseCollectionFile(filePath) {
|
||||
let src;
|
||||
try {
|
||||
src = fs.readFileSync(filePath, "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
let ast;
|
||||
try {
|
||||
ast = parse(src, {
|
||||
sourceType: "module",
|
||||
ecmaVersion: "latest",
|
||||
loc: false,
|
||||
range: false,
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const node of ast.body) {
|
||||
if (node.type !== "ExportNamedDeclaration" || !node.declaration) continue;
|
||||
if (node.declaration.type !== "VariableDeclaration") continue;
|
||||
for (const decl of node.declaration.declarations) {
|
||||
if (!decl.init) continue;
|
||||
const value = extractValue(decl.init);
|
||||
if (
|
||||
value &&
|
||||
typeof value === "object" &&
|
||||
typeof value.slug === "string"
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- Collection file discovery ----
|
||||
|
||||
/**
|
||||
* Find all Payload collection TypeScript files across packages.
|
||||
*/
|
||||
export function findCollectionFiles(repoRoot = REPO_ROOT) {
|
||||
const packagesDir = path.join(repoRoot, "packages");
|
||||
if (!fs.existsSync(packagesDir)) return [];
|
||||
|
||||
const files = [];
|
||||
for (const pkg of fs.readdirSync(packagesDir).sort()) {
|
||||
const collectionsDir = path.join(
|
||||
packagesDir,
|
||||
pkg,
|
||||
"src",
|
||||
"integrations",
|
||||
"cms",
|
||||
"collections",
|
||||
);
|
||||
if (!fs.existsSync(collectionsDir)) continue;
|
||||
for (const file of fs.readdirSync(collectionsDir).sort()) {
|
||||
if (file.endsWith(".ts") && !file.endsWith(".test.ts")) {
|
||||
files.push(path.join(collectionsDir, file));
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// ---- Data map builder ----
|
||||
|
||||
/** Build a single PII entry. */
|
||||
function makePiiEntry(fieldName, pii, source) {
|
||||
return {
|
||||
field: fieldName,
|
||||
source,
|
||||
category: pii.category,
|
||||
purpose: Array.isArray(pii.purpose) ? [...pii.purpose] : [],
|
||||
exportable: pii.exportable ?? false,
|
||||
restrictable: pii.restrictable ?? false,
|
||||
...(pii.retention ? { retention: { ...pii.retention } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Collect field-level PII entries from a collection's fields array. */
|
||||
function collectFieldPiiEntries(fields) {
|
||||
const entries = [];
|
||||
if (!Array.isArray(fields)) return entries;
|
||||
for (const field of fields) {
|
||||
if (!field || typeof field.name !== "string") continue;
|
||||
const pii = field.custom && field.custom.pii;
|
||||
if (!pii || typeof pii !== "object") continue;
|
||||
entries.push(makePiiEntry(field.name, pii, "field-tag"));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** Collect auth PII entries by merging defaults with per-collection overrides. */
|
||||
function collectAuthPiiEntries(authPiiDefaults, authPiiOverrides) {
|
||||
const merged = { ...authPiiDefaults, ...authPiiOverrides };
|
||||
const entries = [];
|
||||
for (const [fieldName, pii] of Object.entries(merged)) {
|
||||
if (pii === null) continue;
|
||||
const source =
|
||||
fieldName in authPiiOverrides ? "auth-override" : "auth-default";
|
||||
entries.push(makePiiEntry(fieldName, pii, source));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk parsed collection configs and build the PII data map.
|
||||
* Applies authPiiDefaults (PAYLOAD_AUTH_PII_DEFAULTS) and custom.authPii
|
||||
* overrides for auth-enabled collections.
|
||||
*
|
||||
* Returns: Record<slug, { auth, piiFields, slug }>
|
||||
*/
|
||||
export function buildDataMap(
|
||||
rawCollections,
|
||||
authPiiDefaults = PAYLOAD_AUTH_PII_DEFAULTS,
|
||||
) {
|
||||
const map = {};
|
||||
for (const coll of rawCollections) {
|
||||
if (!coll || typeof coll.slug !== "string") continue;
|
||||
const isAuth = coll.auth === true;
|
||||
|
||||
const piiFields = [
|
||||
...collectFieldPiiEntries(coll.fields),
|
||||
...(isAuth
|
||||
? collectAuthPiiEntries(
|
||||
authPiiDefaults,
|
||||
(coll.custom && coll.custom.authPii) || {},
|
||||
)
|
||||
: []),
|
||||
];
|
||||
piiFields.sort((a, b) => a.field.localeCompare(b.field));
|
||||
|
||||
map[coll.slug] = { auth: isAuth, piiFields, slug: coll.slug };
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// ---- YAML serialization ----
|
||||
|
||||
const YAML_HEADER = [
|
||||
"# compliance/data-map.yml — PII field inventory",
|
||||
"# Generated by scripts/compliance/emit-data-map.mjs — do not edit manually.",
|
||||
"# Run `pnpm compliance:data-map` to regenerate.",
|
||||
].join("\n");
|
||||
|
||||
/** Quote a YAML string scalar only when necessary. */
|
||||
function yamlStr(s) {
|
||||
if (typeof s !== "string") return String(s);
|
||||
if (
|
||||
s === "" ||
|
||||
["true", "false", "null", "yes", "no", "on", "off"].includes(s) ||
|
||||
/[[{},:#&*!|>'"%@`\]]/u.test(s) ||
|
||||
/^\s|\s$/.test(s)
|
||||
) {
|
||||
return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
function renderPiiField(f) {
|
||||
let out = "";
|
||||
out += ` - category: ${yamlStr(f.category)}\n`;
|
||||
out += ` exportable: ${f.exportable}\n`;
|
||||
out += ` field: ${yamlStr(f.field)}\n`;
|
||||
|
||||
if (!f.purpose || f.purpose.length === 0) {
|
||||
out += ` purpose: []\n`;
|
||||
} else {
|
||||
out += ` purpose:\n`;
|
||||
for (const p of f.purpose) {
|
||||
out += ` - ${yamlStr(p)}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
out += ` restrictable: ${f.restrictable}\n`;
|
||||
|
||||
if (f.retention) {
|
||||
out += ` retention:\n`;
|
||||
out += ` action: ${yamlStr(f.retention.action)}\n`;
|
||||
out += ` duration: ${yamlStr(f.retention.duration)}\n`;
|
||||
out += ` trigger: ${yamlStr(f.retention.trigger)}\n`;
|
||||
}
|
||||
|
||||
out += ` source: ${yamlStr(f.source)}\n`;
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the data map as deterministic YAML.
|
||||
* Collections sorted by slug; piiFields sorted by field name (done in buildDataMap).
|
||||
*/
|
||||
export function renderDataMapYaml(dataMap) {
|
||||
let yaml = YAML_HEADER + "\n";
|
||||
yaml += "collections:\n";
|
||||
|
||||
for (const slug of Object.keys(dataMap).sort()) {
|
||||
const coll = dataMap[slug];
|
||||
yaml += ` ${yamlStr(slug)}:\n`;
|
||||
yaml += ` auth: ${coll.auth}\n`;
|
||||
|
||||
if (!coll.piiFields || coll.piiFields.length === 0) {
|
||||
yaml += ` piiFields: []\n`;
|
||||
} else {
|
||||
yaml += ` piiFields:\n`;
|
||||
for (const f of coll.piiFields) {
|
||||
yaml += renderPiiField(f);
|
||||
}
|
||||
}
|
||||
|
||||
yaml += ` slug: ${yamlStr(slug)}\n`;
|
||||
}
|
||||
|
||||
return yaml;
|
||||
}
|
||||
|
||||
// ---- Diff helper for --check mode ----
|
||||
|
||||
/**
|
||||
* Produce a readable line-diff between expected (committed) and actual (generated).
|
||||
* Returns null if equal.
|
||||
*/
|
||||
export function unifiedDiff(expected, actual, filename) {
|
||||
const expLines = expected.split("\n");
|
||||
const actLines = actual.split("\n");
|
||||
const maxLen = Math.max(expLines.length, actLines.length);
|
||||
|
||||
const hunks = [];
|
||||
for (let i = 0; i < maxLen; i++) {
|
||||
const expLine = expLines[i] ?? "";
|
||||
const actLine = actLines[i] ?? "";
|
||||
if (expLine !== actLine) {
|
||||
hunks.push(`Line ${i + 1}:`);
|
||||
hunks.push(` - ${expLine}`);
|
||||
hunks.push(` + ${actLine}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (hunks.length === 0) return null;
|
||||
return (
|
||||
`--- ${filename} (committed)\n` +
|
||||
`+++ ${filename} (generated)\n` +
|
||||
hunks.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { mode: "write" }; // 'write' | 'print' | 'check'
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--print") out.mode = "print";
|
||||
else if (argv[i] === "--check") out.mode = "check";
|
||||
else if (argv[i] === "--help" || argv[i] === "-h") {
|
||||
console.log(
|
||||
[
|
||||
"Usage: node scripts/compliance/emit-data-map.mjs [--print | --check]",
|
||||
" (default): write compliance/data-map.yml",
|
||||
" --print: write YAML to stdout",
|
||||
" --check: diff vs committed file; exit 1 on mismatch",
|
||||
].join("\n"),
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const repoRoot = process.cwd();
|
||||
|
||||
const files = findCollectionFiles(repoRoot);
|
||||
const rawCollections = files
|
||||
.map((f) => parseCollectionFile(f))
|
||||
.filter(Boolean);
|
||||
const dataMap = buildDataMap(rawCollections);
|
||||
const yaml = renderDataMapYaml(dataMap);
|
||||
|
||||
const outPath = path.resolve(repoRoot, OUTPUT_PATH);
|
||||
|
||||
if (args.mode === "print") {
|
||||
process.stdout.write(yaml);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.mode === "check") {
|
||||
if (!fs.existsSync(outPath)) {
|
||||
process.stderr.write(
|
||||
`[compliance:data-map] --check: no committed file at ${OUTPUT_PATH}\n` +
|
||||
`Run \`pnpm compliance:data-map\` to generate it first.\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const committed = fs.readFileSync(outPath, "utf8");
|
||||
const diff = unifiedDiff(committed, yaml, OUTPUT_PATH);
|
||||
if (diff === null) {
|
||||
console.log(`✓ compliance:data-map — ${OUTPUT_PATH} is up to date`);
|
||||
process.exit(0);
|
||||
}
|
||||
process.stderr.write(
|
||||
`✗ compliance:data-map — ${OUTPUT_PATH} is out of date:\n${diff}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Default: write to file
|
||||
const dir = path.dirname(outPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(outPath, yaml, "utf8");
|
||||
console.log(`✓ compliance:data-map — wrote ${OUTPUT_PATH}`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
354
scripts/compliance/emit-data-map.test.mjs
Normal file
354
scripts/compliance/emit-data-map.test.mjs
Normal file
@@ -0,0 +1,354 @@
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
findCollectionFiles,
|
||||
parseCollectionFile,
|
||||
buildDataMap,
|
||||
renderDataMapYaml,
|
||||
unifiedDiff,
|
||||
} from "./emit-data-map.mjs";
|
||||
|
||||
// ---- Fixtures ----
|
||||
|
||||
const USERS_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const users: CollectionConfig = {
|
||||
slug: "users",
|
||||
auth: true,
|
||||
custom: {
|
||||
retention: { purgeSchedule: "daily", postDeletion: { duration: "P30D", trigger: "after-deletion", action: "hard-delete" } },
|
||||
},
|
||||
fields: [
|
||||
{
|
||||
name: "displayName",
|
||||
type: "text",
|
||||
custom: {
|
||||
pii: {
|
||||
category: "identification-username",
|
||||
purpose: ["service-delivery"],
|
||||
exportable: true,
|
||||
restrictable: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
{ name: "role", type: "select" },
|
||||
],
|
||||
};
|
||||
`;
|
||||
|
||||
const ARTICLES_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const articles: CollectionConfig = {
|
||||
slug: "articles",
|
||||
custom: {
|
||||
retention: { purgeSchedule: "monthly", postDeletion: { duration: "P90D", trigger: "after-deletion", action: "hard-delete" } },
|
||||
},
|
||||
fields: [
|
||||
{ name: "title", type: "text" },
|
||||
{ name: "content", type: "richText" },
|
||||
],
|
||||
};
|
||||
`;
|
||||
|
||||
const AUTH_PII_OVERRIDE_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const members: CollectionConfig = {
|
||||
slug: "members",
|
||||
auth: true,
|
||||
custom: {
|
||||
retention: { purgeSchedule: "daily" },
|
||||
authPii: {
|
||||
email: {
|
||||
category: "contact-email",
|
||||
purpose: ["account-authentication", "marketing-communications"],
|
||||
exportable: false,
|
||||
restrictable: true,
|
||||
},
|
||||
password: null,
|
||||
},
|
||||
},
|
||||
fields: [],
|
||||
};
|
||||
`;
|
||||
|
||||
// ---- Test helpers ----
|
||||
|
||||
function makeRepo(collections) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-data-map-"));
|
||||
for (const [pkgSlug, src] of Object.entries(collections)) {
|
||||
const dir = path.join(
|
||||
root,
|
||||
"packages",
|
||||
pkgSlug,
|
||||
"src",
|
||||
"integrations",
|
||||
"cms",
|
||||
"collections",
|
||||
);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, `${pkgSlug}.ts`), src, "utf8");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function buildMapFromFixtures(fixtures) {
|
||||
const root = makeRepo(fixtures);
|
||||
const files = findCollectionFiles(root);
|
||||
const raw = files.map((f) => parseCollectionFile(f)).filter(Boolean);
|
||||
return buildDataMap(raw);
|
||||
}
|
||||
|
||||
// ---- Tests ----
|
||||
|
||||
describe("findCollectionFiles", () => {
|
||||
test("returns all .ts files in packages/*/integrations/cms/collections/", () => {
|
||||
const root = makeRepo({ blog: ARTICLES_TS, auth: USERS_TS });
|
||||
const files = findCollectionFiles(root);
|
||||
assert.equal(files.length, 2);
|
||||
assert.ok(files.every((f) => f.endsWith(".ts")));
|
||||
assert.ok(files.some((f) => f.includes("auth")));
|
||||
assert.ok(files.some((f) => f.includes("blog")));
|
||||
});
|
||||
|
||||
test("returns empty array when packages/ does not exist", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-data-map-empty-"));
|
||||
assert.deepEqual(findCollectionFiles(root), []);
|
||||
});
|
||||
|
||||
test("excludes .test.ts files", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-data-map-test-"));
|
||||
const dir = path.join(
|
||||
root,
|
||||
"packages",
|
||||
"blog",
|
||||
"src",
|
||||
"integrations",
|
||||
"cms",
|
||||
"collections",
|
||||
);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, "articles.ts"), ARTICLES_TS, "utf8");
|
||||
fs.writeFileSync(path.join(dir, "articles.test.ts"), "// test", "utf8");
|
||||
const files = findCollectionFiles(root);
|
||||
assert.equal(files.length, 1);
|
||||
assert.ok(!files[0].endsWith(".test.ts"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseCollectionFile", () => {
|
||||
test("extracts slug, auth, fields, and custom from a collection file", () => {
|
||||
const root = makeRepo({ auth: USERS_TS });
|
||||
const [file] = findCollectionFiles(root);
|
||||
const coll = parseCollectionFile(file);
|
||||
assert.equal(coll.slug, "users");
|
||||
assert.equal(coll.auth, true);
|
||||
assert.ok(Array.isArray(coll.fields));
|
||||
assert.equal(coll.fields.length, 2);
|
||||
assert.equal(coll.fields[0].name, "displayName");
|
||||
assert.equal(coll.fields[0].custom.pii.category, "identification-username");
|
||||
});
|
||||
|
||||
test("returns null for an unreadable path", () => {
|
||||
assert.equal(parseCollectionFile("/nonexistent/path.ts"), null);
|
||||
});
|
||||
|
||||
test("extracts a collection without auth or PII", () => {
|
||||
const root = makeRepo({ blog: ARTICLES_TS });
|
||||
const [file] = findCollectionFiles(root);
|
||||
const coll = parseCollectionFile(file);
|
||||
assert.equal(coll.slug, "articles");
|
||||
assert.equal(coll.auth, undefined);
|
||||
assert.equal(coll.fields.length, 2);
|
||||
assert.ok(!coll.fields[0].custom);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDataMap — empty collections", () => {
|
||||
test("returns empty map for no collections", () => {
|
||||
assert.deepEqual(buildDataMap([]), {});
|
||||
});
|
||||
|
||||
test("includes collection with no PII fields as piiFields: []", () => {
|
||||
const map = buildMapFromFixtures({ blog: ARTICLES_TS });
|
||||
assert.ok("articles" in map);
|
||||
assert.deepEqual(map.articles.piiFields, []);
|
||||
assert.equal(map.articles.auth, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDataMap — auth defaults applied", () => {
|
||||
test("applies PAYLOAD_AUTH_PII_DEFAULTS for auth-enabled collections", () => {
|
||||
const map = buildMapFromFixtures({ auth: USERS_TS });
|
||||
const emailField = map.users.piiFields.find((f) => f.field === "email");
|
||||
assert.ok(emailField, "email field should be present from auth defaults");
|
||||
assert.equal(emailField.category, "contact-email");
|
||||
assert.equal(emailField.source, "auth-default");
|
||||
assert.deepEqual(emailField.purpose, [
|
||||
"account-authentication",
|
||||
"transactional-notifications",
|
||||
]);
|
||||
});
|
||||
|
||||
test("excludes null-valued auth defaults (e.g. password, salt)", () => {
|
||||
const map = buildMapFromFixtures({ auth: USERS_TS });
|
||||
const nullFields = ["password", "salt", "hash", "resetPasswordToken"];
|
||||
for (const name of nullFields) {
|
||||
const found = map.users.piiFields.find((f) => f.field === name);
|
||||
assert.equal(
|
||||
found,
|
||||
undefined,
|
||||
`${name} should be excluded (null default)`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("does NOT apply auth defaults for non-auth collections", () => {
|
||||
const map = buildMapFromFixtures({ blog: ARTICLES_TS });
|
||||
assert.deepEqual(map.articles.piiFields, []);
|
||||
});
|
||||
|
||||
test("includes field-level PII tags alongside auth defaults", () => {
|
||||
const map = buildMapFromFixtures({ auth: USERS_TS });
|
||||
const displayName = map.users.piiFields.find(
|
||||
(f) => f.field === "displayName",
|
||||
);
|
||||
const email = map.users.piiFields.find((f) => f.field === "email");
|
||||
assert.ok(displayName, "displayName should come from field-tag");
|
||||
assert.equal(displayName.source, "field-tag");
|
||||
assert.ok(email, "email should come from auth-default");
|
||||
assert.equal(email.source, "auth-default");
|
||||
});
|
||||
|
||||
test("piiFields are sorted by field name", () => {
|
||||
const map = buildMapFromFixtures({ auth: USERS_TS });
|
||||
const names = map.users.piiFields.map((f) => f.field);
|
||||
assert.deepEqual(names, [...names].sort());
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildDataMap — authPii override applied", () => {
|
||||
test("custom.authPii overrides PAYLOAD_AUTH_PII_DEFAULTS", () => {
|
||||
const map = buildMapFromFixtures({ members: AUTH_PII_OVERRIDE_TS });
|
||||
const emailField = map.members.piiFields.find((f) => f.field === "email");
|
||||
assert.ok(emailField, "email should be present");
|
||||
assert.equal(emailField.source, "auth-override");
|
||||
assert.equal(emailField.exportable, false);
|
||||
assert.ok(
|
||||
emailField.purpose.includes("marketing-communications"),
|
||||
"overridden purpose should be used",
|
||||
);
|
||||
});
|
||||
|
||||
test("authPii null overrides exclude the field", () => {
|
||||
const map = buildMapFromFixtures({ members: AUTH_PII_OVERRIDE_TS });
|
||||
const pwField = map.members.piiFields.find((f) => f.field === "password");
|
||||
assert.equal(
|
||||
pwField,
|
||||
undefined,
|
||||
"password null override should be excluded",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderDataMapYaml", () => {
|
||||
test("renders collections in alphabetical order", () => {
|
||||
const map = {
|
||||
users: { auth: true, piiFields: [], slug: "users" },
|
||||
articles: { auth: false, piiFields: [], slug: "articles" },
|
||||
};
|
||||
const yaml = renderDataMapYaml(map);
|
||||
const articlesIdx = yaml.indexOf(" articles:");
|
||||
const usersIdx = yaml.indexOf(" users:");
|
||||
assert.ok(articlesIdx < usersIdx, "articles should come before users");
|
||||
});
|
||||
|
||||
test("renders piiFields: [] for empty fields", () => {
|
||||
const map = { articles: { auth: false, piiFields: [], slug: "articles" } };
|
||||
const yaml = renderDataMapYaml(map);
|
||||
assert.ok(yaml.includes("piiFields: []"));
|
||||
});
|
||||
|
||||
test("renders PII field with all required keys", () => {
|
||||
const map = {
|
||||
users: {
|
||||
auth: true,
|
||||
slug: "users",
|
||||
piiFields: [
|
||||
{
|
||||
field: "email",
|
||||
source: "auth-default",
|
||||
category: "contact-email",
|
||||
purpose: ["account-authentication"],
|
||||
exportable: true,
|
||||
restrictable: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const yaml = renderDataMapYaml(map);
|
||||
assert.ok(yaml.includes("field: email"));
|
||||
assert.ok(yaml.includes("category: contact-email"));
|
||||
assert.ok(yaml.includes("source: auth-default"));
|
||||
assert.ok(yaml.includes("exportable: true"));
|
||||
assert.ok(yaml.includes("restrictable: true"));
|
||||
assert.ok(yaml.includes("- account-authentication"));
|
||||
});
|
||||
|
||||
test("rendered YAML includes header comment", () => {
|
||||
const yaml = renderDataMapYaml({});
|
||||
assert.ok(yaml.startsWith("# compliance/data-map.yml"));
|
||||
});
|
||||
|
||||
test("output is deterministic across multiple calls", () => {
|
||||
const map = buildMapFromFixtures({ auth: USERS_TS, blog: ARTICLES_TS });
|
||||
assert.equal(renderDataMapYaml(map), renderDataMapYaml(map));
|
||||
});
|
||||
});
|
||||
|
||||
describe("unifiedDiff", () => {
|
||||
test("returns null when strings are equal", () => {
|
||||
assert.equal(unifiedDiff("a\nb\n", "a\nb\n", "file.yml"), null);
|
||||
});
|
||||
|
||||
test("returns a readable diff when strings differ", () => {
|
||||
const diff = unifiedDiff("line1\nold\n", "line1\nnew\n", "file.yml");
|
||||
assert.ok(diff !== null);
|
||||
assert.ok(diff.includes("--- file.yml"));
|
||||
assert.ok(diff.includes("+++ file.yml"));
|
||||
assert.ok(diff.includes("- old"));
|
||||
assert.ok(diff.includes("+ new"));
|
||||
});
|
||||
|
||||
test("diff includes line numbers", () => {
|
||||
const diff = unifiedDiff("a\n", "b\n", "x.yml");
|
||||
assert.ok(diff.includes("Line 1:"));
|
||||
});
|
||||
|
||||
test("handles different lengths (extra lines)", () => {
|
||||
const diff = unifiedDiff("a\nb\n", "a\nb\nc\n", "x.yml");
|
||||
assert.ok(diff !== null);
|
||||
assert.ok(diff.includes("+ c"));
|
||||
});
|
||||
});
|
||||
|
||||
describe("--check mode (integration)", () => {
|
||||
test("passes when committed file matches generated output", () => {
|
||||
const map = buildMapFromFixtures({ blog: ARTICLES_TS });
|
||||
const yaml = renderDataMapYaml(map);
|
||||
const diff = unifiedDiff(yaml, yaml, "compliance/data-map.yml");
|
||||
assert.equal(diff, null, "no diff expected when file matches");
|
||||
});
|
||||
|
||||
test("fails when committed file does not match generated output", () => {
|
||||
const map = buildMapFromFixtures({ blog: ARTICLES_TS });
|
||||
const yaml = renderDataMapYaml(map);
|
||||
const stale = "# stale content\ncollections: {}\n";
|
||||
const diff = unifiedDiff(stale, yaml, "compliance/data-map.yml");
|
||||
assert.ok(diff !== null, "diff expected when file is stale");
|
||||
assert.ok(diff.includes("- # stale content"));
|
||||
});
|
||||
});
|
||||
228
scripts/compliance/emit-retention-policy.mjs
Normal file
228
scripts/compliance/emit-retention-policy.mjs
Normal file
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* emit-retention-policy.mjs — Collection retention policy emitter.
|
||||
*
|
||||
* Walks packages/*\/src/integrations/cms/collections/*.ts, validates that each
|
||||
* collection declares custom.retention.purgeSchedule, and emits a deterministic
|
||||
* YAML retention schedule at compliance/retention-policy.yml.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compliance/emit-retention-policy.mjs # write compliance/retention-policy.yml
|
||||
* node scripts/compliance/emit-retention-policy.mjs --print # write to stdout
|
||||
* node scripts/compliance/emit-retention-policy.mjs --check # diff vs committed file; exit 1 on mismatch
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
findCollectionFiles,
|
||||
parseCollectionFile,
|
||||
unifiedDiff,
|
||||
REPO_ROOT,
|
||||
} from "./emit-data-map.mjs";
|
||||
|
||||
export { findCollectionFiles, parseCollectionFile, unifiedDiff, REPO_ROOT };
|
||||
|
||||
export const OUTPUT_PATH = "compliance/retention-policy.yml";
|
||||
|
||||
// ---- Validation ----
|
||||
|
||||
/**
|
||||
* Validate that every collection has custom.retention.purgeSchedule.
|
||||
* Returns an array of error messages (empty array = valid).
|
||||
*/
|
||||
export function validateRetentionPolicy(rawCollections) {
|
||||
const errors = [];
|
||||
for (const coll of rawCollections) {
|
||||
if (!coll || typeof coll.slug !== "string") continue;
|
||||
const retention = coll.custom && coll.custom.retention;
|
||||
if (!retention || typeof retention.purgeSchedule !== "string") {
|
||||
errors.push(
|
||||
`Collection "${coll.slug}" is missing custom.retention.purgeSchedule.\n` +
|
||||
` Hint: add to your collection config:\n` +
|
||||
` custom: { retention: { purgeSchedule: "daily" | "weekly" | "monthly" } }`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
// ---- Retention policy builder ----
|
||||
|
||||
/**
|
||||
* Build the retention policy map from parsed collections.
|
||||
* Returns Record<slug, { slug, purgeSchedule, activeRetention?, postDeletion?, coldArchive? }>
|
||||
*/
|
||||
export function buildRetentionPolicy(rawCollections) {
|
||||
const map = {};
|
||||
for (const coll of rawCollections) {
|
||||
if (!coll || typeof coll.slug !== "string") continue;
|
||||
const retention = (coll.custom && coll.custom.retention) || {};
|
||||
const entry = { slug: coll.slug, purgeSchedule: retention.purgeSchedule };
|
||||
|
||||
if (retention.activeRetention) {
|
||||
entry.activeRetention = { ...retention.activeRetention };
|
||||
}
|
||||
if (retention.coldArchive) {
|
||||
entry.coldArchive = { ...retention.coldArchive };
|
||||
}
|
||||
if (retention.postDeletion) {
|
||||
entry.postDeletion = { ...retention.postDeletion };
|
||||
}
|
||||
|
||||
map[coll.slug] = entry;
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// ---- YAML serialization ----
|
||||
|
||||
const YAML_HEADER = [
|
||||
"# compliance/retention-policy.yml — Collection retention schedules",
|
||||
"# Generated by scripts/compliance/emit-retention-policy.mjs — do not edit manually.",
|
||||
"# Run `pnpm compliance:retention-policy` to regenerate.",
|
||||
].join("\n");
|
||||
|
||||
/** Quote a YAML string scalar only when necessary. */
|
||||
function yamlStr(s) {
|
||||
if (typeof s !== "string") return String(s);
|
||||
if (
|
||||
s === "" ||
|
||||
["true", "false", "null", "yes", "no", "on", "off"].includes(s) ||
|
||||
/[[{},:#&*!|>'"%@`\]]/u.test(s) ||
|
||||
/^\s|\s$/.test(s)
|
||||
) {
|
||||
return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render an optional retention sub-object (activeRetention, postDeletion, coldArchive).
|
||||
* Keys are sorted alphabetically for determinism.
|
||||
*/
|
||||
function renderRetentionBlock(label, obj, indent) {
|
||||
let out = `${indent}${label}:\n`;
|
||||
for (const key of Object.keys(obj).sort()) {
|
||||
out += `${indent} ${key}: ${yamlStr(obj[key])}\n`;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the retention policy as deterministic YAML.
|
||||
* Collections sorted by slug; optional sub-objects rendered alphabetically by key.
|
||||
*/
|
||||
export function renderRetentionPolicyYaml(policyMap) {
|
||||
let yaml = YAML_HEADER + "\n";
|
||||
yaml += "collections:\n";
|
||||
|
||||
for (const slug of Object.keys(policyMap).sort()) {
|
||||
const entry = policyMap[slug];
|
||||
yaml += ` ${yamlStr(slug)}:\n`;
|
||||
|
||||
if (entry.activeRetention) {
|
||||
yaml += renderRetentionBlock(
|
||||
"activeRetention",
|
||||
entry.activeRetention,
|
||||
" ",
|
||||
);
|
||||
}
|
||||
if (entry.coldArchive) {
|
||||
yaml += renderRetentionBlock("coldArchive", entry.coldArchive, " ");
|
||||
}
|
||||
if (entry.postDeletion) {
|
||||
yaml += renderRetentionBlock("postDeletion", entry.postDeletion, " ");
|
||||
}
|
||||
|
||||
yaml += ` purgeSchedule: ${yamlStr(entry.purgeSchedule)}\n`;
|
||||
yaml += ` slug: ${yamlStr(slug)}\n`;
|
||||
}
|
||||
|
||||
return yaml;
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { mode: "write" }; // 'write' | 'print' | 'check'
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--print") out.mode = "print";
|
||||
else if (argv[i] === "--check") out.mode = "check";
|
||||
else if (argv[i] === "--help" || argv[i] === "-h") {
|
||||
console.log(
|
||||
[
|
||||
"Usage: node scripts/compliance/emit-retention-policy.mjs [--print | --check]",
|
||||
" (default): write compliance/retention-policy.yml",
|
||||
" --print: write YAML to stdout",
|
||||
" --check: diff vs committed file; exit 1 on mismatch",
|
||||
].join("\n"),
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const repoRoot = process.cwd();
|
||||
|
||||
const files = findCollectionFiles(repoRoot);
|
||||
const rawCollections = files
|
||||
.map((f) => parseCollectionFile(f))
|
||||
.filter(Boolean);
|
||||
|
||||
const errors = validateRetentionPolicy(rawCollections);
|
||||
if (errors.length > 0) {
|
||||
for (const err of errors) {
|
||||
process.stderr.write(`[compliance:retention-policy] ${err}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const policyMap = buildRetentionPolicy(rawCollections);
|
||||
const yaml = renderRetentionPolicyYaml(policyMap);
|
||||
|
||||
const outPath = path.resolve(repoRoot, OUTPUT_PATH);
|
||||
|
||||
if (args.mode === "print") {
|
||||
process.stdout.write(yaml);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.mode === "check") {
|
||||
if (!fs.existsSync(outPath)) {
|
||||
process.stderr.write(
|
||||
`[compliance:retention-policy] --check: no committed file at ${OUTPUT_PATH}\n` +
|
||||
`Run \`pnpm compliance:retention-policy\` to generate it first.\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const committed = fs.readFileSync(outPath, "utf8");
|
||||
const diff = unifiedDiff(committed, yaml, OUTPUT_PATH);
|
||||
if (diff === null) {
|
||||
console.log(
|
||||
`✓ compliance:retention-policy — ${OUTPUT_PATH} is up to date`,
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
process.stderr.write(
|
||||
`✗ compliance:retention-policy — ${OUTPUT_PATH} is out of date:\n${diff}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Default: write to file
|
||||
const dir = path.dirname(outPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(outPath, yaml, "utf8");
|
||||
console.log(`✓ compliance:retention-policy — wrote ${OUTPUT_PATH}`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
292
scripts/compliance/emit-retention-policy.test.mjs
Normal file
292
scripts/compliance/emit-retention-policy.test.mjs
Normal file
@@ -0,0 +1,292 @@
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
findCollectionFiles,
|
||||
parseCollectionFile,
|
||||
validateRetentionPolicy,
|
||||
buildRetentionPolicy,
|
||||
renderRetentionPolicyYaml,
|
||||
unifiedDiff,
|
||||
OUTPUT_PATH,
|
||||
} from "./emit-retention-policy.mjs";
|
||||
|
||||
// ---- Fixtures ----
|
||||
|
||||
const USERS_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const users: CollectionConfig = {
|
||||
slug: "users",
|
||||
auth: true,
|
||||
custom: {
|
||||
retention: {
|
||||
purgeSchedule: "daily",
|
||||
postDeletion: { duration: "P30D", trigger: "after-deletion", action: "hard-delete" },
|
||||
},
|
||||
},
|
||||
fields: [{ name: "displayName", type: "text" }],
|
||||
};
|
||||
`;
|
||||
|
||||
const ARTICLES_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const articles: CollectionConfig = {
|
||||
slug: "articles",
|
||||
custom: {
|
||||
retention: {
|
||||
purgeSchedule: "monthly",
|
||||
postDeletion: { duration: "P90D", trigger: "after-deletion", action: "hard-delete" },
|
||||
},
|
||||
},
|
||||
fields: [{ name: "title", type: "text" }],
|
||||
};
|
||||
`;
|
||||
|
||||
const ALL_FIELDS_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const full: CollectionConfig = {
|
||||
slug: "full",
|
||||
custom: {
|
||||
retention: {
|
||||
purgeSchedule: "weekly",
|
||||
activeRetention: { duration: "P1Y", trigger: "from-last-access" },
|
||||
postDeletion: { duration: "P30D", trigger: "after-deletion", action: "pseudonymize" },
|
||||
coldArchive: { duration: "P2Y", trigger: "from-creation" },
|
||||
},
|
||||
},
|
||||
fields: [],
|
||||
};
|
||||
`;
|
||||
|
||||
const MISSING_PURGE_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const orphan: CollectionConfig = {
|
||||
slug: "orphan",
|
||||
custom: { retention: { postDeletion: { duration: "P30D", trigger: "after-deletion", action: "hard-delete" } } },
|
||||
fields: [],
|
||||
};
|
||||
`;
|
||||
|
||||
const NO_RETENTION_TS = `
|
||||
import type { CollectionConfig } from "payload";
|
||||
export const bare: CollectionConfig = {
|
||||
slug: "bare",
|
||||
fields: [],
|
||||
};
|
||||
`;
|
||||
|
||||
// ---- Test helpers ----
|
||||
|
||||
function makeRepo(collections) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-retention-policy-"));
|
||||
for (const [pkgSlug, src] of Object.entries(collections)) {
|
||||
const dir = path.join(
|
||||
root,
|
||||
"packages",
|
||||
pkgSlug,
|
||||
"src",
|
||||
"integrations",
|
||||
"cms",
|
||||
"collections",
|
||||
);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(path.join(dir, `${pkgSlug}.ts`), src, "utf8");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function parseFixtures(fixtures) {
|
||||
const root = makeRepo(fixtures);
|
||||
const files = findCollectionFiles(root);
|
||||
return files.map((f) => parseCollectionFile(f)).filter(Boolean);
|
||||
}
|
||||
|
||||
// ---- Tests ----
|
||||
|
||||
describe("validateRetentionPolicy — required fields", () => {
|
||||
test("returns no errors when all collections have purgeSchedule", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS, blog: ARTICLES_TS });
|
||||
assert.deepEqual(validateRetentionPolicy(raw), []);
|
||||
});
|
||||
|
||||
test("returns an error for a collection missing purgeSchedule", () => {
|
||||
const raw = parseFixtures({ orphan: MISSING_PURGE_TS });
|
||||
const errors = validateRetentionPolicy(raw);
|
||||
assert.equal(errors.length, 1);
|
||||
assert.ok(errors[0].includes(`"orphan"`));
|
||||
assert.ok(errors[0].includes("purgeSchedule"));
|
||||
assert.ok(errors[0].includes("Hint:"));
|
||||
});
|
||||
|
||||
test("returns an error for a collection with no custom.retention at all", () => {
|
||||
const raw = parseFixtures({ bare: NO_RETENTION_TS });
|
||||
const errors = validateRetentionPolicy(raw);
|
||||
assert.equal(errors.length, 1);
|
||||
assert.ok(errors[0].includes(`"bare"`));
|
||||
});
|
||||
|
||||
test("returns multiple errors when multiple collections are missing purgeSchedule", () => {
|
||||
const raw = [
|
||||
{ slug: "alpha", custom: {} },
|
||||
{ slug: "beta", custom: { retention: {} } },
|
||||
];
|
||||
const errors = validateRetentionPolicy(raw);
|
||||
assert.equal(errors.length, 2);
|
||||
assert.ok(errors.some((e) => e.includes('"alpha"')));
|
||||
assert.ok(errors.some((e) => e.includes('"beta"')));
|
||||
});
|
||||
|
||||
test("returns no errors for empty collections array", () => {
|
||||
assert.deepEqual(validateRetentionPolicy([]), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildRetentionPolicy", () => {
|
||||
test("builds a map with slug and purgeSchedule for each collection", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS, blog: ARTICLES_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
assert.ok("users" in map);
|
||||
assert.ok("articles" in map);
|
||||
assert.equal(map.users.purgeSchedule, "daily");
|
||||
assert.equal(map.articles.purgeSchedule, "monthly");
|
||||
assert.equal(map.users.slug, "users");
|
||||
});
|
||||
|
||||
test("includes postDeletion when present", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
assert.ok(map.users.postDeletion);
|
||||
assert.equal(map.users.postDeletion.duration, "P30D");
|
||||
assert.equal(map.users.postDeletion.trigger, "after-deletion");
|
||||
assert.equal(map.users.postDeletion.action, "hard-delete");
|
||||
});
|
||||
|
||||
test("includes all optional fields (activeRetention, postDeletion, coldArchive)", () => {
|
||||
const raw = parseFixtures({ full: ALL_FIELDS_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
assert.ok(map.full.activeRetention);
|
||||
assert.equal(map.full.activeRetention.duration, "P1Y");
|
||||
assert.equal(map.full.activeRetention.trigger, "from-last-access");
|
||||
assert.ok(map.full.coldArchive);
|
||||
assert.equal(map.full.coldArchive.duration, "P2Y");
|
||||
assert.equal(map.full.coldArchive.trigger, "from-creation");
|
||||
assert.ok(map.full.postDeletion);
|
||||
assert.equal(map.full.postDeletion.action, "pseudonymize");
|
||||
});
|
||||
|
||||
test("omits absent optional fields from entry", () => {
|
||||
const raw = [
|
||||
{ slug: "simple", custom: { retention: { purgeSchedule: "weekly" } } },
|
||||
];
|
||||
const map = buildRetentionPolicy(raw);
|
||||
assert.ok(!("activeRetention" in map.simple));
|
||||
assert.ok(!("coldArchive" in map.simple));
|
||||
assert.ok(!("postDeletion" in map.simple));
|
||||
});
|
||||
|
||||
test("returns empty map for no collections", () => {
|
||||
assert.deepEqual(buildRetentionPolicy([]), {});
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderRetentionPolicyYaml", () => {
|
||||
test("renders collections in alphabetical order", () => {
|
||||
const map = {
|
||||
users: { slug: "users", purgeSchedule: "daily" },
|
||||
articles: { slug: "articles", purgeSchedule: "monthly" },
|
||||
};
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
const articlesIdx = yaml.indexOf(" articles:");
|
||||
const usersIdx = yaml.indexOf(" users:");
|
||||
assert.ok(articlesIdx < usersIdx, "articles should appear before users");
|
||||
});
|
||||
|
||||
test("renders purgeSchedule and slug for every collection", () => {
|
||||
const map = { blog: { slug: "blog", purgeSchedule: "monthly" } };
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
assert.ok(yaml.includes("purgeSchedule: monthly"));
|
||||
assert.ok(yaml.includes("slug: blog"));
|
||||
});
|
||||
|
||||
test("renders postDeletion sub-object with sorted keys", () => {
|
||||
const map = {
|
||||
users: {
|
||||
slug: "users",
|
||||
purgeSchedule: "daily",
|
||||
postDeletion: {
|
||||
action: "hard-delete",
|
||||
duration: "P30D",
|
||||
trigger: "after-deletion",
|
||||
},
|
||||
},
|
||||
};
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
assert.ok(yaml.includes("postDeletion:"));
|
||||
assert.ok(yaml.includes("action: hard-delete"));
|
||||
assert.ok(yaml.includes("duration: P30D"));
|
||||
assert.ok(yaml.includes("trigger: after-deletion"));
|
||||
// action (a) comes before duration (d) comes before trigger (t)
|
||||
const actionIdx = yaml.indexOf("action:");
|
||||
const durationIdx = yaml.indexOf("duration:");
|
||||
const triggerIdx = yaml.indexOf("trigger:");
|
||||
assert.ok(actionIdx < durationIdx && durationIdx < triggerIdx);
|
||||
});
|
||||
|
||||
test("renders activeRetention and coldArchive when present", () => {
|
||||
const raw = parseFixtures({ full: ALL_FIELDS_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
assert.ok(yaml.includes("activeRetention:"));
|
||||
assert.ok(yaml.includes("coldArchive:"));
|
||||
});
|
||||
|
||||
test("rendered YAML includes header comment", () => {
|
||||
const yaml = renderRetentionPolicyYaml({});
|
||||
assert.ok(yaml.startsWith("# compliance/retention-policy.yml"));
|
||||
assert.ok(yaml.includes("emit-retention-policy.mjs"));
|
||||
});
|
||||
|
||||
test("output is deterministic across multiple calls", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS, blog: ARTICLES_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
assert.equal(
|
||||
renderRetentionPolicyYaml(map),
|
||||
renderRetentionPolicyYaml(map),
|
||||
);
|
||||
});
|
||||
|
||||
test("OUTPUT_PATH is compliance/retention-policy.yml", () => {
|
||||
assert.equal(OUTPUT_PATH, "compliance/retention-policy.yml");
|
||||
});
|
||||
});
|
||||
|
||||
describe("--check mode (integration)", () => {
|
||||
test("passes when committed file matches generated output", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
const diff = unifiedDiff(yaml, yaml, OUTPUT_PATH);
|
||||
assert.equal(diff, null, "no diff expected when files match");
|
||||
});
|
||||
|
||||
test("fails with readable diff when committed file is stale", () => {
|
||||
const raw = parseFixtures({ auth: USERS_TS });
|
||||
const map = buildRetentionPolicy(raw);
|
||||
const yaml = renderRetentionPolicyYaml(map);
|
||||
const stale = "# stale content\ncollections: {}\n";
|
||||
const diff = unifiedDiff(stale, yaml, OUTPUT_PATH);
|
||||
assert.ok(diff !== null, "diff expected when file is stale");
|
||||
assert.ok(
|
||||
diff.includes(`--- ${OUTPUT_PATH}`),
|
||||
"diff should include the filename header",
|
||||
);
|
||||
assert.ok(
|
||||
diff.includes("- # stale content"),
|
||||
"diff should show removed line",
|
||||
);
|
||||
assert.ok(diff.includes("Line"), "diff should include line numbers");
|
||||
});
|
||||
});
|
||||
339
scripts/compliance/emit-sub-processors.mjs
Normal file
339
scripts/compliance/emit-sub-processors.mjs
Normal file
@@ -0,0 +1,339 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* emit-sub-processors.mjs — Third-party sub-processor inventory emitter.
|
||||
*
|
||||
* Walks docs/library-decisions/*.md, filters entries where is-sub-processor: true,
|
||||
* merges compliance/sub-processors.manual.yml (if present) with source: manual flag,
|
||||
* and emits a sorted deterministic YAML inventory at compliance/sub-processors.yml.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compliance/emit-sub-processors.mjs # write compliance/sub-processors.yml
|
||||
* node scripts/compliance/emit-sub-processors.mjs --print # write to stdout
|
||||
* node scripts/compliance/emit-sub-processors.mjs --check # diff vs committed file; exit 1 on mismatch
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import { unifiedDiff, REPO_ROOT } from "./emit-data-map.mjs";
|
||||
|
||||
export { unifiedDiff, REPO_ROOT };
|
||||
|
||||
export const OUTPUT_PATH = "compliance/sub-processors.yml";
|
||||
export const MANUAL_PATH = "compliance/sub-processors.manual.yml";
|
||||
|
||||
// ---- Frontmatter parser ----
|
||||
|
||||
/** Parse a YAML scalar value string into its JS equivalent. */
|
||||
function parseScalarValue(raw) {
|
||||
if (raw === "true") return true;
|
||||
if (raw === "false") return false;
|
||||
if (raw === "null") return null;
|
||||
if (
|
||||
(raw.startsWith('"') && raw.endsWith('"')) ||
|
||||
(raw.startsWith("'") && raw.endsWith("'"))
|
||||
) {
|
||||
return raw.slice(1, -1);
|
||||
}
|
||||
return raw;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse top-level scalar fields from YAML frontmatter in a markdown file.
|
||||
* Returns an object of key → value pairs, or null if no frontmatter is found.
|
||||
* Skips comment lines, empty lines, and indented lines (nested block fields).
|
||||
*/
|
||||
export function parseFrontmatter(src) {
|
||||
const match = /^---\r?\n([\s\S]*?)\r?\n---/.exec(src);
|
||||
if (!match) return null;
|
||||
|
||||
const result = {};
|
||||
for (const line of match[1].split("\n")) {
|
||||
if (!line.trim() || line.startsWith("#")) continue;
|
||||
if (line.startsWith(" ") || line.startsWith("\t")) continue;
|
||||
|
||||
const colonIdx = line.indexOf(":");
|
||||
if (colonIdx === -1) continue;
|
||||
|
||||
const key = line.slice(0, colonIdx).trim();
|
||||
const rawValue = line.slice(colonIdx + 1).trim();
|
||||
if (!rawValue) continue; // block header like "filter-results:"
|
||||
|
||||
result[key] = parseScalarValue(rawValue);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// ---- Library decision file discovery ----
|
||||
|
||||
/**
|
||||
* Find all library decision markdown files under docs/library-decisions/.
|
||||
* Excludes _template.md and any file starting with `_`.
|
||||
*/
|
||||
export function findLibraryDecisionFiles(repoRoot = REPO_ROOT) {
|
||||
const dir = path.join(repoRoot, "docs", "library-decisions");
|
||||
if (!fs.existsSync(dir)) return [];
|
||||
|
||||
return fs
|
||||
.readdirSync(dir)
|
||||
.sort()
|
||||
.filter((f) => f.endsWith(".md") && !f.startsWith("_"))
|
||||
.map((f) => path.join(dir, f));
|
||||
}
|
||||
|
||||
// ---- Sub-processor parsing from library traces ----
|
||||
|
||||
const TRACE_FIELDS = [
|
||||
"package",
|
||||
"version",
|
||||
"decision",
|
||||
"data-sent",
|
||||
"region",
|
||||
"dpa-signed",
|
||||
"sccs-required",
|
||||
"contact",
|
||||
];
|
||||
|
||||
/**
|
||||
* Walk library decision files and return sub-processor entries where
|
||||
* is-sub-processor: true. Each entry gets source: "library-trace" injected.
|
||||
*/
|
||||
export function parseLibraryTraceSubProcessors(repoRoot = REPO_ROOT) {
|
||||
const files = findLibraryDecisionFiles(repoRoot);
|
||||
const entries = [];
|
||||
|
||||
for (const filePath of files) {
|
||||
let src;
|
||||
try {
|
||||
src = fs.readFileSync(filePath, "utf8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const meta = parseFrontmatter(src);
|
||||
if (!meta || meta["is-sub-processor"] !== true) continue;
|
||||
|
||||
const entry = { source: "library-trace" };
|
||||
for (const field of TRACE_FIELDS) {
|
||||
if (meta[field] !== undefined) entry[field] = meta[field];
|
||||
}
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
// ---- Manual entries ----
|
||||
|
||||
/**
|
||||
* Parse a simple YAML list-of-objects: each item starts with "- key: value"
|
||||
* and continuation lines are indented with spaces or tabs.
|
||||
* Returns an array of plain objects.
|
||||
*/
|
||||
export function parseSimpleYamlList(src) {
|
||||
const entries = [];
|
||||
let current = null;
|
||||
|
||||
for (const line of src.split("\n")) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith("#")) continue;
|
||||
|
||||
if (line.startsWith("- ")) {
|
||||
if (current) entries.push(current);
|
||||
current = {};
|
||||
const rest = line.slice(2).trim();
|
||||
const colonIdx = rest.indexOf(":");
|
||||
if (colonIdx !== -1) {
|
||||
const key = rest.slice(0, colonIdx).trim();
|
||||
const rawVal = rest.slice(colonIdx + 1).trim();
|
||||
if (rawVal) current[key] = parseScalarValue(rawVal);
|
||||
}
|
||||
} else if (current && (line.startsWith(" ") || line.startsWith("\t"))) {
|
||||
const colonIdx = line.indexOf(":");
|
||||
if (colonIdx !== -1) {
|
||||
const key = line.slice(0, colonIdx).trim();
|
||||
const rawVal = line.slice(colonIdx + 1).trim();
|
||||
if (rawVal) current[key] = parseScalarValue(rawVal);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (current) entries.push(current);
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Load manual sub-processor entries from compliance/sub-processors.manual.yml.
|
||||
* Returns empty array if the file doesn't exist (graceful skip).
|
||||
* Injects source: "manual" into each entry.
|
||||
*/
|
||||
export function loadManualEntries(repoRoot = REPO_ROOT) {
|
||||
const manualPath = path.resolve(repoRoot, MANUAL_PATH);
|
||||
if (!fs.existsSync(manualPath)) return [];
|
||||
|
||||
let src;
|
||||
try {
|
||||
src = fs.readFileSync(manualPath, "utf8");
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
|
||||
return parseSimpleYamlList(src).map((entry) => ({
|
||||
...entry,
|
||||
source: "manual",
|
||||
}));
|
||||
}
|
||||
|
||||
// ---- Builder ----
|
||||
|
||||
/**
|
||||
* Merge library-trace and manual sub-processor entries.
|
||||
* Sorts by package name for deterministic output.
|
||||
*/
|
||||
export function buildSubProcessors(traced, manual) {
|
||||
const all = [...traced, ...manual];
|
||||
all.sort((a, b) =>
|
||||
String(a.package ?? "").localeCompare(String(b.package ?? "")),
|
||||
);
|
||||
return all;
|
||||
}
|
||||
|
||||
// ---- YAML serialization ----
|
||||
|
||||
const YAML_HEADER = [
|
||||
"# compliance/sub-processors.yml — Third-party sub-processor inventory",
|
||||
"# Generated by scripts/compliance/emit-sub-processors.mjs — do not edit manually.",
|
||||
"# Run `pnpm compliance:sub-processors` to regenerate.",
|
||||
].join("\n");
|
||||
|
||||
/** Quote a YAML string scalar only when necessary. */
|
||||
function yamlStr(s) {
|
||||
if (typeof s !== "string") return String(s);
|
||||
if (
|
||||
s === "" ||
|
||||
["true", "false", "null", "yes", "no", "on", "off"].includes(s) ||
|
||||
/[[{},:#&*!|>'"%@`\]]/u.test(s) ||
|
||||
/^\s|\s$/.test(s)
|
||||
) {
|
||||
return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// package first (natural identifier), then remaining fields alphabetically
|
||||
const ENTRY_FIELDS = [
|
||||
"package",
|
||||
"contact",
|
||||
"data-sent",
|
||||
"decision",
|
||||
"dpa-signed",
|
||||
"region",
|
||||
"sccs-required",
|
||||
"source",
|
||||
"version",
|
||||
];
|
||||
|
||||
/**
|
||||
* Render sub-processors as deterministic YAML.
|
||||
* Entries sorted by package name (done in buildSubProcessors).
|
||||
* Fields rendered in a fixed order: package first, rest alphabetical.
|
||||
*/
|
||||
export function renderSubProcessorsYaml(entries) {
|
||||
let yaml = YAML_HEADER + "\n";
|
||||
yaml += "sub-processors:\n";
|
||||
|
||||
if (entries.length === 0) {
|
||||
yaml += " []\n";
|
||||
return yaml;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
let first = true;
|
||||
for (const field of ENTRY_FIELDS) {
|
||||
const value = entry[field];
|
||||
if (value === undefined) continue;
|
||||
|
||||
const rendered =
|
||||
typeof value === "boolean" ? String(value) : yamlStr(String(value));
|
||||
|
||||
if (first) {
|
||||
yaml += ` - ${field}: ${rendered}\n`;
|
||||
first = false;
|
||||
} else {
|
||||
yaml += ` ${field}: ${rendered}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return yaml;
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { mode: "write" }; // 'write' | 'print' | 'check'
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--print") out.mode = "print";
|
||||
else if (argv[i] === "--check") out.mode = "check";
|
||||
else if (argv[i] === "--help" || argv[i] === "-h") {
|
||||
console.log(
|
||||
[
|
||||
"Usage: node scripts/compliance/emit-sub-processors.mjs [--print | --check]",
|
||||
" (default): write compliance/sub-processors.yml",
|
||||
" --print: write YAML to stdout",
|
||||
" --check: diff vs committed file; exit 1 on mismatch",
|
||||
].join("\n"),
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const repoRoot = process.cwd();
|
||||
|
||||
const traced = parseLibraryTraceSubProcessors(repoRoot);
|
||||
const manual = loadManualEntries(repoRoot);
|
||||
const entries = buildSubProcessors(traced, manual);
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
|
||||
const outPath = path.resolve(repoRoot, OUTPUT_PATH);
|
||||
|
||||
if (args.mode === "print") {
|
||||
process.stdout.write(yaml);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.mode === "check") {
|
||||
if (!fs.existsSync(outPath)) {
|
||||
process.stderr.write(
|
||||
`[compliance:sub-processors] --check: no committed file at ${OUTPUT_PATH}\n` +
|
||||
`Run \`pnpm compliance:sub-processors\` to generate it first.\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const committed = fs.readFileSync(outPath, "utf8");
|
||||
const diff = unifiedDiff(committed, yaml, OUTPUT_PATH);
|
||||
if (diff === null) {
|
||||
console.log(`✓ compliance:sub-processors — ${OUTPUT_PATH} is up to date`);
|
||||
process.exit(0);
|
||||
}
|
||||
process.stderr.write(
|
||||
`✗ compliance:sub-processors — ${OUTPUT_PATH} is out of date:\n${diff}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Default: write to file
|
||||
const dir = path.dirname(outPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(outPath, yaml, "utf8");
|
||||
console.log(`✓ compliance:sub-processors — wrote ${OUTPUT_PATH}`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
512
scripts/compliance/emit-sub-processors.test.mjs
Normal file
512
scripts/compliance/emit-sub-processors.test.mjs
Normal file
@@ -0,0 +1,512 @@
|
||||
import { test, describe } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
import {
|
||||
parseFrontmatter,
|
||||
findLibraryDecisionFiles,
|
||||
parseLibraryTraceSubProcessors,
|
||||
parseSimpleYamlList,
|
||||
loadManualEntries,
|
||||
buildSubProcessors,
|
||||
renderSubProcessorsYaml,
|
||||
unifiedDiff,
|
||||
OUTPUT_PATH,
|
||||
MANUAL_PATH,
|
||||
} from "./emit-sub-processors.mjs";
|
||||
|
||||
// ---- Fixtures ----
|
||||
|
||||
const SUB_PROCESSOR_MD = `---
|
||||
package: stripe
|
||||
version: "^14.0.0"
|
||||
tier: feature
|
||||
decision: approved
|
||||
date: 2026-05-18
|
||||
deciders: [Danijel Martinek]
|
||||
adr: null
|
||||
lastRevalidated: null
|
||||
is-sub-processor: true
|
||||
processes-pii: true
|
||||
data-sent: payment card details and billing address
|
||||
region: eu-west-1
|
||||
dpa-signed: true
|
||||
sccs-required: false
|
||||
contact: https://stripe.com/privacy
|
||||
filter-results:
|
||||
license: MIT
|
||||
types: native
|
||||
maintenance: active
|
||||
boundary-fit: pass
|
||||
shadow-check: pass
|
||||
eu-residency: ok
|
||||
cve-scan: clean
|
||||
named-consumer: pass
|
||||
socketRisk: clean
|
||||
verification-commands:
|
||||
- npm view stripe license
|
||||
accepted-cves: []
|
||||
---
|
||||
|
||||
## Filter: license
|
||||
|
||||
MIT.
|
||||
`;
|
||||
|
||||
const NON_SUB_PROCESSOR_MD = `---
|
||||
package: zod
|
||||
version: "^3.0.0"
|
||||
tier: core
|
||||
decision: approved
|
||||
date: 2026-05-14
|
||||
deciders: [Danijel Martinek]
|
||||
adr: null
|
||||
lastRevalidated: null
|
||||
is-sub-processor: false
|
||||
processes-pii: false
|
||||
filter-results:
|
||||
license: MIT
|
||||
types: native
|
||||
maintenance: active
|
||||
boundary-fit: pass
|
||||
shadow-check: pass
|
||||
eu-residency: n/a
|
||||
cve-scan: clean
|
||||
named-consumer: pass
|
||||
socketRisk: clean
|
||||
verification-commands:
|
||||
- npm view zod license
|
||||
accepted-cves: []
|
||||
---
|
||||
`;
|
||||
|
||||
const ANOTHER_SUB_PROCESSOR_MD = `---
|
||||
package: sendgrid
|
||||
version: "^7.0.0"
|
||||
tier: feature
|
||||
decision: approved
|
||||
date: 2026-05-18
|
||||
deciders: [Danijel Martinek]
|
||||
adr: null
|
||||
lastRevalidated: null
|
||||
is-sub-processor: true
|
||||
processes-pii: true
|
||||
data-sent: email address and name for transactional emails
|
||||
region: eu
|
||||
dpa-signed: false
|
||||
sccs-required: true
|
||||
contact: https://sendgrid.com/privacy
|
||||
filter-results:
|
||||
license: MIT
|
||||
types: native
|
||||
maintenance: active
|
||||
boundary-fit: pass
|
||||
shadow-check: pass
|
||||
eu-residency: ok
|
||||
cve-scan: clean
|
||||
named-consumer: pass
|
||||
socketRisk: clean
|
||||
verification-commands:
|
||||
- npm view @sendgrid/mail license
|
||||
accepted-cves: []
|
||||
---
|
||||
`;
|
||||
|
||||
const MANUAL_YAML = `- package: aws-s3
|
||||
data-sent: file uploads and user avatars
|
||||
region: eu-west-1
|
||||
dpa-signed: true
|
||||
sccs-required: false
|
||||
contact: https://aws.amazon.com/compliance/eu-data-privacy/
|
||||
decision: approved
|
||||
version: "^3.0.0"
|
||||
`;
|
||||
|
||||
// ---- Test helpers ----
|
||||
|
||||
function makeLibraryDecisions(files) {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "emit-sub-processors-"));
|
||||
const dir = path.join(root, "docs", "library-decisions");
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
for (const [name, src] of Object.entries(files)) {
|
||||
fs.writeFileSync(path.join(dir, name), src, "utf8");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
function makeRepoWithManual(libraryFiles, manualYaml) {
|
||||
const root = makeLibraryDecisions(libraryFiles);
|
||||
if (manualYaml !== undefined) {
|
||||
const complianceDir = path.join(root, "compliance");
|
||||
fs.mkdirSync(complianceDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(root, MANUAL_PATH), manualYaml, "utf8");
|
||||
}
|
||||
return root;
|
||||
}
|
||||
|
||||
// ---- Tests ----
|
||||
|
||||
describe("parseFrontmatter", () => {
|
||||
test("parses top-level scalar fields", () => {
|
||||
const meta = parseFrontmatter(SUB_PROCESSOR_MD);
|
||||
assert.equal(meta.package, "stripe");
|
||||
assert.equal(meta["is-sub-processor"], true);
|
||||
assert.equal(meta["dpa-signed"], true);
|
||||
assert.equal(meta["sccs-required"], false);
|
||||
assert.equal(meta.decision, "approved");
|
||||
assert.equal(meta.contact, "https://stripe.com/privacy");
|
||||
});
|
||||
|
||||
test("parses false boolean correctly (is-sub-processor: false)", () => {
|
||||
const meta = parseFrontmatter(NON_SUB_PROCESSOR_MD);
|
||||
assert.equal(meta["is-sub-processor"], false);
|
||||
assert.equal(meta["processes-pii"], false);
|
||||
});
|
||||
|
||||
test("strips quotes from version strings", () => {
|
||||
const meta = parseFrontmatter(SUB_PROCESSOR_MD);
|
||||
assert.equal(meta.version, "^14.0.0");
|
||||
});
|
||||
|
||||
test("skips nested block fields (filter-results)", () => {
|
||||
const meta = parseFrontmatter(SUB_PROCESSOR_MD);
|
||||
assert.ok(
|
||||
!("license" in meta),
|
||||
"should not include nested filter-results.license",
|
||||
);
|
||||
assert.ok(
|
||||
!("filter-results" in meta),
|
||||
"filter-results block header should be skipped",
|
||||
);
|
||||
});
|
||||
|
||||
test("returns null when no frontmatter is present", () => {
|
||||
assert.equal(parseFrontmatter("no frontmatter here"), null);
|
||||
});
|
||||
|
||||
test("returns null for empty string", () => {
|
||||
assert.equal(parseFrontmatter(""), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe("findLibraryDecisionFiles", () => {
|
||||
test("finds all .md files and excludes _template.md", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-stripe.md": SUB_PROCESSOR_MD,
|
||||
"2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
|
||||
"_template.md": "# template",
|
||||
});
|
||||
const files = findLibraryDecisionFiles(root);
|
||||
assert.equal(files.length, 2);
|
||||
assert.ok(files.every((f) => !path.basename(f).startsWith("_")));
|
||||
});
|
||||
|
||||
test("returns files in sorted order", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-18-stripe.md": SUB_PROCESSOR_MD,
|
||||
"2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
|
||||
});
|
||||
const files = findLibraryDecisionFiles(root);
|
||||
assert.ok(
|
||||
path.basename(files[0]) < path.basename(files[1]),
|
||||
"files should be sorted alphabetically",
|
||||
);
|
||||
});
|
||||
|
||||
test("returns empty array when docs/library-decisions does not exist", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "empty-repo-"));
|
||||
assert.deepEqual(findLibraryDecisionFiles(root), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseLibraryTraceSubProcessors — discriminated union", () => {
|
||||
test("returns only entries where is-sub-processor: true", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-stripe.md": SUB_PROCESSOR_MD,
|
||||
"2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
|
||||
});
|
||||
const entries = parseLibraryTraceSubProcessors(root);
|
||||
assert.equal(entries.length, 1);
|
||||
assert.equal(entries[0].package, "stripe");
|
||||
});
|
||||
|
||||
test("sets source to library-trace for parsed entries", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-stripe.md": SUB_PROCESSOR_MD,
|
||||
});
|
||||
const [entry] = parseLibraryTraceSubProcessors(root);
|
||||
assert.equal(entry.source, "library-trace");
|
||||
});
|
||||
|
||||
test("extracts all sub-processor fields from frontmatter", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-stripe.md": SUB_PROCESSOR_MD,
|
||||
});
|
||||
const [entry] = parseLibraryTraceSubProcessors(root);
|
||||
assert.equal(entry.package, "stripe");
|
||||
assert.equal(entry.version, "^14.0.0");
|
||||
assert.equal(entry.decision, "approved");
|
||||
assert.equal(
|
||||
entry["data-sent"],
|
||||
"payment card details and billing address",
|
||||
);
|
||||
assert.equal(entry.region, "eu-west-1");
|
||||
assert.equal(entry["dpa-signed"], true);
|
||||
assert.equal(entry["sccs-required"], false);
|
||||
assert.equal(entry.contact, "https://stripe.com/privacy");
|
||||
});
|
||||
|
||||
test("returns empty array when no sub-processors exist", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
|
||||
});
|
||||
assert.deepEqual(parseLibraryTraceSubProcessors(root), []);
|
||||
});
|
||||
|
||||
test("returns empty array when library-decisions dir is absent", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "no-decisions-"));
|
||||
assert.deepEqual(parseLibraryTraceSubProcessors(root), []);
|
||||
});
|
||||
|
||||
test("handles multiple sub-processor files", () => {
|
||||
const root = makeLibraryDecisions({
|
||||
"2026-05-14-stripe.md": SUB_PROCESSOR_MD,
|
||||
"2026-05-14-sendgrid.md": ANOTHER_SUB_PROCESSOR_MD,
|
||||
"2026-05-14-zod.md": NON_SUB_PROCESSOR_MD,
|
||||
});
|
||||
const entries = parseLibraryTraceSubProcessors(root);
|
||||
assert.equal(entries.length, 2);
|
||||
const packages = entries.map((e) => e.package).sort();
|
||||
assert.deepEqual(packages, ["sendgrid", "stripe"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseSimpleYamlList", () => {
|
||||
test("parses a simple YAML list into an array of objects", () => {
|
||||
const entries = parseSimpleYamlList(MANUAL_YAML);
|
||||
assert.equal(entries.length, 1);
|
||||
assert.equal(entries[0].package, "aws-s3");
|
||||
assert.equal(entries[0]["dpa-signed"], true);
|
||||
assert.equal(entries[0]["sccs-required"], false);
|
||||
assert.equal(entries[0].region, "eu-west-1");
|
||||
});
|
||||
|
||||
test("strips quotes from quoted values", () => {
|
||||
const entries = parseSimpleYamlList(MANUAL_YAML);
|
||||
assert.equal(entries[0].version, "^3.0.0");
|
||||
});
|
||||
|
||||
test("parses multiple list items", () => {
|
||||
const src = `- package: alpha
|
||||
region: eu
|
||||
- package: beta
|
||||
region: us
|
||||
`;
|
||||
const entries = parseSimpleYamlList(src);
|
||||
assert.equal(entries.length, 2);
|
||||
assert.equal(entries[0].package, "alpha");
|
||||
assert.equal(entries[1].package, "beta");
|
||||
});
|
||||
|
||||
test("returns empty array for empty input", () => {
|
||||
assert.deepEqual(parseSimpleYamlList(""), []);
|
||||
});
|
||||
|
||||
test("returns empty array for comment-only input", () => {
|
||||
assert.deepEqual(
|
||||
parseSimpleYamlList("# just a comment\n# another line"),
|
||||
[],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("loadManualEntries", () => {
|
||||
test("returns empty array when manual file is absent (graceful skip)", () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), "no-manual-"));
|
||||
assert.deepEqual(loadManualEntries(root), []);
|
||||
});
|
||||
|
||||
test("loads manual entries and injects source: manual", () => {
|
||||
const root = makeRepoWithManual({}, MANUAL_YAML);
|
||||
const entries = loadManualEntries(root);
|
||||
assert.equal(entries.length, 1);
|
||||
assert.equal(entries[0].package, "aws-s3");
|
||||
assert.equal(entries[0].source, "manual");
|
||||
});
|
||||
|
||||
test("preserves all fields from manual file", () => {
|
||||
const root = makeRepoWithManual({}, MANUAL_YAML);
|
||||
const [entry] = loadManualEntries(root);
|
||||
assert.equal(entry["data-sent"], "file uploads and user avatars");
|
||||
assert.equal(entry["dpa-signed"], true);
|
||||
assert.equal(entry["sccs-required"], false);
|
||||
assert.equal(entry.decision, "approved");
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildSubProcessors — merge and sort", () => {
|
||||
test("merges traced and manual entries sorted by package name", () => {
|
||||
const traced = [{ package: "stripe", source: "library-trace" }];
|
||||
const manual = [{ package: "aws-s3", source: "manual" }];
|
||||
const merged = buildSubProcessors(traced, manual);
|
||||
assert.equal(merged.length, 2);
|
||||
assert.equal(merged[0].package, "aws-s3");
|
||||
assert.equal(merged[1].package, "stripe");
|
||||
});
|
||||
|
||||
test("preserves source field for each entry type", () => {
|
||||
const traced = [{ package: "stripe", source: "library-trace" }];
|
||||
const manual = [{ package: "aws-s3", source: "manual" }];
|
||||
const merged = buildSubProcessors(traced, manual);
|
||||
assert.equal(merged[0].source, "manual");
|
||||
assert.equal(merged[1].source, "library-trace");
|
||||
});
|
||||
|
||||
test("returns empty array when both inputs are empty", () => {
|
||||
assert.deepEqual(buildSubProcessors([], []), []);
|
||||
});
|
||||
|
||||
test("returns traced-only when no manual entries", () => {
|
||||
const traced = [{ package: "stripe", source: "library-trace" }];
|
||||
const merged = buildSubProcessors(traced, []);
|
||||
assert.equal(merged.length, 1);
|
||||
assert.equal(merged[0].source, "library-trace");
|
||||
});
|
||||
|
||||
test("returns manual-only when no traced entries", () => {
|
||||
const manual = [{ package: "aws-s3", source: "manual" }];
|
||||
const merged = buildSubProcessors([], manual);
|
||||
assert.equal(merged.length, 1);
|
||||
assert.equal(merged[0].source, "manual");
|
||||
});
|
||||
|
||||
test("sorts multiple entries alphabetically by package", () => {
|
||||
const traced = [
|
||||
{ package: "zod", source: "library-trace" },
|
||||
{ package: "stripe", source: "library-trace" },
|
||||
];
|
||||
const merged = buildSubProcessors(traced, []);
|
||||
assert.equal(merged[0].package, "stripe");
|
||||
assert.equal(merged[1].package, "zod");
|
||||
});
|
||||
});
|
||||
|
||||
describe("renderSubProcessorsYaml", () => {
|
||||
test("renders YAML header comment", () => {
|
||||
const yaml = renderSubProcessorsYaml([]);
|
||||
assert.ok(yaml.startsWith("# compliance/sub-processors.yml"));
|
||||
assert.ok(yaml.includes("emit-sub-processors.mjs"));
|
||||
});
|
||||
|
||||
test("renders empty list as sub-processors: []", () => {
|
||||
const yaml = renderSubProcessorsYaml([]);
|
||||
assert.ok(yaml.includes("sub-processors:"));
|
||||
assert.ok(yaml.includes(" []"));
|
||||
});
|
||||
|
||||
test("renders entries in package-name order", () => {
|
||||
const entries = [
|
||||
{ package: "aws-s3", source: "manual", decision: "approved" },
|
||||
{ package: "stripe", source: "library-trace", decision: "approved" },
|
||||
];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
const awsIdx = yaml.indexOf("package: aws-s3");
|
||||
const stripeIdx = yaml.indexOf("package: stripe");
|
||||
assert.ok(awsIdx < stripeIdx, "aws-s3 should appear before stripe");
|
||||
});
|
||||
|
||||
test("renders all defined fields per entry", () => {
|
||||
const entries = [
|
||||
{
|
||||
package: "stripe",
|
||||
version: "^14.0.0",
|
||||
decision: "approved",
|
||||
"data-sent": "payment card details",
|
||||
region: "eu-west-1",
|
||||
"dpa-signed": true,
|
||||
"sccs-required": false,
|
||||
contact: "https://stripe.com/privacy",
|
||||
source: "library-trace",
|
||||
},
|
||||
];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
assert.ok(yaml.includes("package: stripe"));
|
||||
assert.ok(yaml.includes("version: ^14.0.0"));
|
||||
assert.ok(yaml.includes("decision: approved"));
|
||||
assert.ok(yaml.includes("data-sent: payment card details"));
|
||||
assert.ok(yaml.includes("region: eu-west-1"));
|
||||
assert.ok(yaml.includes("dpa-signed: true"));
|
||||
assert.ok(yaml.includes("sccs-required: false"));
|
||||
assert.ok(yaml.includes("contact:"));
|
||||
assert.ok(yaml.includes("source: library-trace"));
|
||||
});
|
||||
|
||||
test("omits undefined fields from entry", () => {
|
||||
const entries = [{ package: "stripe", source: "library-trace" }];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
assert.ok(
|
||||
!yaml.includes("data-sent:"),
|
||||
"should not render missing data-sent",
|
||||
);
|
||||
assert.ok(!yaml.includes("region:"), "should not render missing region");
|
||||
assert.ok(
|
||||
!yaml.includes("dpa-signed:"),
|
||||
"should not render missing dpa-signed",
|
||||
);
|
||||
});
|
||||
|
||||
test("output is deterministic across multiple calls", () => {
|
||||
const entries = [
|
||||
{ package: "stripe", source: "library-trace", decision: "approved" },
|
||||
];
|
||||
assert.equal(
|
||||
renderSubProcessorsYaml(entries),
|
||||
renderSubProcessorsYaml(entries),
|
||||
);
|
||||
});
|
||||
|
||||
test("package field is the first field in each list item (starts with '- package:')", () => {
|
||||
const entries = [{ package: "stripe", source: "library-trace" }];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
assert.ok(
|
||||
yaml.includes(" - package: stripe"),
|
||||
"entry should start with ' - package:'",
|
||||
);
|
||||
});
|
||||
|
||||
test("OUTPUT_PATH is compliance/sub-processors.yml", () => {
|
||||
assert.equal(OUTPUT_PATH, "compliance/sub-processors.yml");
|
||||
});
|
||||
});
|
||||
|
||||
describe("--check mode (integration)", () => {
|
||||
test("passes when committed file matches generated output", () => {
|
||||
const entries = [
|
||||
{ package: "stripe", source: "library-trace", decision: "approved" },
|
||||
];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
const diff = unifiedDiff(yaml, yaml, OUTPUT_PATH);
|
||||
assert.equal(diff, null, "no diff expected when files match");
|
||||
});
|
||||
|
||||
test("fails with readable diff when committed file is stale", () => {
|
||||
const entries = [
|
||||
{ package: "stripe", source: "library-trace", decision: "approved" },
|
||||
];
|
||||
const yaml = renderSubProcessorsYaml(entries);
|
||||
const stale = "# stale content\nsub-processors: []\n";
|
||||
const diff = unifiedDiff(stale, yaml, OUTPUT_PATH);
|
||||
assert.ok(diff !== null, "diff expected when file is stale");
|
||||
assert.ok(
|
||||
diff.includes(`--- ${OUTPUT_PATH}`),
|
||||
"diff should include filename header",
|
||||
);
|
||||
assert.ok(
|
||||
diff.includes("- # stale content"),
|
||||
"diff should show removed line",
|
||||
);
|
||||
assert.ok(diff.includes("Line"), "diff should include line numbers");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user