feat(scripts): add emit-data-map compliance script + tests
Adds scripts/compliance/emit-data-map.mjs which walks Payload collection configs (packages/*/integrations/cms/collections/*.ts), applies PAYLOAD_AUTH_PII_DEFAULTS + custom.authPii overrides, and emits a deterministic YAML PII inventory at compliance/data-map.yml. Supports --print (stdout) and --check (diff vs committed, exit 1 on mismatch) modes. Ships with 26 unit tests covering happy path, auth defaults, authPii overrides, --check match/mismatch, and empty collections. Wired as `compliance:data-map` root package script. Adds @typescript-eslint/parser to root devDependencies (already in workspace via core-eslint, now made explicit for scripts/ usage). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
412
scripts/compliance/emit-data-map.mjs
Normal file
412
scripts/compliance/emit-data-map.mjs
Normal file
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* emit-data-map.mjs — PII field inventory emitter.
|
||||
*
|
||||
* Walks packages/*\/src/integrations/cms/collections/*.ts, applies
|
||||
* PAYLOAD_AUTH_PII_DEFAULTS + custom.authPii overrides, and emits a
|
||||
* deterministic YAML PII inventory at compliance/data-map.yml.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/compliance/emit-data-map.mjs # write compliance/data-map.yml
|
||||
* node scripts/compliance/emit-data-map.mjs --print # write to stdout
|
||||
* node scripts/compliance/emit-data-map.mjs --check # diff vs committed file; exit 1 on mismatch
|
||||
*/
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { parse } from "@typescript-eslint/parser";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
export const REPO_ROOT = path.resolve(__dirname, "../..");
|
||||
export const OUTPUT_PATH = "compliance/data-map.yml";
|
||||
|
||||
// ---- Constants ----
|
||||
|
||||
/**
|
||||
* Default PII classification for Payload auth fields.
|
||||
* Null means the field is NOT PII and must be excluded from the data map.
|
||||
* Mirrors packages/core-shared/src/payload/pii-types.ts PAYLOAD_AUTH_PII_DEFAULTS.
|
||||
*/
|
||||
export const PAYLOAD_AUTH_PII_DEFAULTS = {
|
||||
email: {
|
||||
category: "contact-email",
|
||||
purpose: ["account-authentication", "transactional-notifications"],
|
||||
exportable: true,
|
||||
restrictable: true,
|
||||
},
|
||||
password: null,
|
||||
salt: null,
|
||||
hash: null,
|
||||
resetPasswordToken: null,
|
||||
resetPasswordExpiration: null,
|
||||
loginAttempts: null,
|
||||
lockUntil: null,
|
||||
apiKey: null,
|
||||
apiKeyIndex: null,
|
||||
};
|
||||
|
||||
// ---- AST parsing helpers ----
|
||||
|
||||
/**
|
||||
* Recursively extract a plain JS value from an AST node.
|
||||
* Returns undefined for unresolvable nodes (identifier references, etc.).
|
||||
*/
|
||||
function extractValue(node) {
|
||||
if (!node) return null;
|
||||
if (node.type === "Literal") return node.value;
|
||||
if (node.type === "TSAsExpression") return extractValue(node.expression);
|
||||
if (node.type === "Identifier") {
|
||||
if (node.name === "true") return true;
|
||||
if (node.name === "false") return false;
|
||||
if (node.name === "null") return null;
|
||||
return undefined;
|
||||
}
|
||||
if (node.type === "ObjectExpression") {
|
||||
const obj = {};
|
||||
for (const prop of node.properties) {
|
||||
if (prop.type !== "Property") continue;
|
||||
const key =
|
||||
prop.key.type === "Identifier"
|
||||
? prop.key.name
|
||||
: prop.key.type === "Literal"
|
||||
? String(prop.key.value)
|
||||
: null;
|
||||
if (!key) continue;
|
||||
const v = extractValue(prop.value);
|
||||
if (v !== undefined) obj[key] = v;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
if (node.type === "ArrayExpression") {
|
||||
return node.elements
|
||||
.map((e) => (e ? extractValue(e) : null))
|
||||
.filter((e) => e !== undefined);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a Payload collection TypeScript file.
|
||||
* Returns the first exported variable whose value is an object with a string
|
||||
* `slug` field, or null on parse failure / missing file.
|
||||
*/
|
||||
export function parseCollectionFile(filePath) {
|
||||
let src;
|
||||
try {
|
||||
src = fs.readFileSync(filePath, "utf8");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
let ast;
|
||||
try {
|
||||
ast = parse(src, {
|
||||
sourceType: "module",
|
||||
ecmaVersion: "latest",
|
||||
loc: false,
|
||||
range: false,
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
for (const node of ast.body) {
|
||||
if (node.type !== "ExportNamedDeclaration" || !node.declaration) continue;
|
||||
if (node.declaration.type !== "VariableDeclaration") continue;
|
||||
for (const decl of node.declaration.declarations) {
|
||||
if (!decl.init) continue;
|
||||
const value = extractValue(decl.init);
|
||||
if (
|
||||
value &&
|
||||
typeof value === "object" &&
|
||||
typeof value.slug === "string"
|
||||
) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ---- Collection file discovery ----
|
||||
|
||||
/**
|
||||
* Find all Payload collection TypeScript files across packages.
|
||||
*/
|
||||
export function findCollectionFiles(repoRoot = REPO_ROOT) {
|
||||
const packagesDir = path.join(repoRoot, "packages");
|
||||
if (!fs.existsSync(packagesDir)) return [];
|
||||
|
||||
const files = [];
|
||||
for (const pkg of fs.readdirSync(packagesDir).sort()) {
|
||||
const collectionsDir = path.join(
|
||||
packagesDir,
|
||||
pkg,
|
||||
"src",
|
||||
"integrations",
|
||||
"cms",
|
||||
"collections",
|
||||
);
|
||||
if (!fs.existsSync(collectionsDir)) continue;
|
||||
for (const file of fs.readdirSync(collectionsDir).sort()) {
|
||||
if (file.endsWith(".ts") && !file.endsWith(".test.ts")) {
|
||||
files.push(path.join(collectionsDir, file));
|
||||
}
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
// ---- Data map builder ----
|
||||
|
||||
/** Build a single PII entry. */
|
||||
function makePiiEntry(fieldName, pii, source) {
|
||||
return {
|
||||
field: fieldName,
|
||||
source,
|
||||
category: pii.category,
|
||||
purpose: Array.isArray(pii.purpose) ? [...pii.purpose] : [],
|
||||
exportable: pii.exportable ?? false,
|
||||
restrictable: pii.restrictable ?? false,
|
||||
...(pii.retention ? { retention: { ...pii.retention } } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Collect field-level PII entries from a collection's fields array. */
|
||||
function collectFieldPiiEntries(fields) {
|
||||
const entries = [];
|
||||
if (!Array.isArray(fields)) return entries;
|
||||
for (const field of fields) {
|
||||
if (!field || typeof field.name !== "string") continue;
|
||||
const pii = field.custom && field.custom.pii;
|
||||
if (!pii || typeof pii !== "object") continue;
|
||||
entries.push(makePiiEntry(field.name, pii, "field-tag"));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
/** Collect auth PII entries by merging defaults with per-collection overrides. */
|
||||
function collectAuthPiiEntries(authPiiDefaults, authPiiOverrides) {
|
||||
const merged = { ...authPiiDefaults, ...authPiiOverrides };
|
||||
const entries = [];
|
||||
for (const [fieldName, pii] of Object.entries(merged)) {
|
||||
if (pii === null) continue;
|
||||
const source =
|
||||
fieldName in authPiiOverrides ? "auth-override" : "auth-default";
|
||||
entries.push(makePiiEntry(fieldName, pii, source));
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk parsed collection configs and build the PII data map.
|
||||
* Applies authPiiDefaults (PAYLOAD_AUTH_PII_DEFAULTS) and custom.authPii
|
||||
* overrides for auth-enabled collections.
|
||||
*
|
||||
* Returns: Record<slug, { auth, piiFields, slug }>
|
||||
*/
|
||||
export function buildDataMap(
|
||||
rawCollections,
|
||||
authPiiDefaults = PAYLOAD_AUTH_PII_DEFAULTS,
|
||||
) {
|
||||
const map = {};
|
||||
for (const coll of rawCollections) {
|
||||
if (!coll || typeof coll.slug !== "string") continue;
|
||||
const isAuth = coll.auth === true;
|
||||
|
||||
const piiFields = [
|
||||
...collectFieldPiiEntries(coll.fields),
|
||||
...(isAuth
|
||||
? collectAuthPiiEntries(
|
||||
authPiiDefaults,
|
||||
(coll.custom && coll.custom.authPii) || {},
|
||||
)
|
||||
: []),
|
||||
];
|
||||
piiFields.sort((a, b) => a.field.localeCompare(b.field));
|
||||
|
||||
map[coll.slug] = { auth: isAuth, piiFields, slug: coll.slug };
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// ---- YAML serialization ----
|
||||
|
||||
const YAML_HEADER = [
|
||||
"# compliance/data-map.yml — PII field inventory",
|
||||
"# Generated by scripts/compliance/emit-data-map.mjs — do not edit manually.",
|
||||
"# Run `pnpm compliance:data-map` to regenerate.",
|
||||
].join("\n");
|
||||
|
||||
/** Quote a YAML string scalar only when necessary. */
|
||||
function yamlStr(s) {
|
||||
if (typeof s !== "string") return String(s);
|
||||
if (
|
||||
s === "" ||
|
||||
["true", "false", "null", "yes", "no", "on", "off"].includes(s) ||
|
||||
/[[{},:#&*!|>'"%@`\]]/u.test(s) ||
|
||||
/^\s|\s$/.test(s)
|
||||
) {
|
||||
return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
function renderPiiField(f) {
|
||||
let out = "";
|
||||
out += ` - category: ${yamlStr(f.category)}\n`;
|
||||
out += ` exportable: ${f.exportable}\n`;
|
||||
out += ` field: ${yamlStr(f.field)}\n`;
|
||||
|
||||
if (!f.purpose || f.purpose.length === 0) {
|
||||
out += ` purpose: []\n`;
|
||||
} else {
|
||||
out += ` purpose:\n`;
|
||||
for (const p of f.purpose) {
|
||||
out += ` - ${yamlStr(p)}\n`;
|
||||
}
|
||||
}
|
||||
|
||||
out += ` restrictable: ${f.restrictable}\n`;
|
||||
|
||||
if (f.retention) {
|
||||
out += ` retention:\n`;
|
||||
out += ` action: ${yamlStr(f.retention.action)}\n`;
|
||||
out += ` duration: ${yamlStr(f.retention.duration)}\n`;
|
||||
out += ` trigger: ${yamlStr(f.retention.trigger)}\n`;
|
||||
}
|
||||
|
||||
out += ` source: ${yamlStr(f.source)}\n`;
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the data map as deterministic YAML.
|
||||
* Collections sorted by slug; piiFields sorted by field name (done in buildDataMap).
|
||||
*/
|
||||
export function renderDataMapYaml(dataMap) {
|
||||
let yaml = YAML_HEADER + "\n";
|
||||
yaml += "collections:\n";
|
||||
|
||||
for (const slug of Object.keys(dataMap).sort()) {
|
||||
const coll = dataMap[slug];
|
||||
yaml += ` ${yamlStr(slug)}:\n`;
|
||||
yaml += ` auth: ${coll.auth}\n`;
|
||||
|
||||
if (!coll.piiFields || coll.piiFields.length === 0) {
|
||||
yaml += ` piiFields: []\n`;
|
||||
} else {
|
||||
yaml += ` piiFields:\n`;
|
||||
for (const f of coll.piiFields) {
|
||||
yaml += renderPiiField(f);
|
||||
}
|
||||
}
|
||||
|
||||
yaml += ` slug: ${yamlStr(slug)}\n`;
|
||||
}
|
||||
|
||||
return yaml;
|
||||
}
|
||||
|
||||
// ---- Diff helper for --check mode ----
|
||||
|
||||
/**
|
||||
* Produce a readable line-diff between expected (committed) and actual (generated).
|
||||
* Returns null if equal.
|
||||
*/
|
||||
export function unifiedDiff(expected, actual, filename) {
|
||||
const expLines = expected.split("\n");
|
||||
const actLines = actual.split("\n");
|
||||
const maxLen = Math.max(expLines.length, actLines.length);
|
||||
|
||||
const hunks = [];
|
||||
for (let i = 0; i < maxLen; i++) {
|
||||
const expLine = expLines[i] ?? "";
|
||||
const actLine = actLines[i] ?? "";
|
||||
if (expLine !== actLine) {
|
||||
hunks.push(`Line ${i + 1}:`);
|
||||
hunks.push(` - ${expLine}`);
|
||||
hunks.push(` + ${actLine}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (hunks.length === 0) return null;
|
||||
return (
|
||||
`--- ${filename} (committed)\n` +
|
||||
`+++ ${filename} (generated)\n` +
|
||||
hunks.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
// ---- CLI ----
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { mode: "write" }; // 'write' | 'print' | 'check'
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
if (argv[i] === "--print") out.mode = "print";
|
||||
else if (argv[i] === "--check") out.mode = "check";
|
||||
else if (argv[i] === "--help" || argv[i] === "-h") {
|
||||
console.log(
|
||||
[
|
||||
"Usage: node scripts/compliance/emit-data-map.mjs [--print | --check]",
|
||||
" (default): write compliance/data-map.yml",
|
||||
" --print: write YAML to stdout",
|
||||
" --check: diff vs committed file; exit 1 on mismatch",
|
||||
].join("\n"),
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv);
|
||||
const repoRoot = process.cwd();
|
||||
|
||||
const files = findCollectionFiles(repoRoot);
|
||||
const rawCollections = files
|
||||
.map((f) => parseCollectionFile(f))
|
||||
.filter(Boolean);
|
||||
const dataMap = buildDataMap(rawCollections);
|
||||
const yaml = renderDataMapYaml(dataMap);
|
||||
|
||||
const outPath = path.resolve(repoRoot, OUTPUT_PATH);
|
||||
|
||||
if (args.mode === "print") {
|
||||
process.stdout.write(yaml);
|
||||
return;
|
||||
}
|
||||
|
||||
if (args.mode === "check") {
|
||||
if (!fs.existsSync(outPath)) {
|
||||
process.stderr.write(
|
||||
`[compliance:data-map] --check: no committed file at ${OUTPUT_PATH}\n` +
|
||||
`Run \`pnpm compliance:data-map\` to generate it first.\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
const committed = fs.readFileSync(outPath, "utf8");
|
||||
const diff = unifiedDiff(committed, yaml, OUTPUT_PATH);
|
||||
if (diff === null) {
|
||||
console.log(`✓ compliance:data-map — ${OUTPUT_PATH} is up to date`);
|
||||
process.exit(0);
|
||||
}
|
||||
process.stderr.write(
|
||||
`✗ compliance:data-map — ${OUTPUT_PATH} is out of date:\n${diff}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Default: write to file
|
||||
const dir = path.dirname(outPath);
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
fs.writeFileSync(outPath, yaml, "utf8");
|
||||
console.log(`✓ compliance:data-map — wrote ${OUTPUT_PATH}`);
|
||||
}
|
||||
|
||||
if (import.meta.url === `file://${process.argv[1]}`) {
|
||||
main();
|
||||
}
|
||||
Reference in New Issue
Block a user