Lands L3 of the agent-first coverage architecture (ADR-020) — the
mutation-testing layer. Stryker on entities + use-cases (the pure
business-logic surface) catches the third dimension of test quality:
tests that exist + execute the code but assert nothing.
Deps (root devDependencies):
- @stryker-mutator/core ^8.7.0
- @stryker-mutator/vitest-runner ^8.7.0
Shared base: packages/core-testing/stryker.base.json
- testRunner: vitest (uses each feature's vitest.config.ts)
- mutate: src/entities/** + src/application/use-cases/** (excludes
tests, factories, contracts)
- thresholds: high 90 / low 80 / break 80
- reporters: progress + html + json (reports/mutation/{index.html,
mutation.json})
- incremental mode enabled, concurrency 4, timeout 10s
- exposed via @repo/core-testing/stryker.base.json subpath export
Per-feature config: packages/auth/stryker.config.json
- 4-line file that extends the shared base
- Proof-of-concept; other features get a config when L0 unification
closes their existing test gaps
Driver: scripts/coverage/mutate.mjs (zero-dep Node ESM)
- discoverStrykerConfigs: walks packages/* and apps/* for
stryker.config.json
- Supports --filter <name>, --since <ref> (incremental), --json
- Runs Stryker per-feature via node_modules/.bin/stryker run
- Surfaces per-package pass/fail summary; exits 1 on any failure
- Tests: scripts/coverage/mutate.test.mjs (3 tests, all green)
CI: .github/workflows/mutation-nightly.yml
- Cron at 02:30 UTC + workflow_dispatch with filter input
- Uploads reports/mutation/** as artifact (30-day retention)
- On failure, opens a tracking issue labelled mutation-testing
- permissions: contents: read, issues: write
- 60-min timeout (Stryker is slow by design)
Generator: turbo gen feature now scaffolds stryker.config.json from
turbo/generators/templates/feature/stryker.config.json.hbs — new
features ship mutation-ready out of the box.
Guide: docs/guides/coverage.md L3 section fleshed out with run
syntax, config shape, base config inventory, CI behavior, and a
"what you're looking for" primer on mutation scores.
Lockfile churn: pnpm regenerated the lockfile for the new deps;
~5K-line net reduction is collateral (pnpm version drift) but
mechanical.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
111 lines
3.6 KiB
YAML
111 lines
3.6 KiB
YAML
# Mutation testing (L3) — nightly run + on-demand. ADR-020.
|
|
#
|
|
# Stryker is slow (~minutes per feature) so it's NOT part of the default
|
|
# CI loop. This workflow runs nightly (and on manual dispatch) across every
|
|
# feature with a stryker.config.json, then uploads the HTML + JSON
|
|
# mutation reports as artifacts.
|
|
#
|
|
# On a meaningful score drop (>5%) it opens a tracking issue.
|
|
|
|
name: Mutation testing (nightly)
|
|
|
|
on:
|
|
schedule:
|
|
# 02:30 UTC nightly
|
|
- cron: "30 2 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
filter:
|
|
description: "Feature filter (e.g. @repo/auth). Empty = all features."
|
|
required: false
|
|
default: ""
|
|
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
|
|
jobs:
|
|
mutate:
|
|
runs-on: ubuntu-latest
|
|
services:
|
|
postgres:
|
|
image: postgres:16-alpine
|
|
env:
|
|
POSTGRES_PASSWORD: postgres
|
|
POSTGRES_USER: postgres
|
|
POSTGRES_DB: cms_test
|
|
ports:
|
|
- 5432:5432
|
|
options: >-
|
|
--health-cmd "pg_isready -U postgres"
|
|
--health-interval 10s
|
|
--health-timeout 5s
|
|
--health-retries 5
|
|
timeout-minutes: 60
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- uses: pnpm/action-setup@v4
|
|
with:
|
|
version: 9
|
|
- uses: actions/setup-node@v4
|
|
with:
|
|
node-version: 22
|
|
cache: pnpm
|
|
- run: pnpm install --frozen-lockfile
|
|
- name: Run mutation testing
|
|
env:
|
|
DATABASE_URL: postgres://postgres:postgres@localhost:5432/cms_test
|
|
PAYLOAD_SECRET: test-secret-do-not-use-in-prod
|
|
run: |
|
|
if [ -n "${{ inputs.filter }}" ]; then
|
|
pnpm mutate -- --filter "${{ inputs.filter }}"
|
|
else
|
|
pnpm mutate
|
|
fi
|
|
continue-on-error: true
|
|
- name: Upload mutation reports
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: mutation-reports
|
|
path: packages/*/reports/mutation/
|
|
retention-days: 30
|
|
- name: Open tracking issue on >5% score drop
|
|
if: failure()
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const reports = [];
|
|
const pkgsDir = path.join(process.cwd(), 'packages');
|
|
if (fs.existsSync(pkgsDir)) {
|
|
for (const pkg of fs.readdirSync(pkgsDir)) {
|
|
const json = path.join(pkgsDir, pkg, 'reports', 'mutation', 'mutation.json');
|
|
if (fs.existsSync(json)) {
|
|
try {
|
|
const data = JSON.parse(fs.readFileSync(json, 'utf8'));
|
|
const score = data.thresholds?.high && data.systemUnderTestMetrics?.metrics?.mutationScore;
|
|
if (typeof score === 'number') {
|
|
reports.push({ pkg, score: score.toFixed(2) });
|
|
}
|
|
} catch { /* skip */ }
|
|
}
|
|
}
|
|
}
|
|
if (reports.length === 0) return;
|
|
const body = [
|
|
'Nightly mutation testing run flagged failures. Latest scores:',
|
|
'',
|
|
...reports.map(r => `- **${r.pkg}**: ${r.score}%`),
|
|
'',
|
|
`Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
|
|
].join('\n');
|
|
await github.rest.issues.create({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
title: `Mutation score drop — ${new Date().toISOString().slice(0, 10)}`,
|
|
body,
|
|
labels: ['mutation-testing', 'automated'],
|
|
});
|