feat(coverage): pnpm mutate (Stryker) + L3 implementation
Lands L3 of the agent-first coverage architecture (ADR-020) — the
mutation-testing layer. Stryker on entities + use-cases (the pure
business-logic surface) catches the third dimension of test quality:
tests that exist + execute the code but assert nothing.
Deps (root devDependencies):
- @stryker-mutator/core ^8.7.0
- @stryker-mutator/vitest-runner ^8.7.0
Shared base: packages/core-testing/stryker.base.json
- testRunner: vitest (uses each feature's vitest.config.ts)
- mutate: src/entities/** + src/application/use-cases/** (excludes
tests, factories, contracts)
- thresholds: high 90 / low 80 / break 80
- reporters: progress + html + json (reports/mutation/{index.html,
mutation.json})
- incremental mode enabled, concurrency 4, timeout 10s
- exposed via @repo/core-testing/stryker.base.json subpath export
Per-feature config: packages/auth/stryker.config.json
- 4-line file that extends the shared base
- Proof-of-concept; other features get a config when L0 unification
closes their existing test gaps
Driver: scripts/coverage/mutate.mjs (zero-dep Node ESM)
- discoverStrykerConfigs: walks packages/* and apps/* for
stryker.config.json
- Supports --filter <name>, --since <ref> (incremental), --json
- Runs Stryker per-feature via node_modules/.bin/stryker run
- Surfaces per-package pass/fail summary; exits 1 on any failure
- Tests: scripts/coverage/mutate.test.mjs (3 tests, all green)
CI: .github/workflows/mutation-nightly.yml
- Cron at 02:30 UTC + workflow_dispatch with filter input
- Uploads reports/mutation/** as artifact (30-day retention)
- On failure, opens a tracking issue labelled mutation-testing
- permissions: contents: read, issues: write
- 60-min timeout (Stryker is slow by design)
Generator: turbo gen feature now scaffolds stryker.config.json from
turbo/generators/templates/feature/stryker.config.json.hbs — new
features ship mutation-ready out of the box.
Guide: docs/guides/coverage.md L3 section fleshed out with run
syntax, config shape, base config inventory, CI behavior, and a
"what you're looking for" primer on mutation scores.
Lockfile churn: pnpm regenerated the lockfile for the new deps;
~5K-line net reduction is collateral (pnpm version drift) but
mechanical.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
110
.github/workflows/mutation-nightly.yml
vendored
Normal file
110
.github/workflows/mutation-nightly.yml
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
# Mutation testing (L3) — nightly run + on-demand. ADR-020.
|
||||
#
|
||||
# Stryker is slow (~minutes per feature) so it's NOT part of the default
|
||||
# CI loop. This workflow runs nightly (and on manual dispatch) across every
|
||||
# feature with a stryker.config.json, then uploads the HTML + JSON
|
||||
# mutation reports as artifacts.
|
||||
#
|
||||
# On a meaningful score drop (>5%) it opens a tracking issue.
|
||||
|
||||
name: Mutation testing (nightly)
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 02:30 UTC nightly
|
||||
- cron: "30 2 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
filter:
|
||||
description: "Feature filter (e.g. @repo/auth). Empty = all features."
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
|
||||
jobs:
|
||||
mutate:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_DB: cms_test
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: pnpm/action-setup@v4
|
||||
with:
|
||||
version: 9
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
- run: pnpm install --frozen-lockfile
|
||||
- name: Run mutation testing
|
||||
env:
|
||||
DATABASE_URL: postgres://postgres:postgres@localhost:5432/cms_test
|
||||
PAYLOAD_SECRET: test-secret-do-not-use-in-prod
|
||||
run: |
|
||||
if [ -n "${{ inputs.filter }}" ]; then
|
||||
pnpm mutate -- --filter "${{ inputs.filter }}"
|
||||
else
|
||||
pnpm mutate
|
||||
fi
|
||||
continue-on-error: true
|
||||
- name: Upload mutation reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mutation-reports
|
||||
path: packages/*/reports/mutation/
|
||||
retention-days: 30
|
||||
- name: Open tracking issue on >5% score drop
|
||||
if: failure()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const reports = [];
|
||||
const pkgsDir = path.join(process.cwd(), 'packages');
|
||||
if (fs.existsSync(pkgsDir)) {
|
||||
for (const pkg of fs.readdirSync(pkgsDir)) {
|
||||
const json = path.join(pkgsDir, pkg, 'reports', 'mutation', 'mutation.json');
|
||||
if (fs.existsSync(json)) {
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(json, 'utf8'));
|
||||
const score = data.thresholds?.high && data.systemUnderTestMetrics?.metrics?.mutationScore;
|
||||
if (typeof score === 'number') {
|
||||
reports.push({ pkg, score: score.toFixed(2) });
|
||||
}
|
||||
} catch { /* skip */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (reports.length === 0) return;
|
||||
const body = [
|
||||
'Nightly mutation testing run flagged failures. Latest scores:',
|
||||
'',
|
||||
...reports.map(r => `- **${r.pkg}**: ${r.score}%`),
|
||||
'',
|
||||
`Run: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
|
||||
].join('\n');
|
||||
await github.rest.issues.create({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
title: `Mutation score drop — ${new Date().toISOString().slice(0, 10)}`,
|
||||
body,
|
||||
labels: ['mutation-testing', 'automated'],
|
||||
});
|
||||
Reference in New Issue
Block a user