From c56670ce190e1fe8d8c779c2a0ce811289b15e8f Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Thu, 23 Jul 2026 22:58:35 -0400 Subject: [PATCH 1/3] tools: draft RELEASE-NOTES.md from the release range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Automates the half of release notes that is a computer's job, and deliberately refuses the other half. WHAT IT FILLS IN (previously looked up by hand, every release): - the previous tag, the commit range, and the compare URL - commits grouped by conventional-commit type, with the PRs they came from - test coverage MEASURED by running the suites and reading each one's own count, rather than recalled — a stale count is a small lie nobody catches WHAT IT WILL NOT DO: write the prose. Choosing which 2 of 15 commits a user actually cares about, and how to frame them, is judgement. v0.9.2 needed the sentence "credits are a change of unit, not of price" — no commit subject contains that, and inventing it from subjects is how release notes become unreadable. The draft marks those spots TODO instead of guessing. IT SHOWS ITS WORKING. Every commit classified as internal is listed under "excluded" in the scaffolding block. A tool that silently drops commits is worse than no tool: you cannot review an omission you never see. Verified against the real v0.9.1..HEAD range — it excluded exactly what I had excluded by hand (the CI Playwright fix and the notes commits) and showed all three. Guardrails: rejects a missing/non-semver version, refuses a version whose tag already exists (notes are written BEFORE tagging so the tag contains them), warns on a dirty tree, and EXITS NON-ZERO if any suite fails — a red suite is a release blocker, not a footnote. Co-Authored-By: Claude Opus 4.8 --- scripts/draft-release-notes.mjs | 171 ++++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 scripts/draft-release-notes.mjs diff --git a/scripts/draft-release-notes.mjs b/scripts/draft-release-notes.mjs new file mode 100644 index 0000000..255ce0a --- /dev/null +++ b/scripts/draft-release-notes.mjs @@ -0,0 +1,171 @@ +/*--------------------------------------------------------------------------------------------- + * LevelCode — draft RELEASE-NOTES.md for a release. + * + * Usage: node scripts/draft-release-notes.mjs 0.9.2 [--write] + * (--write replaces RELEASE-NOTES.md; without it the draft goes to stdout) + * + * WHY THIS EXISTS + * + * Writing release notes has two halves, and only one of them is a computer's job. + * + * The FACTS are: which commits are in the range, which PRs they came from, what the previous tag + * was, how many test suites there are and how many cases each holds, and the compare URL. Every one + * of those was previously looked up by hand for each release, which is exactly the kind of thing + * that gets misremembered — a stale test count or a compare link pointing at the wrong tag is a + * small lie that nobody catches. + * + * The PROSE is: which two of fourteen commits actually matter to a user, what to lead with, and how + * to frame a change so it is not misread (v0.9.2 had to say "credits are a change of unit, not of + * price" — no commit subject contains that). This script does NOT attempt that, on purpose. A + * changelog auto-generated from commit subjects is the reason most release notes go unread. + * + * So: this fills in everything factual and leaves clearly-marked TODOs where judgement is required. + * + * IT ALSO SHOWS ITS WORKING. Every commit it classifies as internal is listed under "excluded" in + * the draft. A tool that silently drops commits is worse than no tool — you cannot review an + * omission you never see. Delete that section once you have checked it. + *--------------------------------------------------------------------------------------------*/ +import { execSync, spawnSync } from 'node:child_process'; +import { readdirSync, writeFileSync, existsSync } from 'node:fs'; +import { join } from 'node:path'; + +const REPO = process.cwd(); +const sh = (cmd) => execSync(cmd, { cwd: REPO, encoding: 'utf8' }).trim(); +const die = (msg) => { console.error('draft-release-notes: ' + msg); process.exit(1); }; + +// ---- arguments --------------------------------------------------------------------------------- + +const args = process.argv.slice(2); +const write = args.includes('--write'); +const version = args.find((a) => !a.startsWith('--')); +if (!version) { die('usage: node scripts/draft-release-notes.mjs [--write] e.g. 0.9.2'); } +if (!/^\d+\.\d+\.\d+$/.test(version)) { die(`"${version}" is not a bare semver (expected e.g. 0.9.2, no leading v)`); } + +const tag = 'v' + version; +if (sh('git tag --list ' + tag)) { + die(`${tag} already exists. Notes are written BEFORE tagging, so the tag contains them.`); +} + +// ---- the range --------------------------------------------------------------------------------- + +// Newest existing release tag, which is what this release is measured against. +const prevTag = sh("git tag --list 'v*' --sort=-v:refname").split('\n').filter(Boolean)[0]; +if (!prevTag) { die('no previous v* tag found — cannot compute a range or a compare link'); } + +const dirty = sh('git status --porcelain').split('\n').filter((l) => l && !l.startsWith('??')); +const warnings = []; +if (dirty.length) { + warnings.push(`working tree has ${dirty.length} uncommitted change(s) — the notes may describe code that is not in the tag`); +} + +// %x1f separates fields, %x1e separates records: commit subjects contain almost anything else. +const raw = sh(`git log --format=%H%x1f%s%x1f%an%x1e ${prevTag}..HEAD`); +const commits = raw.split('\x1e').map((r) => r.trim()).filter(Boolean).map((r) => { + const [hash, subject, author] = r.split('\x1f'); + return { hash: hash.slice(0, 7), subject, author }; +}); +if (!commits.length) { die(`no commits between ${prevTag} and HEAD — nothing to release`); } + +// ---- classification ---------------------------------------------------------------------------- +// +// Conventional-commit type decides the SECTION, not whether the change matters — that is your call. +// Merge commits are dropped (their PR title is already carried by the squashed/branch commits), but +// their PR numbers are collected so the draft can cite them. + +const prNumbers = []; +const isMerge = (c) => { + const m = /^Merge pull request #(\d+)/.exec(c.subject); + if (m) { prNumbers.push(m[1]); return true; } + return /^Merge branch /.test(c.subject); +}; + +const typeOf = (subject) => (/^(\w+)(\([^)]*\))?!?:/.exec(subject) || [])[1] || 'other'; +const USER_FACING = new Set(['feat', 'fix', 'perf', 'revert']); +const INTERNAL = new Set(['ci', 'build', 'chore', 'test', 'docs', 'refactor', 'style']); + +const kept = commits.filter((c) => !isMerge(c)); +const features = kept.filter((c) => typeOf(c.subject) === 'feat'); +const fixes = kept.filter((c) => ['fix', 'perf', 'revert'].includes(typeOf(c.subject))); +const excluded = kept.filter((c) => INTERNAL.has(typeOf(c.subject))); +const unclassified = kept.filter((c) => !USER_FACING.has(typeOf(c.subject)) && !INTERNAL.has(typeOf(c.subject))); + +// ---- test coverage, measured rather than recalled ----------------------------------------------- +// +// Runs the same suites the release gate runs and reads each one's own reported count. If a suite +// fails, that is a release blocker, not a footnote — say so loudly and exit non-zero. + +function measureSuites() { + const extRoot = join(REPO, 'extensions'); + if (!existsSync(extRoot)) { return { suites: [], failed: [] }; } + const suites = []; + const failed = []; + for (const ext of readdirSync(extRoot)) { + const testDir = join(extRoot, ext, 'test'); + if (!existsSync(testDir)) { continue; } + for (const file of readdirSync(testDir).filter((f) => f.endsWith('.test.js'))) { + const rel = join('extensions', ext, 'test', file); + const run = spawnSync('node', [rel], { cwd: REPO, encoding: 'utf8' }); + if (run.status !== 0) { failed.push(rel); continue; } + const m = /(\d+) tests? passed/.exec(run.stdout || ''); + suites.push({ file, cases: m ? Number(m[1]) : null }); + } + } + return { suites, failed }; +} + +const { suites, failed } = measureSuites(); +if (failed.length) { + die(`these suites FAIL — fix before drafting notes:\n ${failed.join('\n ')}`); +} +const totalCases = suites.reduce((n, s) => n + (s.cases || 0), 0); +const biggest = [...suites].sort((a, b) => (b.cases || 0) - (a.cases || 0)).slice(0, 3); + +// ---- render ------------------------------------------------------------------------------------- + +const bullet = (c) => `- \`${c.hash}\` ${c.subject}`; +const section = (title, list) => (list.length ? `\n### ${title}\n${list.map(bullet).join('\n')}\n` : ''); + +const draft = `# LevelCode v${version} + + + +## Highlights +${section('Candidates — feat (write these up, or move them down / delete)', features)}${section('Candidates — fix/perf (usually "Under the hood", unless a user hit the bug)', fixes)} + + +## Under the hood + + + +## Test coverage + +- **${suites.length} suites** across the bundled extensions, ${totalCases} cases in total — all green. +${biggest.map((s) => `- \`${s.file}\`${s.cases != null ? ` (${s.cases} cases)` : ''} — `).join('\n')} + +**Full changelog:** https://github.com/levelcodeai/levelcode/compare/${prevTag}...${tag} + + +`; + +if (write) { + writeFileSync(join(REPO, 'RELEASE-NOTES.md'), draft); + console.error(`draft-release-notes: wrote RELEASE-NOTES.md for ${tag} (${prevTag}..HEAD)`); + console.error(' Fill in the TODOs, delete the scaffolding block, then commit BEFORE tagging.'); + if (warnings.length) { warnings.forEach((w) => console.error(' WARNING: ' + w)); } +} else { + process.stdout.write(draft); +} From 791a52909a512ad4821540b698dade34b03fbe9e Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Thu, 23 Jul 2026 23:07:49 -0400 Subject: [PATCH 2/3] fix(tools): harden the release-notes drafter (PR #38 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All five points were real. Two of them made the tool quietly report LESS than the truth, which is the failure mode that matters most for something whose whole job is producing accurate facts. 1. prevTag was interpolated into a shell string. A tag name is repo-controlled but still runtime-discovered input, so this is now defended twice: all git calls go through spawnSync with an argv array (a shell cannot be quoted out of one), and the base tag must match ^vX.Y.Z$ before use. Shape matters independently of injection — a stray v0.9.2-rc1 sorts into the 'v*' glob and would silently produce the wrong range, wrong commit list and wrong compare link. Verified: v0.9.2-rc1, v-wip and "v0.9.2; rm -rf /" are all rejected. 2. PRs were only detected from merge commits, so on a squash-merging repo the list would read "(none detected)" while every subject carried "(#123)". Both shapes are now collected. 3. Suites were keyed by basename, so two extensions with the same test filename would produce an ambiguous "biggest suites" list. Keyed by relative path now. 4. The coverage line printed "N cases in total" while suites whose summary could not be parsed silently contributed 0 — authoritative-sounding and UNDER- reporting. This one bit immediately: 5 of 24 suites do not print a count, so "273 cases in total" was wrong. It now says how many suites were counted and names the ones that were not. 5. The fix/perf heading also carried reverts; renamed to say so. Also fixes a bug I introduced while making #2: prNumbers became a Set, but the renderer still read .length, which is undefined on a Set — so the PR list emptied itself silently. Caught by re-running against the real range (expected #33-36, got "(none detected)"). Materialised to a sorted array. Verified end to end against v0.9.1..HEAD: PRs #33-36 detected, coverage line now honest about the 5 unparsed suites, and all guardrails still fire. Co-Authored-By: Claude Opus 4.8 --- scripts/draft-release-notes.mjs | 62 +++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/scripts/draft-release-notes.mjs b/scripts/draft-release-notes.mjs index 255ce0a..87b37cc 100644 --- a/scripts/draft-release-notes.mjs +++ b/scripts/draft-release-notes.mjs @@ -25,14 +25,26 @@ * the draft. A tool that silently drops commits is worse than no tool — you cannot review an * omission you never see. Delete that section once you have checked it. *--------------------------------------------------------------------------------------------*/ -import { execSync, spawnSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import { readdirSync, writeFileSync, existsSync } from 'node:fs'; import { join } from 'node:path'; const REPO = process.cwd(); -const sh = (cmd) => execSync(cmd, { cwd: REPO, encoding: 'utf8' }).trim(); const die = (msg) => { console.error('draft-release-notes: ' + msg); process.exit(1); }; +// git via argv, never a shell string. A tag name is repo-controlled but still UNTRUSTED input here: +// it is discovered at runtime and then used to build a revision range, so passing it through a shell +// would make a tag containing metacharacters an injection. spawnSync with an array cannot be quoted +// out of. (RELEASE_TAG below is the second half of that defence: shape, not just escaping.) +const git = (...args) => { + const r = spawnSync('git', args, { cwd: REPO, encoding: 'utf8' }); + if (r.status !== 0) { die(`git ${args.join(' ')} failed: ${(r.stderr || '').trim()}`); } + return r.stdout.trim(); +}; + +// The ONLY tag shape this tool will measure a release against. +const RELEASE_TAG = /^v\d+\.\d+\.\d+$/; + // ---- arguments --------------------------------------------------------------------------------- const args = process.argv.slice(2); @@ -42,24 +54,27 @@ if (!version) { die('usage: node scripts/draft-release-notes.mjs [--wr if (!/^\d+\.\d+\.\d+$/.test(version)) { die(`"${version}" is not a bare semver (expected e.g. 0.9.2, no leading v)`); } const tag = 'v' + version; -if (sh('git tag --list ' + tag)) { +if (git('tag', '--list', tag)) { die(`${tag} already exists. Notes are written BEFORE tagging, so the tag contains them.`); } // ---- the range --------------------------------------------------------------------------------- -// Newest existing release tag, which is what this release is measured against. -const prevTag = sh("git tag --list 'v*' --sort=-v:refname").split('\n').filter(Boolean)[0]; -if (!prevTag) { die('no previous v* tag found — cannot compute a range or a compare link'); } +// Newest existing RELEASE tag — filtered by shape, not merely by the 'v*' glob. A stray tag like +// v0.9.2-rc1 or v-wip sorts into that glob and would silently produce the wrong range (and therefore +// the wrong compare link and the wrong commit list), which is a subtler failure than any injection. +const prevTag = git('tag', '--list', 'v*', '--sort=-v:refname') + .split('\n').map((t) => t.trim()).filter((t) => RELEASE_TAG.test(t))[0]; +if (!prevTag) { die('no previous vX.Y.Z release tag found — cannot compute a range or a compare link'); } -const dirty = sh('git status --porcelain').split('\n').filter((l) => l && !l.startsWith('??')); +const dirty = git('status', '--porcelain').split('\n').filter((l) => l && !l.startsWith('??')); const warnings = []; if (dirty.length) { warnings.push(`working tree has ${dirty.length} uncommitted change(s) — the notes may describe code that is not in the tag`); } // %x1f separates fields, %x1e separates records: commit subjects contain almost anything else. -const raw = sh(`git log --format=%H%x1f%s%x1f%an%x1e ${prevTag}..HEAD`); +const raw = git('log', '--format=%H%x1f%s%x1f%an%x1e', `${prevTag}..HEAD`); const commits = raw.split('\x1e').map((r) => r.trim()).filter(Boolean).map((r) => { const [hash, subject, author] = r.split('\x1f'); return { hash: hash.slice(0, 7), subject, author }; @@ -72,18 +87,27 @@ if (!commits.length) { die(`no commits between ${prevTag} and HEAD — nothing t // Merge commits are dropped (their PR title is already carried by the squashed/branch commits), but // their PR numbers are collected so the draft can cite them. -const prNumbers = []; +// PR numbers arrive in one of two shapes depending on the merge strategy, and a tool that only knows +// one of them silently reports "no PRs" on a repo that squash-merges. Collect both: +// merge commit -> "Merge pull request #37 from ..." +// squash commit -> "feat(ai): auto-open the browser (#35)" +const prNumbers = new Set(); const isMerge = (c) => { const m = /^Merge pull request #(\d+)/.exec(c.subject); - if (m) { prNumbers.push(m[1]); return true; } + if (m) { prNumbers.add(m[1]); return true; } return /^Merge branch /.test(c.subject); }; +const notePrInSubject = (c) => { + const m = /\(#(\d+)\)\s*$/.exec(c.subject); + if (m) { prNumbers.add(m[1]); } +}; const typeOf = (subject) => (/^(\w+)(\([^)]*\))?!?:/.exec(subject) || [])[1] || 'other'; const USER_FACING = new Set(['feat', 'fix', 'perf', 'revert']); const INTERNAL = new Set(['ci', 'build', 'chore', 'test', 'docs', 'refactor', 'style']); const kept = commits.filter((c) => !isMerge(c)); +kept.forEach(notePrInSubject); // squash-merge repos carry the PR in the subject, not a merge commit const features = kept.filter((c) => typeOf(c.subject) === 'feat'); const fixes = kept.filter((c) => ['fix', 'perf', 'revert'].includes(typeOf(c.subject))); const excluded = kept.filter((c) => INTERNAL.has(typeOf(c.subject))); @@ -107,7 +131,9 @@ function measureSuites() { const run = spawnSync('node', [rel], { cwd: REPO, encoding: 'utf8' }); if (run.status !== 0) { failed.push(rel); continue; } const m = /(\d+) tests? passed/.exec(run.stdout || ''); - suites.push({ file, cases: m ? Number(m[1]) : null }); + // Keyed by RELATIVE PATH: two extensions may legitimately both have catalog.test.js, and a + // basename would make the "biggest suites" list ambiguous about which one it means. + suites.push({ file: rel, cases: m ? Number(m[1]) : null }); } } return { suites, failed }; @@ -118,10 +144,17 @@ if (failed.length) { die(`these suites FAIL — fix before drafting notes:\n ${failed.join('\n ')}`); } const totalCases = suites.reduce((n, s) => n + (s.cases || 0), 0); +// A suite whose summary line we could not parse contributes 0, so the total would quietly UNDER-report +// coverage while sounding authoritative. Say which number we actually have. +const uncounted = suites.filter((s) => s.cases == null); const biggest = [...suites].sort((a, b) => (b.cases || 0) - (a.cases || 0)).slice(0, 3); // ---- render ------------------------------------------------------------------------------------- +// Set -> sorted array. (A Set has .size, not .length; reading .length silently yields undefined, +// which is how the PR list quietly emptied itself the first time.) +const prList = [...prNumbers].sort((a, b) => Number(a) - Number(b)); + const bullet = (c) => `- \`${c.hash}\` ${c.subject}`; const section = (title, list) => (list.length ? `\n### ${title}\n${list.map(bullet).join('\n')}\n` : ''); @@ -131,7 +164,7 @@ const draft = `# LevelCode v${version} not the biggest diff. Two features is a fine release; say so plainly. --> ## Highlights -${section('Candidates — feat (write these up, or move them down / delete)', features)}${section('Candidates — fix/perf (usually "Under the hood", unless a user hit the bug)', fixes)} +${section('Candidates — feat (write these up, or move them down / delete)', features)}${section('Candidates — fix/perf/revert (usually "Under the hood", unless a user hit the bug)', fixes)} @@ -142,7 +175,8 @@ ${section('Candidates — feat (write these up, or move them down / delete)', fe ## Test coverage -- **${suites.length} suites** across the bundled extensions, ${totalCases} cases in total — all green. +- **${suites.length} suites** across the bundled extensions — all green. +- ${totalCases} cases counted${uncounted.length ? ` across ${suites.length - uncounted.length} of them; ${uncounted.length} suite(s) did not report a count (${uncounted.map((s) => s.file).join(', ')}), so the real total is higher` : ' in total'}. ${biggest.map((s) => `- \`${s.file}\`${s.cases != null ? ` (${s.cases} cases)` : ''} — `).join('\n')} **Full changelog:** https://github.com/levelcodeai/levelcode/compare/${prevTag}...${tag} @@ -151,7 +185,7 @@ ${biggest.map((s) => `- \`${s.file}\`${s.cases != null ? ` (${s.cases} cases)` : EVERYTHING BELOW IS SCAFFOLDING — delete it before committing. Range: ${prevTag}..HEAD (${commits.length} commits, ${kept.length} after dropping merges) - PRs merged: ${prNumbers.length ? prNumbers.map((n) => '#' + n).join(', ') : '(none detected)'} + PRs merged: ${prList.length ? prList.map((n) => '#' + n).join(', ') : '(none detected)'} ${warnings.length ? '\n WARNINGS:\n' + warnings.map((w) => ' - ' + w).join('\n') + '\n' : ''} EXCLUDED as internal — check this list; anything user-visible in here belongs above: ${excluded.length ? excluded.map((c) => ` ${c.hash} ${c.subject}`).join('\n') : ' (none)'} From 351940388dd6ef01257d077f93ba360029855527 Mon Sep 17 00:00:00 2001 From: Sergii Demianchuk Date: Thu, 23 Jul 2026 23:19:16 -0400 Subject: [PATCH 3/3] fix(tools): anchor the drafter to the repo root, and never call an unmeasured tree green MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #38 review. Reproduced before fixing — run from scripts/ and the tool did not fail, it produced: - **0 suites** across the bundled extensions — all green. It measured nothing and certified it. --write would also have dropped RELEASE-NOTES.md into whatever directory you happened to be standing in. Fixed at the root: everything is anchored to `git rev-parse --show-toplevel` instead of process.cwd(), so the invocation directory stops mattering. Verified from scripts/ — now reports the real 24 suites / 273 cases. Also fixed the CLASS, not just the instance. Zero suites is "nothing was measured", not "all green", and the two must never render the same. A backstop now refuses to draft at all in that case. Anchoring should make it unreachable; it stays because the cost of being wrong is a release note certifying coverage nobody ran. Verified in a scratch repo with no extensions/: exits 1 with a plain explanation. Running outside a git repo exits 1 too, rather than silently treating some parent directory as the root. This is the fourth false-green in this codebase this week (tsc --noEmit checking zero files, pgrep matching the verifier itself, a status assertion that passed on the SPA fallback). The shape is always the same: a check that cannot fail reads exactly like a check that passed. Co-Authored-By: Claude Opus 4.8 --- scripts/draft-release-notes.mjs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/draft-release-notes.mjs b/scripts/draft-release-notes.mjs index 87b37cc..08ff701 100644 --- a/scripts/draft-release-notes.mjs +++ b/scripts/draft-release-notes.mjs @@ -29,9 +29,16 @@ import { spawnSync } from 'node:child_process'; import { readdirSync, writeFileSync, existsSync } from 'node:fs'; import { join } from 'node:path'; -const REPO = process.cwd(); const die = (msg) => { console.error('draft-release-notes: ' + msg); process.exit(1); }; +// Anchor EVERYTHING to the repo root, never to the invocation directory. Run from scripts/ with a cwd +// -based root and the tool does not fail — it reports "0 suites … all green", because extensions/ is +// simply not there to look in, and --write drops RELEASE-NOTES.md into the subdirectory. A wrong answer +// delivered confidently is the one failure mode this tool must not have. +const bootstrap = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: process.cwd(), encoding: 'utf8' }); +if (bootstrap.status !== 0) { die('not inside a git repository'); } +const REPO = bootstrap.stdout.trim(); + // git via argv, never a shell string. A tag name is repo-controlled but still UNTRUSTED input here: // it is discovered at runtime and then used to build a revision range, so passing it through a shell // would make a tag containing metacharacters an injection. spawnSync with an array cannot be quoted @@ -143,6 +150,12 @@ const { suites, failed } = measureSuites(); if (failed.length) { die(`these suites FAIL — fix before drafting notes:\n ${failed.join('\n ')}`); } +// Zero suites is not "all green", it is "nothing was measured" — and the two must never render the +// same. Anchoring REPO above should make this unreachable; it stays as the backstop, because the cost +// of being wrong here is a release note that certifies coverage nobody ran. +if (!suites.length) { + die(`found no test suites under ${join(REPO, 'extensions')} — refusing to draft notes that would claim "all green" without measuring anything`); +} const totalCases = suites.reduce((n, s) => n + (s.cases || 0), 0); // A suite whose summary line we could not parse contributes 0, so the total would quietly UNDER-report // coverage while sounding authoritative. Say which number we actually have.