From d415f4fa40bf5b730b7464e3eac20567be3fa7c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:46:40 +0000 Subject: [PATCH 1/3] chore(tooling): TEMPORARY runner-memory probe for the tsc heap ceiling (#14569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverted before this PR's final diff. `CI_TSC_HEAP_CEILING_MB` may only move on a reading taken where the verdict is taken -- the `Type Check · debt ledger` job on `ubuntu-latest` -- and this container cannot download job logs. Check-run ANNOTATIONS are readable over REST, so the probe emits its readings as `::notice` workflow commands from that job: - the runner's MemTotal/MemAvailable/Swap, image, nproc, and the gate process's own V8 `heap_size_limit` (the runner's default old space); - what else is resident at the point the re-measure starts (`ps` RSS census); - the `packages/qa/http-conformance` TEST_DEBT program -- the same generated project `measureTestDebt` writes -- run with `--extendedDiagnostics` under `--max-old-space-size=4096` and under `6144`, reporting tsc's own "Memory used", peak RSS, and the machine's minimum MemAvailable during each run. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WLJQhde67SeTccsmnBVarV --- .github/workflows/lint.yml | 6 ++ scripts/check-type-check-coverage.mjs | 124 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e463691b5e..54fe814816 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4737,6 +4737,12 @@ jobs: - name: Build the ledgered packages' dependencies run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' + # TEMPORARY measurement probe (#14569) — reverted before this PR's final + # diff. Prints this runner's memory readings as workflow-command + # annotations, which are readable through the check-run annotations API. + - name: Runner memory probe (TEMPORARY, #14569) + run: node scripts/check-type-check-coverage.mjs --runner-reading + - name: Re-measure the type-check DEBT / TEST_DEBT ledger run: pnpm check:type-check-debt diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 7a070e6fdd..02c3b4bca6 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -5591,6 +5591,130 @@ console.log( // clean: a ledger entry naming a package that no longer exists has nothing to // measure, and a wall of tsc output would bury the real failure. Reported after // the summary so the two verdicts read in the order they were reached. +// ── TEMPORARY runner-memory probe (#14569) ────────────────────────────────── +// Emits this job's memory readings as workflow-command annotations so they can +// be read back through the check-run annotations API from an agent container +// that cannot download job logs. ⛔ NOT part of the shipped change: this block +// and the workflow step that calls it are reverted before the PR's final diff. +if (process.argv.includes('--runner-reading')) { + const PROBE_PKG = '@objectstack/http-conformance'; + const notice = (title, body) => { + const text = String(body).trim(); + console.log(`::notice title=${title}::${text.replace(/\r?\n/g, '%0A').slice(0, 3800)}`); + console.log(`[probe] ${title}\n${text}\n`); + }; + const meminfoKb = (key) => { + const m = readFileSync('/proc/meminfo', 'utf8').match(new RegExp(`^${key}:\\s+(\\d+) kB`, 'm')); + return m === null ? null : Number(m[1]); + }; + const psSnapshot = () => { + const run = spawnSync('ps', ['-eo', 'rss=,comm='], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + const rows = String(run.stdout ?? '') + .trim() + .split('\n') + .map((l) => l.trim().split(/\s+/)) + .map(([rss, ...c]) => ({ rss: Number(rss), comm: c.join(' ') })) + .filter((r) => Number.isFinite(r.rss)); + rows.sort((a, b) => b.rss - a.rss); + return { total: rows.reduce((s, r) => s + r.rss, 0), rows: rows.slice(0, 12), count: rows.length }; + }; + const childHeapLimit = (mb) => { + const run = spawnSync(process.execPath, ['-e', 'console.log(require("node:v8").getHeapStatistics().heap_size_limit)'], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${mb}` }, + }); + return Math.floor(Number(String(run.stdout ?? '0').trim()) / (1024 * 1024)); + }; + + const snap = psSnapshot(); + notice('probe-runner-env', [ + `date=${new Date().toISOString()}`, + `runner os=${process.env.RUNNER_OS} arch=${process.env.RUNNER_ARCH} image=${process.env.ImageOS} ${process.env.ImageVersion}`, + `node=${process.version} nproc=${String(spawnSync('nproc', { encoding: 'utf8' }).stdout ?? '').trim()}`, + `MemTotal=${meminfoKb('MemTotal')} kB MemAvailable=${meminfoKb('MemAvailable')} kB SwapTotal=${meminfoKb('SwapTotal')} kB SwapFree=${meminfoKb('SwapFree')} kB`, + `gate process heap_size_limit=${Math.floor(getHeapStatistics().heap_size_limit / (1024 * 1024))} MB (runner V8 default)`, + `child heap_size_limit under --max-old-space-size=6144 = ${childHeapLimit(6144)} MB`, + `child heap_size_limit under --max-old-space-size=4096 = ${childHeapLimit(4096)} MB`, + `NODE_OPTIONS=${JSON.stringify(process.env.NODE_OPTIONS ?? '')}`, + `REMEASURE_HEAP=${JSON.stringify(REMEASURE_HEAP)}`, + ].join('\n')); + notice('probe-consumers', [ + `processes=${snap.count} total_rss=${snap.total} kB (${(snap.total / 1024).toFixed(0)} MB) at the point the re-measure starts`, + ...snap.rows.map((r) => `${String(r.rss).padStart(9)} kB ${r.comm}`), + ].join('\n')); + + // The generated TEST_DEBT program for the probe package -- the same project + // `measureTestDebt` writes, run directly so the heap cap can be varied. + const probePkg = packages.find((p) => p.name === PROBE_PKG); + const probeDir = probePkg.dir; + const probeRootAbs = ROOT.replaceAll('\\', '/').replace(/\/$/, ''); + const probePkgAbs = join(ROOT, probeDir).replaceAll('\\', '/').replace(/\/$/, ''); + const probeParsed = JSON.parse( + readFileSync(join(ROOT, probeDir, 'tsconfig.json'), 'utf8').replace(/^\s*\/\/.*$/gm, ''), + ); + const probeRoots = readTsconfig(probeDir, 'tsconfig.json').roots; + const probeHidden = probePkg.hiddenTests ?? []; + const probeUnreachable = probeHidden.filter( + (rel) => !probeRoots.some((r) => r === '' || rel === r || rel.startsWith(`${r}/`)), + ); + const probeProject = remeasureProject({ + pkgAbs: probePkgAbs, + rootAbs: probeRootAbs, + parsed: probeParsed, + unreachable: probeUnreachable, + chain: tsconfigChainFacts(probeDir), + }); + const probeHolder = mkdtempSync(join(tmpdir(), 'objectstack-probe-')); + const probeConfig = join(probeHolder, REMEASURE_CONFIG); + writeFileSync(probeConfig, `${JSON.stringify(probeProject, null, 2)}\n`); + const probeTsc = join(ROOT, 'node_modules', '.bin', 'tsc'); + const hasTime = existsSync('/usr/bin/time'); + + for (const capMb of [4096, 6144]) { + const out = join(probeHolder, `tsc-${capMb}.out`); + const err = join(probeHolder, `tsc-${capMb}.err`); + const sample = join(probeHolder, `mem-${capMb}.txt`); + const shell = [ + 'set -u', + `( while :; do awk '/^MemAvailable:/{print $2}' /proc/meminfo; sleep 0.5; done > ${sample} ) &`, + 'SAMPLER=$!', + `${hasTime ? '/usr/bin/time -v ' : ''}${probeTsc} --noEmit --pretty false --extendedDiagnostics -p ${probeConfig} > ${out} 2> ${err}`, + 'STATUS=$?', + 'kill "$SAMPLER" 2>/dev/null || true', + 'wait "$SAMPLER" 2>/dev/null || true', + 'echo "status=$STATUS"', + ].join('\n'); + const started = Date.now(); + const run = spawnSync('bash', ['-c', shell], { + cwd: ROOT, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${capMb}` }, + }); + const wall = ((Date.now() - started) / 1000).toFixed(1); + const stdout = existsSync(out) ? readFileSync(out, 'utf8') : ''; + const stderr = existsSync(err) ? readFileSync(err, 'utf8') : ''; + const pick = (re) => ((stdout.match(re) ?? stderr.match(re) ?? [null, null])[1]); + const samples = existsSync(sample) + ? readFileSync(sample, 'utf8').trim().split('\n').map(Number).filter(Number.isFinite) + : []; + const errs = stdout.match(/error TS\d+/g) ?? []; + notice(`probe-tsc-${capMb}`, [ + `cap=--max-old-space-size=${capMb} wall=${wall}s ${String(run.stdout ?? '').trim()}`, + `Files=${pick(/^Files:\s+(\d+)/m)} LinesOfDefinitions=${pick(/^Lines of Definitions:\s+(\d+)/m)}`, + `Types=${pick(/^Types:\s+(\d+)/m)} Instantiations=${pick(/^Instantiations:\s+(\d+)/m)}`, + `MemoryUsed=${pick(/^Memory used:\s+([\d,]+K)/m)}`, + `CheckTime=${pick(/^Check time:\s+([\d.]+)s/m)}s TotalTime=${pick(/^Total time:\s+([\d.]+)s/m)}s`, + `MaximumRSS=${pick(/Maximum resident set size \(kbytes\):\s+(\d+)/m)} kB`, + `MemAvailable during run: min=${samples.length ? Math.min(...samples) : 'n/a'} kB max=${samples.length ? Math.max(...samples) : 'n/a'} kB samples=${samples.length}`, + `heapOOM=${/JavaScript heap out of memory/.test(`${stdout}${stderr}`)} errCount=${errs.length}`, + `stderrTail=${stderr.trim().split('\n').slice(-3).join(' | ').slice(0, 400)}`, + ].join('\n')); + } + rmSync(probeHolder, { force: true, recursive: true }); + process.exit(0); +} + if (process.argv.includes('--re-measure')) { // The ceiling FIRST, before the four minutes of tsc it shapes (#12856). Two // jobs, and the second is the one that keeps the constant honest: on CI this From 994eb21ae52480fb04a718f7e639a36c8af97065 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:54:31 +0000 Subject: [PATCH 2/3] Revert "chore(tooling): TEMPORARY runner-memory probe for the tsc heap ceiling (#14569)" This reverts commit d415f4fa40bf5b730b7464e3eac20567be3fa7c9. --- .github/workflows/lint.yml | 6 -- scripts/check-type-check-coverage.mjs | 124 -------------------------- 2 files changed, 130 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 54fe814816..e463691b5e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4737,12 +4737,6 @@ jobs: - name: Build the ledgered packages' dependencies run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' - # TEMPORARY measurement probe (#14569) — reverted before this PR's final - # diff. Prints this runner's memory readings as workflow-command - # annotations, which are readable through the check-run annotations API. - - name: Runner memory probe (TEMPORARY, #14569) - run: node scripts/check-type-check-coverage.mjs --runner-reading - - name: Re-measure the type-check DEBT / TEST_DEBT ledger run: pnpm check:type-check-debt diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 02c3b4bca6..7a070e6fdd 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -5591,130 +5591,6 @@ console.log( // clean: a ledger entry naming a package that no longer exists has nothing to // measure, and a wall of tsc output would bury the real failure. Reported after // the summary so the two verdicts read in the order they were reached. -// ── TEMPORARY runner-memory probe (#14569) ────────────────────────────────── -// Emits this job's memory readings as workflow-command annotations so they can -// be read back through the check-run annotations API from an agent container -// that cannot download job logs. ⛔ NOT part of the shipped change: this block -// and the workflow step that calls it are reverted before the PR's final diff. -if (process.argv.includes('--runner-reading')) { - const PROBE_PKG = '@objectstack/http-conformance'; - const notice = (title, body) => { - const text = String(body).trim(); - console.log(`::notice title=${title}::${text.replace(/\r?\n/g, '%0A').slice(0, 3800)}`); - console.log(`[probe] ${title}\n${text}\n`); - }; - const meminfoKb = (key) => { - const m = readFileSync('/proc/meminfo', 'utf8').match(new RegExp(`^${key}:\\s+(\\d+) kB`, 'm')); - return m === null ? null : Number(m[1]); - }; - const psSnapshot = () => { - const run = spawnSync('ps', ['-eo', 'rss=,comm='], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); - const rows = String(run.stdout ?? '') - .trim() - .split('\n') - .map((l) => l.trim().split(/\s+/)) - .map(([rss, ...c]) => ({ rss: Number(rss), comm: c.join(' ') })) - .filter((r) => Number.isFinite(r.rss)); - rows.sort((a, b) => b.rss - a.rss); - return { total: rows.reduce((s, r) => s + r.rss, 0), rows: rows.slice(0, 12), count: rows.length }; - }; - const childHeapLimit = (mb) => { - const run = spawnSync(process.execPath, ['-e', 'console.log(require("node:v8").getHeapStatistics().heap_size_limit)'], { - encoding: 'utf8', - env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${mb}` }, - }); - return Math.floor(Number(String(run.stdout ?? '0').trim()) / (1024 * 1024)); - }; - - const snap = psSnapshot(); - notice('probe-runner-env', [ - `date=${new Date().toISOString()}`, - `runner os=${process.env.RUNNER_OS} arch=${process.env.RUNNER_ARCH} image=${process.env.ImageOS} ${process.env.ImageVersion}`, - `node=${process.version} nproc=${String(spawnSync('nproc', { encoding: 'utf8' }).stdout ?? '').trim()}`, - `MemTotal=${meminfoKb('MemTotal')} kB MemAvailable=${meminfoKb('MemAvailable')} kB SwapTotal=${meminfoKb('SwapTotal')} kB SwapFree=${meminfoKb('SwapFree')} kB`, - `gate process heap_size_limit=${Math.floor(getHeapStatistics().heap_size_limit / (1024 * 1024))} MB (runner V8 default)`, - `child heap_size_limit under --max-old-space-size=6144 = ${childHeapLimit(6144)} MB`, - `child heap_size_limit under --max-old-space-size=4096 = ${childHeapLimit(4096)} MB`, - `NODE_OPTIONS=${JSON.stringify(process.env.NODE_OPTIONS ?? '')}`, - `REMEASURE_HEAP=${JSON.stringify(REMEASURE_HEAP)}`, - ].join('\n')); - notice('probe-consumers', [ - `processes=${snap.count} total_rss=${snap.total} kB (${(snap.total / 1024).toFixed(0)} MB) at the point the re-measure starts`, - ...snap.rows.map((r) => `${String(r.rss).padStart(9)} kB ${r.comm}`), - ].join('\n')); - - // The generated TEST_DEBT program for the probe package -- the same project - // `measureTestDebt` writes, run directly so the heap cap can be varied. - const probePkg = packages.find((p) => p.name === PROBE_PKG); - const probeDir = probePkg.dir; - const probeRootAbs = ROOT.replaceAll('\\', '/').replace(/\/$/, ''); - const probePkgAbs = join(ROOT, probeDir).replaceAll('\\', '/').replace(/\/$/, ''); - const probeParsed = JSON.parse( - readFileSync(join(ROOT, probeDir, 'tsconfig.json'), 'utf8').replace(/^\s*\/\/.*$/gm, ''), - ); - const probeRoots = readTsconfig(probeDir, 'tsconfig.json').roots; - const probeHidden = probePkg.hiddenTests ?? []; - const probeUnreachable = probeHidden.filter( - (rel) => !probeRoots.some((r) => r === '' || rel === r || rel.startsWith(`${r}/`)), - ); - const probeProject = remeasureProject({ - pkgAbs: probePkgAbs, - rootAbs: probeRootAbs, - parsed: probeParsed, - unreachable: probeUnreachable, - chain: tsconfigChainFacts(probeDir), - }); - const probeHolder = mkdtempSync(join(tmpdir(), 'objectstack-probe-')); - const probeConfig = join(probeHolder, REMEASURE_CONFIG); - writeFileSync(probeConfig, `${JSON.stringify(probeProject, null, 2)}\n`); - const probeTsc = join(ROOT, 'node_modules', '.bin', 'tsc'); - const hasTime = existsSync('/usr/bin/time'); - - for (const capMb of [4096, 6144]) { - const out = join(probeHolder, `tsc-${capMb}.out`); - const err = join(probeHolder, `tsc-${capMb}.err`); - const sample = join(probeHolder, `mem-${capMb}.txt`); - const shell = [ - 'set -u', - `( while :; do awk '/^MemAvailable:/{print $2}' /proc/meminfo; sleep 0.5; done > ${sample} ) &`, - 'SAMPLER=$!', - `${hasTime ? '/usr/bin/time -v ' : ''}${probeTsc} --noEmit --pretty false --extendedDiagnostics -p ${probeConfig} > ${out} 2> ${err}`, - 'STATUS=$?', - 'kill "$SAMPLER" 2>/dev/null || true', - 'wait "$SAMPLER" 2>/dev/null || true', - 'echo "status=$STATUS"', - ].join('\n'); - const started = Date.now(); - const run = spawnSync('bash', ['-c', shell], { - cwd: ROOT, - encoding: 'utf8', - maxBuffer: 64 * 1024 * 1024, - env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${capMb}` }, - }); - const wall = ((Date.now() - started) / 1000).toFixed(1); - const stdout = existsSync(out) ? readFileSync(out, 'utf8') : ''; - const stderr = existsSync(err) ? readFileSync(err, 'utf8') : ''; - const pick = (re) => ((stdout.match(re) ?? stderr.match(re) ?? [null, null])[1]); - const samples = existsSync(sample) - ? readFileSync(sample, 'utf8').trim().split('\n').map(Number).filter(Number.isFinite) - : []; - const errs = stdout.match(/error TS\d+/g) ?? []; - notice(`probe-tsc-${capMb}`, [ - `cap=--max-old-space-size=${capMb} wall=${wall}s ${String(run.stdout ?? '').trim()}`, - `Files=${pick(/^Files:\s+(\d+)/m)} LinesOfDefinitions=${pick(/^Lines of Definitions:\s+(\d+)/m)}`, - `Types=${pick(/^Types:\s+(\d+)/m)} Instantiations=${pick(/^Instantiations:\s+(\d+)/m)}`, - `MemoryUsed=${pick(/^Memory used:\s+([\d,]+K)/m)}`, - `CheckTime=${pick(/^Check time:\s+([\d.]+)s/m)}s TotalTime=${pick(/^Total time:\s+([\d.]+)s/m)}s`, - `MaximumRSS=${pick(/Maximum resident set size \(kbytes\):\s+(\d+)/m)} kB`, - `MemAvailable during run: min=${samples.length ? Math.min(...samples) : 'n/a'} kB max=${samples.length ? Math.max(...samples) : 'n/a'} kB samples=${samples.length}`, - `heapOOM=${/JavaScript heap out of memory/.test(`${stdout}${stderr}`)} errCount=${errs.length}`, - `stderrTail=${stderr.trim().split('\n').slice(-3).join(' | ').slice(0, 400)}`, - ].join('\n')); - } - rmSync(probeHolder, { force: true, recursive: true }); - process.exit(0); -} - if (process.argv.includes('--re-measure')) { // The ceiling FIRST, before the four minutes of tsc it shapes (#12856). Two // jobs, and the second is the one that keeps the constant honest: on CI this From bfcc67b2f071ceb2e78b33db2e6983bf5e0ce457 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:55:56 +0000 Subject: [PATCH 3/3] docs(tooling): record the runner measurement beside CI_TSC_HEAP_CEILING_MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin's provenance was archaeology through a failed job's GC trace, which bracketed the runner's old space into [4040, 4148] MB. It is now a first-hand reading, taken where the verdict is taken -- inside the `Type Check · debt ledger` job, by a temporary probe step (reverted in the previous commit) that emitted its numbers as `::notice` annotations: runner ubuntu24 20260831.293.1, 4 vCPU, MemTotal 16,373,452 kB (~15.6 GiB) -- not the 7 GB the finding assumed consumers 153 processes / 940,316 kB (~918 MB); the job's steps are sequential, so nothing runs beside the re-measure this gate heap_size_limit 4144 MB with NODE_OPTIONS unset -- the runner's V8 default, confirming the 4096 MB old space directly heaviest qa/http-conformance's TEST_DEBT program under two caps: program 4096 -> 4,077,718K used, 4,212,904 kB peak RSS, 26.84s check 6144 -> 4,420,706K used, 4,545,500 kB peak RSS, 21.90s check The pair is the headroom reading the finding asked for: 343 MB more heap keeps 343 MB more live and finishes ~5s sooner, so under 4096 the program is paying GC pressure to fit. The constant does NOT move on it, and the measurement is why: the scarce resource is V8's default old space (4096 MB), not the runner's memory, and this number describes that default exactly. The comment also records what the measurement made mechanically visible -- raising the pin alone cannot deliver a roomier run. `remeasureHeapCeiling` minimises over the pin and the running process's own limit, so a 6144 pin under the runner's default still chooses 4144, and the `stale` arm then refuses the run outright: `--re-measure` exits 1 before the first tsc. Reproduced against a 4144 MB process. Delivering a raise needs the gate PROCESS given the memory first, which is a workflow decision and is escalated on #14569. The self-test row for "a box shaped like CI" gains a note that its `+ 48` is now the measured runner rather than a construction. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WLJQhde67SeTccsmnBVarV --- scripts/check-type-check-coverage.mjs | 63 ++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 7a070e6fdd..2c8e511984 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -2799,10 +2799,54 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // with `NODE_OPTIONS` on a box under this file's own eyes), so a 4096 old space // reports 4144 and commits the 4147.5 above it. // +// ## Re-measured FIRST-HAND on the runner, 2026-09-03 (#14569) +// +// The bracket above is archaeology through a failed job's GC trace. #14569 +// asked for a raise to 6144 to be taken on a measurement rather than on a +// typed number, so the reading was taken where the verdict is taken: inside +// the `Type Check · debt ledger` job itself, by a temporary probe step that +// emitted its numbers as `::notice` annotations (run 33708954003, job +// 100504131338, image `ubuntu24 20260831.293.1`, Node v22.23.2, 4 vCPU). +// +// the runner MemTotal 16,373,452 kB (~15.6 GiB) plus 3,145,724 kB of +// swap -- NOT the 7 GB #14569 assumed. MemAvailable at the +// point the re-measure starts: 14,329,064 kB. +// other consumers 153 processes holding 940,316 kB (~918 MB) altogether: +// Runner.Worker 144 MB, Runner.Listener 98 MB, provjobd +// 96 MB, dockerd 73 MB, containerd 43 MB. The job's steps +// are sequential, so nothing in it runs BESIDE the +// re-measure -- the ledger's tsc has the box to itself. +// this gate's own `heap_size_limit` 4144 MB with `NODE_OPTIONS` unset -- +// ceiling the runner's V8 default, read directly rather than +// inferred. It confirms the 4096 MB old space the GC trace +// above could only bracket. +// the heaviest `packages/qa/http-conformance`'s TEST_DEBT program (906 +// program files, 692,003 lines of definitions, 7,328,937 +// instantiations) under `--extendedDiagnostics`, twice: +// +// cap 4096 Memory used 4,077,718K peak RSS 4,212,904 kB +// check 26.84s +// cap 6144 Memory used 4,420,706K peak RSS 4,545,500 kB +// check 21.90s +// +// Neither OOMs, and the pair IS the headroom finding +// #14569 asked for: handed 343 MB more heap the same +// program keeps 343 MB more live and finishes ~5s sooner, +// so under 4096 it is paying GC pressure to fit rather +// than fitting. Lowest MemAvailable seen at any point +// during either run: 10,562,192 kB. +// +// The scarce resource is therefore NOT the runner's memory -- 15.6 GiB with +// ~918 MB of it spoken for -- but V8's DEFAULT old space on that runner, which +// is 4096 MB. This constant describes that default, and as of 2026-09-03 it +// still describes it exactly. That is why the re-measure leaves it here. +// // ⚠️ If 4096 is wrong, it is wrong DOWNWARD -- the only safe direction. This // number's entire job is to be no HIGHER than CI's ceiling. A pin ABOVE CI's is // worse than no pin at all: it makes local runs pass where CI still OOMs, which -// is exactly this defect with extra confidence attached. +// is exactly this defect with extra confidence attached. The 2026-09-03 +// reading above is the first taken with the runner in hand rather than +// inferred from a crash, and it lands on the same 4096 from the other side. // // ⛔ Do not raise this to make a local measurement complete. `--re-measure` // OOMing under this ceiling is the gate WORKING -- it is CI's failure, @@ -2811,6 +2855,18 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // lesson from the build side: a ceiling above the box's real memory does not // buy a bigger run, it converts a recoverable heap error into an exit-137 // SIGKILL that carries no diagnostic at all.) +// +// ⛔ And raising it ALONE cannot buy the ledger a roomier run -- measured on +// 2026-09-03, not reasoned. `remeasureHeapCeiling` below takes the MINIMUM of +// this pin and the limit the running process actually has, so with the pin at +// 6144 and the gate started under the runner's own default the chosen ceiling +// is still 4144 -- and the `stale` arm below then refuses the run outright: +// `--re-measure` exits 1 before the first tsc ("the pin is now ABOVE the +// ceiling it claims to describe"), reproduced against a 4144 MB process. A +// raise has to hand the gate PROCESS the memory first -- a `NODE_OPTIONS` on +// the job's re-measure step -- so the pin keeps describing what the process +// really has. That is a workflow decision, not one this constant can take on +// its own; #14569 carries it. const CI_TSC_HEAP_CEILING_MB = 4096; /** @@ -5156,6 +5212,11 @@ function selfTest() { expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, }, { + // `+ 48` is the RUNNER, not a construction: the `Type Check · debt + // ledger` job reports a `heap_size_limit` of 4144 MB for its 4096 MB old + // space (measured there 2026-09-03, #14569), so this row is the shape of + // the machine whose verdict the pin exists to describe -- and the row + // above it is every box that is roomier than that one. label: 'on a box shaped like CI the ceiling is a no-op that still names itself', where: { heapLimitMb: CI_TSC_HEAP_CEILING_MB + 48, onCi: true }, expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false },