From 7a0fb255dd951b7696a2f5dbb411f3588788bc7f Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 22 Aug 2026 00:16:33 +0900 Subject: [PATCH 1/2] =?UTF-8?q?devlog:=20bun=201.4=20follow-up=20memory=20?= =?UTF-8?q?roadmap=20(000-040)=20=E2=80=94=20research=20ledger,=20diagnost?= =?UTF-8?q?ics/GC-relief/smol-worker=20plans,=20macmini=20measurement=20pr?= =?UTF-8?q?otocol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../000_plan.md | 18 ++++ .../000_research.md | 71 ++++++++++++ .../010_memory_diagnostics.md | 74 +++++++++++++ .../020_watchdog_gc_relief.md | 101 ++++++++++++++++++ .../030_smol_workers.md | 45 ++++++++ .../040_macmini_measurement.md | 28 +++++ 6 files changed, 337 insertions(+) create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/000_plan.md create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/000_research.md create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/010_memory_diagnostics.md create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/020_watchdog_gc_relief.md create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md create mode 100644 devlog/_plan/260822_260822-bun14-followup-memory/040_macmini_measurement.md diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/000_plan.md b/devlog/_plan/260822_260822-bun14-followup-memory/000_plan.md new file mode 100644 index 0000000000..ddc2d178b1 --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/000_plan.md @@ -0,0 +1,18 @@ +# 000_plan — unit map + +- 000_research.md — claim ledger + gap analysis +- 010_memory_diagnostics.md — extraMemorySize observability (PR parent, base dev) +- 020_watchdog_gc_relief.md — measurement-FIRST GC evaluation (Phase A harness), + conditional idle-gated production hook (Phase B) per the 260731 gate +- 030_smol_workers.md — smol:true gated on per-worker large-fixture A/B +- 040_macmini_measurement.md — live measurement protocol (feeds 020 Phase A) + +Stack shape: PR-A(010, base dev) → PR-B(020 Phase A harness + evaluation, +base PR-A head) → conditional PR for Phase B only on gate PASS; +PR-C(030, base dev, lands per-call-site with A/B evidence). +One decade doc = one work-phase = one PABCD cycle (LOOP-UNIT-CHAIN-01). +Audit round 1: FAIL (4 findings) → docs revised: 020 restructured +measurement-first honoring 260731_macos_rss_retention/040_allocator_residual +gate; 010 static-import sync seam; 030 pre-landing A/B gate; separate +lastReliefAt. See ledger. + diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/000_research.md b/devlog/_plan/260822_260822-bun14-followup-memory/000_research.md new file mode 100644 index 0000000000..1e4462c5c7 --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/000_research.md @@ -0,0 +1,71 @@ +# 000 — Bun 1.4 follow-up memory patches: research and claim ledger + +Date: 2026-08-22 +Unit: 260822_260822-bun14-followup-memory +Question: from today's viewpoint (bundled Bun 1.4.0, released 2026-08-19), which +ADDITIONAL memory patches are possible and worthwhile in opencodex? + +## Method + +Luna 5-lane discovery swarm (official releases / GitHub issues+PRs / JSC-runtime / +community / server-SSE-proxy), then Tier-2 proof by the main agent via `gh api` +against oven-sh/bun. App-side baseline re-audited against +devlog/_fin/260813_bun_canary_dogfood/050_memory_patch_roadmap.md and current src/. + +## Claim ledger (Tier-2 proven unless noted) + +| # | Claim | Proof | Status | +|---|---|---|---| +| C1 | No Bun 1.4.x patch release exists after v1.4.0 (2026-08-19). | `gh api repos/oven-sh/bun/releases` → latest tag `bun-v1.4.0`; bun-v1.4.1/2/3 404. | verified | +| C2 | Bun PR #36467 (TLS Bun.serve use-after-free on `server.stop(true)` sibling-socket close) merged 2026-07-31, sha 529adec09, and IS an ancestor of bun-v1.4.0 (`compare/bun-v1.4.0...sha` → status=behind). Already in our bundled runtime; no action. | gh api pulls/36467 + compare | verified | +| C3 | Bun PR #32662 (fetch: release buffered response body + error reader on streaming abort) merged 2026-07-22, sha 4b7241669, ancestor of v1.4.0. In bundled runtime. | gh api pulls/32662 + compare | verified | +| C4 | Bun PR #35093 (fetch: error body stream when fully-buffered response aborted) merged 2026-07-28, sha 789be97db, ancestor of v1.4.0. In bundled runtime. | gh api pulls/35093 + compare | verified | +| C5 | Bun issue #34917: `--max-old-space-size`, `BUN_JSC_gcMaxHeapSize`, `BUN_JSC_forceRAMSize` are NOT reliable heap caps on the 1.4 line; still OPEN (created 2026-07-21, closed:null). Container/OOM bounding must come from app-side watchdog + supervision, not JSC flags. | gh api issues/34917 | verified | +| C6 | `Bun.gc(true)` on 1.4 asks JSC to collect AND asks mimalloc to release fragmented non-JS pages (allocator shared with JSC since the 1.4 Rust/allocator work). | Bun docs (bun.com/reference/bun/gc) opened by L3; local probe `typeof Bun.gc === "function"` on 1.4.0. | verified (docs) | +| C7 | `bun:jsc` heapStats exposes `extraMemorySize`/`heapCapacity`; `Bun.unsafe.mimallocDump` exists on 1.4.0. | local probe on bundled 1.4.0: `{"heapSize":…,"heapCapacity":…,"extraMemorySize":…}`, mimallocDump:function | verified (executed) | +| C8 | `new Worker(url, {smol:true})` works on bundled 1.4.0 (selects JSC Small heap growth policy per Bun docs). | local probe: "smol worker OK" | verified (executed) | +| C9 | RSS retention after GC (issue #27514) and SSE-proxy reader-cancel segfault (#31159) were closed as DUPLICATES, not demonstrated fixed; #26321 (Windows file-stream RSS) duplicate-closed too. Continued A/B measurement remains necessary. | gh issue pages opened by L2/L5 | verified | +| C10 | Community: Bun 1.4 advertises up to ~35% memory reduction (allocator rewrite, thread-local page purging, lazy zeroing); no long-running independent RSS measurements yet. | Reddit announcements (L4), snippet-grade | lead | +| C11 | Medium post claims 1.4-era HTTP long-connection RSS still grew 280→340MB over 7 days; page returned 403. | unreachable | candidate — unverified | + +## App-side baseline (what is already done — do not re-patch) + +- 260813 roadmap patches #1–#4 ALL landed since: native-main hardened-identity LRU + (src/codex/native-main-claim.ts:25-33), installation-salt LRU + (src/lab/subject/installation-salt.ts:7-17), mode-hint capability LRU + (src/codex/features.ts:1097-1106), Lab ledger event-id process index REMOVED + (no `eventIdIndexByLedger` in src/lab/ledger/store.ts). +- `Bun.serve({ idleTimeout: 255 })` (src/server/index.ts:736) and per-request + `server.timeout(req, 0)` for streaming (src/server/responses/fetch-helpers.ts:113) + already implement the SSE-timeout guidance the swarm surfaced. +- eager-relay vs legacy-tee runtime gate: src/lib/bun-stream-caps.ts + (MIN_FIXED_BUN_VERSION="1.4.0"). +- Memory watchdog: warn-only ring sampler (src/server/memory-watchdog.ts), exposed at + /api/system/memory with bun:jsc heapSize/heapCapacity/objectCount. +- 36-store bounded-memory audit closed (devlog/_fin/260813…/050): only remaining + investigation is model-cache generation tombstones — needs an authority-token + redesign, NOT an eviction patch; excluded from this unit. + +## Gap analysis → patch set for THIS unit + +What Bun 1.4 newly makes possible, that opencodex does not use yet: + +1. **Diagnostics gap** — /api/system/memory and the watchdog ignore + `extraMemorySize` (JSC-visible native memory) and the watchdog samples carry no + JSC data at all. On 1.4, extraMemorySize is the counter that moved most + (external-memory reporting fixes #31422/#32653/#34142). → doc 010. +2. **Reclaim gap** — nothing in the tree ever calls `Bun.gc`. On 1.4 a full + `Bun.gc(true)` also purges mimalloc pages (C6) — the exact mitigation for the + "heap shrinks, RSS stays" pattern (#27514) that JSC flags cannot deliver (C5). + A config-gated, rate-limited watchdog relief hook is now worth having. → doc 020. +3. **Worker heap gap** — history/restore/policy workers are short-lived batch jobs; + `smol: true` (C8) bounds their JSC heap growth policy at a small perf cost, + reducing peak RSS during storage jobs. → doc 030. +4. **Proof gap** — every claim above is config/diagnostic-grade until measured. + macmini-cf (arm64, bun 1.3.14 installed → good A/B host) runs the live + measurement protocol. → doc 040. + +Explicit non-goals: no Bun runtime patching/fork (upstream 1.4.0 already carries +C2–C4); no JSC env-var "caps" (C5 proves them unreliable); no smol for the main +proxy process (throughput cost, unmeasured); no model-cache tombstone work. + diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/010_memory_diagnostics.md b/devlog/_plan/260822_260822-bun14-followup-memory/010_memory_diagnostics.md new file mode 100644 index 0000000000..c7eeb2cd49 --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/010_memory_diagnostics.md @@ -0,0 +1,74 @@ +# 010 — Memory diagnostics: extraMemorySize in samples and API + +Depends on: 000. Standalone PR (parent of the stack, targets dev). + +## Why + +Bun 1.4's biggest memory changes are external-memory reporting fixes +(#31422/#32653/#34142 per 260813 canary table). The counter that reflects them is +`heapStats().extraMemorySize` — JSC-visible native memory. Today +/api/system/memory reports jscHeap {heapSize, heapCapacity, objectCount} but NOT +extraMemorySize, and watchdog samples carry no JSC counters at all, so the exact +signal 1.4 improved is invisible in our 6h ring. + +## Changes + +### src/server/management/system-routes.ts +jscHeap block gains one field: +```diff + jscHeap = { + heapSize: stats.heapSize, + heapCapacity: stats.heapCapacity, + objectCount: stats.objectCount, ++ extraMemorySize: typeof stats.extraMemorySize === "number" ? stats.extraMemorySize : 0, + }; +``` +Type of local `jscHeap` widens accordingly. + +### src/server/memory-watchdog.ts — SYNC-SAFE seam (audit finding 4) + +defaultSample() is synchronous and MUST stay synchronous. Dynamic import is +async, so the seam is a STATIC import: this repository is Bun-native (AGENTS.md +runtime constraint — the proxy and `bun test` always run under Bun), so +`import { heapStats } from "bun:jsc"` at module top is justified; tsc strict +passes with the pinned Bun 1.4 types. The CALL is still guarded: + +```diff ++import { heapStats } from "bun:jsc"; + ... + export type MemorySampleBase = { + ... + arrayBuffers: number; ++ /** JSC heapStats().heapSize, when introspection is available. */ ++ jscHeapSize?: number; ++ /** JSC heapStats().extraMemorySize — JSC-visible native memory. */ ++ jscExtraMemorySize?: number; + }; + ... + function defaultSample(now: () => number): MemorySample { + const usage = process.memoryUsage(); ++ let jscHeapSize: number | undefined; ++ let jscExtraMemorySize: number | undefined; ++ try { ++ const stats = heapStats(); ++ jscHeapSize = stats.heapSize; ++ jscExtraMemorySize = stats.extraMemorySize; ++ } catch { /* introspection failure must never break sampling */ } + const base = { ..., jscHeapSize, jscExtraMemorySize }; +``` +observedMemoryCounter() UNCHANGED — thresholding remains rss/external/ +arrayBuffers. Observability only, no behavior change. Injected `opts.sample` +seam already lets tests supply samples without bun:jsc. + +### src/cli/doctor.ts +Service memory line appends `jscExtra=…` when the API returns +`body.jscHeap.extraMemorySize`. jsShare heuristic unchanged. + +## Tests +tests/memory-watchdog.test.ts: injected sample with jsc fields round-trips +through snapshot(); default sampler under bun test records numeric jsc fields. +system-routes test: /api/system/memory exposes jscHeap.extraMemorySize. + +## Measurement claim +None (diagnostics only) → goalplan c3 rationale: config/diagnostic-only. + diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/020_watchdog_gc_relief.md b/devlog/_plan/260822_260822-bun14-followup-memory/020_watchdog_gc_relief.md new file mode 100644 index 0000000000..9ba054d4be --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/020_watchdog_gc_relief.md @@ -0,0 +1,101 @@ +# 020 — GC relief: measurement-first evaluation, then gated production hook + +Depends on: 010 (diagnostics land first so the evaluation can read +extraMemorySize). THIS DOC LANDS NO PRODUCTION GC CALL BY ITSELF. + +## Prior-decision constraint (controlling) + +devlog/_fin/260731_macos_rss_retention/040_allocator_residual.md:139-161 bans +threshold/idle-triggered production `Bun.gc(true)` and defines the ONLY path +back: three fresh-process runs showing (a) ≥50% of post-load RSS growth gone by +60s after one GC, (b) repeatable across real workloads, (c) idle-only with +measured stop time and unchanged tail latency in a concurrent control, (d) +release-notes/API support on macOS. Written against Bun 1.3.x; Bun 1.4's +shared-allocator purge (000 C6) could flip the result — measure first. + +## Phase A (this unit): harness-only evaluation on Bun 1.4 + +### Child GC control channel (audit r2 finding 1) + +The measured proxy is a spawned child +(scripts/macos-rss-retention-harness.ts:626-638) that today only handles +SIGINT/SIGTERM (harness-child.ts:54-96) — no GC control exists. Add one: + +- scripts/macos-rss-retention-harness-child.ts: subscribe `process.on("SIGUSR2")`; + handler runs `const t0 = Bun.nanoseconds(); Bun.gc(true); const dur = + Bun.nanoseconds() - t0` and writes `{type:"gc", at:Date.now(), + durationMs:dur/1e6}` to stdout JSONL (same channel as "ready"). +- scripts/macos-rss-retention-harness.ts: after each load cell (outside the + latency-measurement window), `processHandle.kill("SIGUSR2")`, await the + `gc` event line (timestamped receipt), then take the +5s and +60s samples. +- GC duration evidence = the child-reported durationMs, not parent guesswork. +(SIGUSR2 is available on darwin/linux — this harness is darwin-targeted; +Windows is out of scope for it, matching the existing script name.) + +### Tail-latency control cells (audit r2 finding 2, r3 finding 1) + +The gate's criterion (c) needs a causally connected control WITHOUT +contaminating the RSS criterion (a). The two criteria use SEPARATE cell types: + +- RSS-retention cells: load stream → intervention (GC via SIGUSR2 with receipt, + or matched idle wait in the control arm) → process stays IDLE through the +5s + and +60s samples. No probe traffic; the +60s sample is pure post-GC idle + evidence for criterion (a). +- Latency cells (separate fresh-process runs): load stream → intervention → + identical POST-INTERVENTION probe stream in both arms; probe-stream p99 delta + (GC arm − control arm) ≤ max(5ms, 5%) is the oracle for criterion (c), with + the GC pause (child durationMs) reported explicitly. RSS numbers from these + cells are recorded but non-normative. + +Deliverable: numbers table in this unit (three fresh-process runs × matched +pairs, per 040 on macmini-cf and locally); verdict PASS/FAIL against the +260731 gate, criterion by criterion. + +## Phase B (conditional follow-up cycle, only on Phase-A PASS) + +- Idle gate: relief only when `getActiveTurnCount() === 0` + (src/server/lifecycle.ts:263 — existing export, no new seam needed). Defer + while busy; re-check next tick. +- Rate limit: OWN `lastReliefAt` (decoupled from lastWarnAt so warn cadence + never suppresses first relief), floor 30min. +- Config: restart-only startup configuration from the config file + (`memoryWatchdog: { gcRelief?: boolean; warnThresholdMb?: number }` in + OcxConfig). NOT in the /api/settings PUT allowlist; restart-only semantics + documented. warnThresholdMb validated at load: integer 256..65536, else + ignored+warn. +- Wiring chain (audit r2 finding 3 — all three layers named): + 1. src/server/index.ts:729 — `acquireServerBackgroundLifecycle(applyPolicy, + { memoryWatchdog: config.memoryWatchdog })`; + 2. src/server/background-lifecycle.ts:129-143 — + `acquireServerBackgroundLifecycle` gains the optional second param and + forwards it to `startProcessLoops(applyPolicy, opts)` (both the + first-owner branch and no change for the re-acquire branch: watchdog + options are first-owner-only, restart-only by definition); + 3. startProcessLoops passes `{ gcRelief, warnThresholdBytes, gc, isIdle }` + into startMemoryWatchdog. +- Test seam (audit r3 finding 2): StartServerDeps (src/server/index.ts:437) + gains an optional `memoryWatchdogDeps?: { gc?: () => void; sample?: () => + MemorySampleBase; now?: () => number; intervalMs?: number; isIdle?: () => + boolean }` forwarded through acquireServerBackgroundLifecycle alongside the + persisted config options into startMemoryWatchdog (deps override config- + derived defaults; production callers pass nothing). The startup integration + test injects gc spy + over-threshold sample + isIdle=true + short intervalMs + through this seam and asserts snapshot().gcRelief === true and the spy fired. + +- snapshot() exposes reliefCount, lastReliefAt, gcRelief. +- Windows caveat: mimalloc page scavenging disabled by design (#34181) — + relief mainly helps darwin/linux RSS. +- docs-site: troubleshooting page gains the new config keys (restart-only). + +## Tests (Phase B) +gcRelief on + above threshold + idle → gc called once; busy → deferred; +second tick within 30min → suppressed by lastReliefAt even when a warn fired +earlier; gc throwing → tick survives; gcRelief absent → never called; config +bounds validation; the startup integration test above. + +## Measurement claim +Phase A IS the measurement (040). Phase B lands only with that evidence +attached — goalplan c3 satisfied by construction. + + + diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md b/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md new file mode 100644 index 0000000000..b1d3919b87 --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/030_smol_workers.md @@ -0,0 +1,45 @@ +# 030 — smol workers: bounded JSC heap for storage/history batch workers + +Depends on: 000. Sibling PR (no shared files with 010/020) — but landing is +GATED on a local large-fixture A/B (audit finding 3). + +## Why + +history-job/restore-job/policy-job spawn short-lived Workers for batch work. +`smol: true` (probe-verified on bundled 1.4.0) selects JSC's Small heap growth +policy → lower peak RSS during storage jobs, at a GC-frequency cost. + +## Risk (audit finding 3 — must be measured before landing) + +These workers are NOT small-payload: policy cleanup materializes all archive +candidates (src/storage/policy.ts:347-379); cleanup snapshots full thread/log/ +memory/goal rows and serializes an aggregate backup +(src/storage/cleanup.ts:765-785,872-914,1171-1269); history reads complete +SQLite result sets with rollout buffers (src/codex/history-provider.ts:586-619, +709-732). smol is a growth-policy choice, not a payload bound — a large job +could GC-thrash or slow past the worker timeout. + +## Pre-landing gate: per-worker large-fixture A/B + +For each worker (history, restore, policy): build a large fixture (≥100MB +aggregate rows / large rollout set), run the job smol-off vs smol-on ×3, +record peak RSS (Subprocess/process sampling), elapsed wall time, completion +status. Acceptance to land each call site: completion success, elapsed within ++25% of baseline, peak RSS reduced. A worker failing the gate keeps its +full-size heap and the doc records the numbers — partial landing (subset of +the three call sites) is an acceptable outcome. + +## Changes (only for call sites that pass the gate) + +src/codex/history-job.ts:309, src/storage/restore-job.ts:170, +src/storage/policy-job.ts:303 — one-line `, { smol: true }` (pinned Bun 1.4 +types include smol; no cast needed per audit). + +## Tests +Existing worker suites stay green (smol changes GC policy, not messaging). +The A/B harness script + numbers are the landing evidence, committed into this +unit. + +## Measurement claim +Local A/B is the primary evidence (host-independent fixtures); macmini optional. + diff --git a/devlog/_plan/260822_260822-bun14-followup-memory/040_macmini_measurement.md b/devlog/_plan/260822_260822-bun14-followup-memory/040_macmini_measurement.md new file mode 100644 index 0000000000..6779d78e97 --- /dev/null +++ b/devlog/_plan/260822_260822-bun14-followup-memory/040_macmini_measurement.md @@ -0,0 +1,28 @@ +# 040 — macmini-cf live measurement protocol + +Depends on: 010 landed on a testable branch. Feeds 020 Phase A verdict. + +## Host facts (verified 2026-08-21) +ssh macmini-cf reachable (BatchMode OK), arm64, bun 1.3.14 installed → natural +1.3.14-vs-1.4.0 A/B host. zsh -lc PATH discipline. + +## Protocol +1. Install test build: `npm pack` locally → scp tarball → `npm i -g ` + on macmini-cf. Record ocx --version + bunVersion/bunRevision/bunRuntimeSource + from /api/system/memory. +2. Baseline: default config; drive SSE churn (harness waves: SSE-normal, + SSE-slow, SSE-abort, idle-recovery); sample /api/system/memory every 60s + ≥30min. Capture extraMemorySize (010). +3. GC evaluation (020 Phase A): matched no-GC/GC cell pairs with identical + concurrent request streams; child-side SIGUSR2 GC with reported durationMs; + +5s/+60s samples; three fresh-process runs; p99 latency delta ≤ max(5ms, 5%) + acceptance; evaluate the 260731 gate verbatim, criterion by criterion. +4. smol A/B (030): if remote numbers wanted beyond the local gate, trigger + storage jobs on both builds; record peak RSS delta. +5. Evidence: scalar-counter JSON only (watchdog privacy contract), committed + into this unit. + +## Acceptance mapping +goalplan c3: 020 carries macmini+local GC-gate numbers incl. latency pairs; +030 carries local A/B numbers; 010 records config/diagnostic-only rationale. + From 1ab34dc4966cf396e990b7d2ac262792da7c34a4 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Sat, 22 Aug 2026 20:19:30 +0900 Subject: [PATCH 2/2] test(scripts): land the Bun 1.4 memory harnesses with their review blockers closed Rebuilds the harness halves of #2303 and #2304 directly on dev, without the #2302 runtime commit those PRs were stacked on. Merging them as stacked would have dragged in the extraMemorySize: 0 fabrication that #2302 still carries, and would also have reverted unrelated coordinator work that landed on src/cli/doctor.ts after the stack was cut. scripts/bun-gc-relief-eval.ts Records rssBeforeLoad and derives postLoadGrowth and recoveryFraction. The controlling 260731 gate is "at least 50% of post-load RSS GROWTH is gone", and the previous shape could not express that: rssAfterLoad - rssPlus60s cannot separate recovery from ordinary drift, and the recorded verdict divided recovered bytes by total post-load RSS, which answers a different question than the gate asks. recoveryFraction is null when growth was not measurable, so a cell that proves nothing does not read as 0% recovery. A child-side gc-error now rejects the waiting cell instead of expiring into a ten-second "gc receipt timeout" that hides the real cause. scripts/macos-rss-retention-harness-child.ts The SIGUSR2 collector is installed only under OCX_GC_EVAL=1. It was gated by a comment saying the 7h retention protocol never sends that signal, which is a claim about one sender rather than a property of the process; a stray signal would have collected inside the measurement that protocol exists to take. scripts/smol-worker-ab.ts payloadMb and runs are validated as bounded integers. Previously runs=0 produced a report claiming completionSuccess over an empty result set with the median fields silently absent, and a negative payload ran a meaningless workload instead of refusing. Medians are computed only once both arms are complete, so a verdict can never be derived from a partial set. The header claimed to measure the audited shapes of history, restore and policy workers; it imports none of them. It now says what it is: a synthetic screening of the array-plus-JSON burst shape those workers share. The FAIL verdicts both harnesses recorded stand. No production Bun.gc(true) call and no smol: true flag is landed here. The GC harness needs a live upstream fixture to produce new numbers, so the recorded RSS cells are NOT regenerated by this commit and the 020 table still carries the old denominator. Re-running the cells and rewriting that table around recoveryFraction is deliberately left as the next measurement pass rather than claimed here. --- scripts/bun-gc-relief-eval.ts | 322 +++++++++++++++++++ scripts/macos-rss-retention-harness-child.ts | 30 ++ scripts/smol-worker-ab.ts | 142 ++++++++ 3 files changed, 494 insertions(+) create mode 100644 scripts/bun-gc-relief-eval.ts create mode 100644 scripts/smol-worker-ab.ts diff --git a/scripts/bun-gc-relief-eval.ts b/scripts/bun-gc-relief-eval.ts new file mode 100644 index 0000000000..9380c8c330 --- /dev/null +++ b/scripts/bun-gc-relief-eval.ts @@ -0,0 +1,322 @@ +/** + * Bun.gc(true) relief evaluation (devlog/_plan/260822_260822-bun14-followup-memory/020 Phase A). + * + * Evaluates the 260731 allocator-residual gate on Bun 1.4: does one full GC + * inside the measured proxy return post-load RSS growth, and does it cost + * request latency? Two SEPARATE cell types keep the criteria uncontaminated: + * + * - rss cells: load stream -> intervention -> process IDLE through +5s/+60s + * samples (criterion a evidence). + * - latency cells: load stream -> intervention -> identical POST-INTERVENTION + * probe stream in both arms; probe p99 is criterion c's oracle. + * + * Arms: control (matched idle wait) vs gc (SIGUSR2 to the child, which runs + * Bun.gc(true) in-process and reports {type:"gc",at,durationMs} on stdout). + * + * This orchestrator reuses macos-rss-retention-harness-child.ts (the real + * startServer proxy) and an inline SSE fixture upstream. It is NOT the locked + * 7h retention protocol; runs are short and labeled. Smoke mode + * (OCX_GC_EVAL_SMOKE=1) shortens durations for pipeline verification only. + * + * Usage: bun scripts/bun-gc-relief-eval.ts + */ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const SMOKE = process.env.OCX_GC_EVAL_SMOKE === "1"; +const RUNS = SMOKE ? 1 : 3; +const LOAD_TURNS = SMOKE ? 3 : 30; +const EVENTS = SMOKE ? 20 : 200; +const EVENT_BYTES = 65_536; +const PROBE_TURNS = SMOKE ? 3 : 20; +const POST_WAIT_1_MS = 5_000; +const POST_WAIT_2_MS = SMOKE ? 10_000 : 60_000; +const READY_TIMEOUT_MS = 15_000; + +const outDir = process.argv[2]; +if (!outDir) throw new Error("usage: bun scripts/bun-gc-relief-eval.ts "); +mkdirSync(outDir, { recursive: true }); + +type Arm = "control" | "gc"; +type CellKind = "rss" | "latency"; + +function frame(event: string | null, data: unknown): Uint8Array { + const payload = typeof data === "string" ? data : JSON.stringify(data); + return new TextEncoder().encode( + (event ? "event: " + event + "\n" : "") + "data: " + payload + "\n\n", + ); +} + +/** Minimal Responses-shaped SSE fixture (mirrors the retention-harness fixture). */ +function startFixture(): { url: string; stop(): Promise } { + let serial = 0; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request) { + if (request.method !== "POST" || new URL(request.url).pathname !== "/v1/responses") { + return new Response("not found", { status: 404 }); + } + await request.json().catch(() => ({})); + const id = ++serial; + let n = 0; + return new Response(new ReadableStream({ + pull(controller) { + let bytes: Uint8Array; + let done = false; + if (n === 0) { + bytes = frame("response.created", { + type: "response.created", + response: { id: "fixture-" + id, status: "in_progress", output: [] }, + }); + } else if (n <= EVENTS) { + bytes = frame("response.output_text.delta", { + type: "response.output_text.delta", + output_index: 0, + content_index: 0, + item_id: "msg-" + id, + delta: "x".repeat(EVENT_BYTES), + }); + } else if (n === EVENTS + 1) { + bytes = frame("response.output_item.done", { + type: "response.output_item.done", + output_index: 0, + item: { id: "msg-" + id, type: "message", status: "completed", role: "assistant", content: [] }, + }); + } else if (n === EVENTS + 2) { + bytes = frame("response.completed", { + type: "response.completed", + response: { id: "fixture-" + id, status: "completed", output: [] }, + }); + } else { + bytes = frame(null, "[DONE]"); + done = true; + } + controller.enqueue(bytes); + n++; + if (done) controller.close(); + }, + }), { headers: { "content-type": "text/event-stream" } }); + }, + }); + return { + url: server.url.toString().replace(/\/$/, ""), + stop: () => server.stop(true), + }; +} + +type ChildHandle = { + port: number; + pid: number; + kill(signal: NodeJS.Signals): void; + nextGcReceipt(): Promise<{ at: number; durationMs: number }>; + rss(): number; + stop(): Promise; +}; + +async function startChild(upstreamUrl: string, dir: string): Promise { + const home = join(dir, "opencodex-home"); + const codexHome = join(dir, "codex-home"); + mkdirSync(home, { recursive: true }); + mkdirSync(codexHome, { recursive: true }); + const child = Bun.spawn([ + process.execPath, + join(import.meta.dir, "macos-rss-retention-harness-child.ts"), + home, + codexHome, + upstreamUrl, + join(dir, "child-series.jsonl"), + "off", + ], { + stdout: "pipe", + stderr: Bun.file(join(dir, "child.stderr.log")), + // The child installs its SIGUSR2 collector only for this evaluation, so the + // locked 7h retention protocol can never collect mid-run. + env: { ...process.env, OCX_GC_EVAL: "1" }, + }); + + let port = 0; + let readyResolve!: () => void; + const ready = new Promise(resolve => { readyResolve = resolve; }); + let gcWaiter: { + resolve: (receipt: { at: number; durationMs: number }) => void; + reject: (error: Error) => void; + } | null = null; + + const reader = child.stdout.getReader(); + const drain = (async () => { + const decoder = new TextDecoder(); + let buffer = ""; + for (;;) { + const part = await reader.read(); + if (part.done) break; + buffer += decoder.decode(part.value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ""; + for (const line of lines) { + try { + const value = JSON.parse(line) as { type?: string; port?: number; at?: number; durationMs?: number }; + if (value.type === "ready" && value.port) { port = value.port; readyResolve(); } + if (value.type === "gc" && gcWaiter && typeof value.at === "number" && typeof value.durationMs === "number") { + const w = gcWaiter; gcWaiter = null; w.resolve({ at: value.at, durationMs: value.durationMs }); + } + // A collection that threw is a failed cell, not a 10s silence. Without + // this the run reports "gc receipt timeout" and hides the real cause. + if (value.type === "gc-error" && gcWaiter) { + const w = gcWaiter; gcWaiter = null; + w.reject(new Error(String((value as { message?: unknown }).message ?? "gc-error"))); + } + } catch { /* startup noise */ } + } + } + })(); + + await Promise.race([ + ready, + child.exited.then(() => { throw new Error("child exited before ready"); }), + Bun.sleep(READY_TIMEOUT_MS).then(() => { throw new Error("readiness timeout"); }), + ]); + + return { + port, + pid: child.pid, + // Bun.spawn's handle.kill() does not deliver SIGUSR2 reliably on Bun 1.4 + // (verified: handle.kill silently no-ops while process.kill(pid) arrives); + // signal through the OS instead. + kill: signal => process.kill(child.pid, signal), + nextGcReceipt: () => new Promise((resolve, reject) => { + gcWaiter = { resolve, reject }; + setTimeout(() => { if (gcWaiter) { gcWaiter = null; reject(new Error("gc receipt timeout")); } }, 10_000); + }), + rss: () => { + const out = Bun.spawnSync(["ps", "-o", "rss=", "-p", String(child.pid)]); + return Number.parseInt(out.stdout.toString().trim(), 10) * 1024; + }, + stop: async () => { + child.kill("SIGTERM"); + await Promise.race([child.exited, Bun.sleep(5_000)]); + child.kill("SIGKILL"); + await drain.catch(() => {}); + }, + }; +} + +async function oneTurn(base: string, label: string, turn: number): Promise { + const started = performance.now(); + const response = await fetch(base + "/v1/responses", { + method: "POST", + headers: { "content-type": "application/json", "x-opencodex-api-key": "fixture-admission" }, + body: JSON.stringify({ model: "fixture/fixture-model", input: label + "-" + turn, stream: true }), + }); + if (response.status !== 200 || !response.body) throw new Error("HTTP " + response.status); + const reader = response.body.getReader(); + for (;;) { + const part = await reader.read(); + if (part.done) break; + } + return performance.now() - started; +} + +function quantile(sorted: number[], q: number): number { + if (sorted.length === 0) return Number.NaN; + const pos = (sorted.length - 1) * q; + const lo = Math.floor(pos); + const hi = Math.ceil(pos); + return sorted[lo]! + (sorted[hi]! - sorted[lo]!) * (pos - lo); +} + +type CellResult = Record; + +async function runCell(kind: CellKind, arm: Arm, runIndex: number): Promise { + const dir = join(outDir, kind + "-" + arm + "-run" + runIndex); + mkdirSync(dir, { recursive: true }); + const fixture = startFixture(); + const child = await startChild(fixture.url, dir); + const base = "http://127.0.0.1:" + child.port; + try { + // Pre-load baseline. The 260731 gate is "at least 50% of post-load RSS + // GROWTH is recovered", so growth has to be measurable: without a baseline, + // `rssAfterLoad - rssPlus60s` cannot tell recovery apart from ordinary drift, + // and dividing by total post-load RSS answers a different question than the + // gate asks. + const rssBeforeLoad = child.rss(); + + // Load stream (identical in both arms). + for (let turn = 0; turn < LOAD_TURNS; turn++) await oneTurn(base, kind + "-load", turn); + const rssAfterLoad = child.rss(); + + // Intervention. + let gcReceipt: { at: number; durationMs: number } | null = null; + if (arm === "gc") { + const receipt = child.nextGcReceipt(); + child.kill("SIGUSR2"); + gcReceipt = await receipt; + } else { + await Bun.sleep(50); // matched (small) intervention window + } + + if (kind === "rss") { + // Idle through both samples: pure criterion-(a) evidence. + await Bun.sleep(POST_WAIT_1_MS); + const rssPlus5s = child.rss(); + await Bun.sleep(POST_WAIT_2_MS - POST_WAIT_1_MS); + const rssPlus60s = child.rss(); + const postLoadGrowth = rssAfterLoad - rssBeforeLoad; + const recoveredByPlus60s = rssAfterLoad - rssPlus60s; + return { + kind, arm, runIndex, smoke: SMOKE, + rssBeforeLoad, rssAfterLoad, rssPlus5s, rssPlus60s, + postLoadGrowth, + gcDurationMs: gcReceipt?.durationMs ?? null, + recoveredByPlus60s, + // The controlling ratio, or null when the load produced no measurable + // growth to recover (a cell that proves nothing rather than a 0% one). + recoveryFraction: postLoadGrowth > 0 ? recoveredByPlus60s / postLoadGrowth : null, + }; + } + + // latency cell: post-intervention probe stream is the oracle. + const latencies: number[] = []; + for (let turn = 0; turn < PROBE_TURNS; turn++) latencies.push(await oneTurn(base, "probe", turn)); + latencies.sort((a, b) => a - b); + return { + kind, arm, runIndex, smoke: SMOKE, + probeTurns: PROBE_TURNS, + p50Ms: quantile(latencies, 0.5), + p99Ms: quantile(latencies, 0.99), + maxMs: latencies[latencies.length - 1], + gcDurationMs: gcReceipt?.durationMs ?? null, + rssNonNormative: child.rss(), + }; + } finally { + await child.stop(); + await fixture.stop(); + } +} + +const results: CellResult[] = []; +for (let run = 0; run < RUNS; run++) { + for (const kind of ["rss", "latency"] as const) { + for (const arm of ["control", "gc"] as const) { + const cell = await runCell(kind, arm, run); + results.push(cell); + console.log(JSON.stringify(cell)); + } + } +} + +const report = { + smoke: SMOKE, + bunVersion: Bun.version, + bunRevision: Bun.revision, + platform: process.platform, + arch: process.arch, + at: new Date().toISOString(), + runs: RUNS, + loadTurns: LOAD_TURNS, + events: EVENTS, + eventBytes: EVENT_BYTES, + results, +}; +writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 2)); +console.log("report: " + join(outDir, "report.json")); diff --git a/scripts/macos-rss-retention-harness-child.ts b/scripts/macos-rss-retention-harness-child.ts index eeb59d3167..3ff2a1af1f 100644 --- a/scripts/macos-rss-retention-harness-child.ts +++ b/scripts/macos-rss-retention-harness-child.ts @@ -65,6 +65,36 @@ process.stdout.write(JSON.stringify({ watchdogIncluded: true, }) + "\n"); +/** + * GC control channel (devlog/_plan/260822_260822-bun14-followup-memory/020): + * SIGUSR2 runs a full collection INSIDE the measured process and reports a + * timestamped receipt with the measured pause on the same stdout JSONL channel + * as "ready". Only the GC-relief evaluation orchestrator uses it, and it sets + * OCX_GC_EVAL=1 to install the handler. + * + * The gate is an env var rather than a comment because the locked 7h retention + * protocol must not be able to collect mid-run: a stray SIGUSR2 from any source + * would silently alter the very measurement that protocol exists to take. + * "Our orchestrator never sends it" is a claim about one sender, not a property + * of the process. + */ +if (process.env.OCX_GC_EVAL === "1") { + process.on("SIGUSR2", () => { + const t0 = Bun.nanoseconds(); + try { + Bun.gc(true); + const durationMs = (Bun.nanoseconds() - t0) / 1e6; + process.stdout.write(JSON.stringify({ type: "gc", at: Date.now(), durationMs }) + "\n"); + } catch (error) { + process.stdout.write(JSON.stringify({ + type: "gc-error", + at: Date.now(), + message: error instanceof Error ? error.message : String(error), + }) + "\n"); + } + }); +} + await new Promise((resolve) => { let closing = false; diff --git a/scripts/smol-worker-ab.ts b/scripts/smol-worker-ab.ts new file mode 100644 index 0000000000..ea1a1bbb0f --- /dev/null +++ b/scripts/smol-worker-ab.ts @@ -0,0 +1,142 @@ +/** + * smol-worker A/B gate (devlog/_plan/260822_260822-bun14-followup-memory/030). + * + * Synthetic SCREENING of whether Worker({ smol: true }) reduces peak RSS on the + * burst shape the production workers share: materialize many rows into arrays, + * then serialize one aggregate JSON. It does NOT import history-provider, + * restore-job, or policy-job, so a verdict here screens that shared shape — it + * is not a per-call-site gate for any individual worker. + * + * Isolation: every run is a FRESH child process (in-process sequential runs + * contaminate baselines — the allocator retains pages across runs, which the + * first version of this script measured as a phantom smol win). The child runs + * the workload in a real Worker thread and exits; the parent reads the child's + * peak RSS from Subprocess.resourceUsage().maxRSS. + * + * Acceptance (audited gate): completion success, elapsed within +25% of + * baseline, peak RSS reduced. + * + * Usage: bun scripts/smol-worker-ab.ts [payloadMb] [runs] + */ +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +const outDir = process.argv[2]; +if (!outDir) throw new Error("usage: bun scripts/smol-worker-ab.ts [payloadMb] [runs]"); + +/** + * A memory-stress script takes explicit, bounded inputs. Unvalidated arguments + * let `runs=0` write a report whose gate claims completionSuccess over an empty + * result set, and let a negative or absurd payload run a meaningless workload or + * exhaust the host. Refuse instead of measuring nothing. + */ +function boundedInt(raw: string | undefined, fallback: number, min: number, max: number, name: string): number { + if (raw === undefined) return fallback; + const parsed = Number(raw); + if (!Number.isInteger(parsed) || parsed < min || parsed > max) { + throw new Error(`${name} must be an integer in ${min}..${max} (received ${JSON.stringify(raw)})`); + } + return parsed; +} +const PAYLOAD_MB = boundedInt(process.argv[3], 100, 1, 512, "payloadMb"); +const RUNS = boundedInt(process.argv[4], 3, 1, 20, "runs"); +mkdirSync(outDir, { recursive: true }); + +/** Child body: runs the audited workload shape inside a real Worker, prints elapsed. */ +const childSource = "const smol = process.argv[2] === \"smol\";\nconst payloadMb = Number.parseInt(process.argv[3] ?? \"100\", 10);\nconst workerSource = \"self.onmessage = (event) => {\\n const { payloadMb } = event.data;\\n const t0 = performance.now();\\n try {\\n const ROW_BYTES = 4096;\\n const rowCount = Math.floor((payloadMb * 1024 * 1024) / ROW_BYTES);\\n const rows = [];\\n for (let i = 0; i < rowCount; i++) {\\n rows.push({\\n id: \\\"row-\\\" + i,\\n kind: i % 3 === 0 ? \\\"thread\\\" : i % 3 === 1 ? \\\"log\\\" : \\\"memory\\\",\\n payload: \\\"x\\\".repeat(ROW_BYTES - 96),\\n at: Date.now(),\\n });\\n }\\n const backup = JSON.stringify({ rows });\\n self.postMessage({ type: \\\"done\\\", rowCount, bytes: backup.length, elapsedMs: performance.now() - t0 });\\n } catch (error) {\\n self.postMessage({ type: \\\"error\\\", message: String(error), elapsedMs: performance.now() - t0 });\\n }\\n};\";\nconst url = URL.createObjectURL(new Blob([workerSource], { type: \"application/javascript\" }));\nconst worker = new Worker(url, smol ? { smol: true } : {});\nconst outcome = await new Promise((resolve, reject) => {\n const timeout = setTimeout(() => reject(new Error(\"worker timeout (120s)\")), 120_000);\n worker.onmessage = (event) => { clearTimeout(timeout); resolve(event.data); };\n worker.onerror = (event) => { clearTimeout(timeout); reject(new Error(event.message || \"worker error\")); };\n worker.postMessage({ payloadMb });\n});\nworker.terminate();\nconsole.log(JSON.stringify(outcome));"; + +const childPath = join(outDir, "ab-child.ts"); +writeFileSync(childPath, childSource); + +type RunResult = { + smol: boolean; + runIndex: number; + ok: boolean; + elapsedMs: number; + maxRssBytes: number; + error?: string; +}; + +async function oneRun(smol: boolean, runIndex: number): Promise { + const child = Bun.spawn([process.execPath, childPath, smol ? "smol" : "full", String(PAYLOAD_MB)], { + stdout: "pipe", + stderr: "pipe", + }); + const exitCode = await child.exited; + const stdout = await new Response(child.stdout).text(); + const stderr = await new Response(child.stderr).text(); + const usage = child.resourceUsage(); + // Bun reports maxRSS in bytes on darwin (ru_maxrss is bytes on macOS, KiB on Linux; + // Bun.resourceUsage normalizes to bytes). + const maxRssBytes = usage?.maxRSS ?? -1; + let elapsedMs = -1; + let ok = false; + let error: string | undefined; + try { + const line = stdout.trim().split(/\r?\n/).pop() ?? ""; + const parsed = JSON.parse(line) as { type: string; elapsedMs: number; message?: string }; + ok = exitCode === 0 && parsed.type === "done"; + elapsedMs = parsed.elapsedMs; + error = parsed.message; + } catch { + error = "unparseable child output; exit " + exitCode + "; stderr: " + stderr.slice(0, 200); + } + return { smol, runIndex, ok, elapsedMs, maxRssBytes, ...(error ? { error } : {}) }; +} + +const results: RunResult[] = []; +for (let run = 0; run < RUNS; run++) { + for (const smol of [false, true]) { + const r = await oneRun(smol, run); + results.push(r); + console.log(JSON.stringify(r)); + } +} + +function median(values: number[]): number { + // An empty arm has no median. Returning undefined here would silently become + // NaN in the gate; the caller must not ask until it has a complete arm. + if (values.length === 0) throw new Error("median of an empty result set"); + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]!; +} + +const off = results.filter(r => !r.smol && r.ok); +const on = results.filter(r => r.smol && r.ok); +const gate = { + completionSuccess: RUNS > 0 && on.length === RUNS && off.length === RUNS, + medianElapsedOffMs: null as number | null, + medianElapsedOnMs: null as number | null, + medianMaxRssOffBytes: null as number | null, + medianMaxRssOnBytes: null as number | null, + elapsedWithin25Pct: false, + peakRssReduced: false, + verdict: "fail" as "pass" | "fail", +}; +if (gate.completionSuccess) { + // Medians are computed only once both arms are complete, so the gate can never + // report a verdict derived from a partial or empty run set. + gate.medianElapsedOffMs = median(off.map(r => r.elapsedMs)); + gate.medianElapsedOnMs = median(on.map(r => r.elapsedMs)); + gate.medianMaxRssOffBytes = median(off.map(r => r.maxRssBytes)); + gate.medianMaxRssOnBytes = median(on.map(r => r.maxRssBytes)); + gate.elapsedWithin25Pct = gate.medianElapsedOnMs <= gate.medianElapsedOffMs * 1.25; + gate.peakRssReduced = gate.medianMaxRssOnBytes < gate.medianMaxRssOffBytes; + gate.verdict = gate.elapsedWithin25Pct && gate.peakRssReduced ? "pass" : "fail"; +} + +const report = { + bunVersion: Bun.version, + bunRevision: Bun.revision, + platform: process.platform, + arch: process.arch, + at: new Date().toISOString(), + payloadMb: PAYLOAD_MB, + runs: RUNS, + isolation: "fresh child process per run; peak = Subprocess.resourceUsage().maxRSS", + results, + gate, +}; +writeFileSync(join(outDir, "report.json"), JSON.stringify(report, null, 2)); +console.log(JSON.stringify(gate)); +console.log("report: " + join(outDir, "report.json"));