perf(pm): memoise the source maskers dispatch-gates re-runs per family (843s to 303s under the verify lock)

scripts/pm/dispatch-gates.mjs

@@ -381,6 +381,71 @@ export { isExtractConfigPath, isMetadataFormModulePath };
 
 const ROOT = new URL('../..', import.meta.url).pathname;
 
+// ── The source maskers are memoised, because discovery masks each file ~12x ──
+//
+// PROFILED, not guessed. `discoverFamilies` hands the SAME source string to six
+// analysers in one pass of its per-family loop — `extractWatchHints`,
+// `readProgramTargetsInSource`, `payloadEnvDependence`, `firstPartyImportTargets`,
+// `spawnedProgramTargets` and `packageManifestTargets` — and every one of them
+// re-derives the masked body from scratch, two of them twice (they mask, then
+// hand the masked text to `anchoredReadTargets`, which masks again). One source
+// therefore pays `maskComments` about seven times and `maskSelfTests` about five,
+// per discovery, for bytes that cannot have changed in between.
+//
+// The cost that buys: a V8 CPU profile of ONE `discoverFamilies()` call on this
+// tree — 201 families, 196 distinct gate sources, 11.8 MB of them — spent 14.3 s,
+// of which `maskSelfTests` was 4.5 s of self time (31.7%) and the `maskComments`
+// inside those six analysers most of another 5.2 s. That is the largest single
+// entry in the profile, and all of it above the first pass is repetition.
+//
+// So the maskers are memoised on their INPUT STRING. Both are pure functions of
+// that string, and JavaScript strings are immutable, so a memo is
+// observationally identical to calling through: same bytes in, same bytes out,
+// and no caller can edit the shared result under another. ⛔ This changes
+// nothing about WHAT is masked, scanned or discovered — it is the same
+// derivation run once instead of a dozen times, which is the only kind of
+// speed-up this tool may take.
+//
+// The bound is in BYTES rather than entries because the corpora these run over
+// differ by three orders of magnitude: the gate set is ~12 MB and fits whole, so
+// repeated discoveries in one process reuse it, while a tracked-corpus sweep
+// would otherwise grow the cache without limit. Eviction is oldest-first, and a
+// miss after eviction is a recomputation — never a different answer.
+const MASK_MEMO_BYTE_BUDGET = 32 * 1024 * 1024;
+
+function memoiseMask(compute) {
+  const cache = new Map();
+  let bytes = 0;
+  return (source) => {
+    // A non-string argument is passed straight through: today's behaviour is
+    // whatever the masker does with it, and a memo must not be the thing that
+    // decides otherwise.
+    if (typeof source !== 'string') return compute(source);
+    const hit = cache.get(source);
+    if (hit !== undefined) return hit;
+    const value = compute(source);
+    cache.set(source, value);
+    bytes += source.length + value.length;
+    // `Map` iterates in insertion order, so the first key is the oldest.
+    while (bytes > MASK_MEMO_BYTE_BUDGET && cache.size > 1) {
+      const oldest = cache.keys().next().value;
+      bytes -= oldest.length + cache.get(oldest).length;
+      cache.delete(oldest);
+    }
+    return value;
+  };
+}
+
+/** `maskComments`, memoised — see the block above. */
+const maskedComments = memoiseMask((source) => maskComments(source));
+
+/**
+ * `maskSelfTests(maskComments(source))`, memoised — see the block above. It
+ * composes through `maskedComments` rather than calling `maskComments` again, so
+ * the comment mask is derived once for the callers that want each half.
+ */
+const maskedModuleBody = memoiseMask((source) => maskSelfTests(maskedComments(source)));
+
 // ── What a gate that IMPORTS this module inherits (#11556) ─────────────────
 //
 // This module is importable and is NOT a discovered gate file — `check:pm-dispatch-gates`
@@ -1538,7 +1603,7 @@ const PAYLOAD_ENV_ACCESS = new RegExp(
  * the difference between classifying a gate and classifying its docblock.
  */
 export function payloadEnvDependence(scriptSource) {
-  const body = maskSelfTests(maskComments(String(scriptSource)));
+  const body = maskedModuleBody(String(scriptSource));
   return PAYLOAD_ENV_ACCESS.test(body) ? WORKFLOW_PAYLOAD_ENV : null;
 }
 
@@ -2366,7 +2431,7 @@ export const COMPOUND_ANCHOR_KEYS = new Map(
  */
 export function compoundAnchorDecls(source) {
   const scan = scanSource(source);
-  const decommented = maskComments(source);
+  const decommented = maskedComments(source);
   const out = [];
   for (const m of decommented.matchAll(SELF_TEST_DECL)) {
     if (scan.comment[m.index] || scan.literal[m.index]) continue;
@@ -2959,7 +3024,7 @@ export function packageRootAnchoredHint(hint, base, tree, files) {
  * have to remember to do it.
  */
 export function extractWatchHints(scriptSource, scriptPath = null, { tree = null } = {}) {
-  const moduleBody = maskSelfTests(maskComments(scriptSource));
+  const moduleBody = maskedModuleBody(scriptSource);
   const hints = new Set();
   for (const m of moduleBody.matchAll(/['"`]([^'"`\n]{2,120})['"`]/g)) {
     const raw = m[1];
@@ -3175,7 +3240,7 @@ export function firstPartyImportTargets(scriptPath, source, { root = ROOT } = {}
   // The same masking hint extraction uses, for the same reason: an import
   // written out in a docblock, or one inside a self-test fixture, is a
   // specifier this script NAMES rather than one it loads.
-  const body = maskSelfTests(maskComments(String(source)));
+  const body = maskedModuleBody(String(source));
   const specifiers = new Set();
   for (const m of body.matchAll(IMPORT_FROM_SPECIFIER)) specifiers.add(m[2]);
   for (const m of body.matchAll(SIDE_EFFECT_IMPORT)) specifiers.add(m[2]);
@@ -5234,7 +5299,7 @@ function combineReadings(readings, name) {
  * own repo-relative path — the anchor spellings resolve against it.
  */
 export function scratchDirSitesInSource(rel, source) {
-  const masked = maskComments(String(source));
+  const masked = maskedComments(String(source));
   // A call spelled inside a STRING is a fixture, not a call — this module's own
   // self-test plants fixture sources as string literals, and read as code they
   // reported four sites in a file that creates none of them. Comments are
@@ -5394,7 +5459,7 @@ export function readProgramTargetsInSource(rel, source, isTracked) {
 
 /** Every TRACKED file the source opens at a path anchored to its own location. */
 export function anchoredReadTargets(rel, source, isTracked) {
-  const masked = maskComments(String(source));
+  const masked = maskedComments(String(source));
   const { literal } = scanSource(masked);
   const ctx = {
     fileSegs: rel.split('/'),
@@ -5533,7 +5598,7 @@ export function spawnedProgramTargets(rel, source, isTracked) {
   // follow inherits a POPULATION, and a spawn written inside a self-test body
   // is a fixture the self-test drives rather than the gate's work. The read
   // scan next door wants the opposite from the same bytes, and says so.
-  const masked = maskSelfTests(maskComments(String(source)));
+  const masked = maskedModuleBody(String(source));
   const { literal } = scanSource(masked);
   const ctx = {
     fileSegs: rel.split('/'),
@@ -5717,7 +5782,7 @@ const PACKAGE_MANIFEST_TARGET = /(?:^|\/)package\.json$/;
 const MANIFEST_EXPORTS_READ = /(?<!\bmodule)\.exports\b|\[\s*(['"`])exports\1\s*\]|\bexports\s*[:?]/;
 
 export function packageManifestTargets(rel, source, isTracked) {
-  const masked = maskSelfTests(maskComments(String(source)));
+  const masked = maskedModuleBody(String(source));
   if (!MANIFEST_EXPORTS_READ.test(masked)) return [];
   return anchoredReadTargets(rel, masked, isTracked).filter((t) => PACKAGE_MANIFEST_TARGET.test(t));
 }
@@ -6635,7 +6700,7 @@ export function stampsAnErrorCodeLiteral(path, readSource = readTrackedSource) {
   // Comments are masked for the reason the gate masks them: a code DISCUSSED in
   // prose is not a code stamped in source. This narrows nothing the gate would
   // have reported, so it costs no recall in the expensive direction.
-  const masked = maskComments(source);
+  const masked = maskedComments(source);
   return CODE_STAMP_POSITION.test(masked) || CODE_CONSTANT_BINDING.test(masked);
 }