diff --git a/scripts/measure-self-test-floor.mjs b/scripts/measure-self-test-floor.mjs index 4afcf7e0be..18088a5e42 100644 --- a/scripts/measure-self-test-floor.mjs +++ b/scripts/measure-self-test-floor.mjs @@ -108,7 +108,7 @@ import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { isEntrypoint } from './invoked-as.mjs'; -import { maskComments } from './js-comment-mask.mjs'; +import { blank, maskComments, scanSource } from './js-comment-mask.mjs'; const ROOT = join(fileURLToPath(new URL('.', import.meta.url)), '..'); @@ -244,14 +244,62 @@ export function selfTestDefs(src) { return [...names]; } -/** Insert `return;` as the first statement of `name`. Returns null when absent. */ +/** + * The source a DEFINITION may be anchored in: comments AND the content of every + * string, template and regex literal blanked, every other byte -- and every + * offset and line number -- left exactly where it was, so a match found here + * slices the ORIGINAL text. + * + * ⛔ NOT for the population criterion above, which must keep reading + * `maskComments`. Every `--self-test` dispatch names the flag with a string + * literal, so this mask blanks the dispatch out of every file in the tree; a + * control below pins the two masks to their opposite answers. + */ +export function maskCommentsAndLiterals(source) { + const { comment, literal } = scanSource(source); + const both = new Uint8Array(source.length); + for (let i = 0; i < both.length; i++) both[i] = comment[i] | literal[i]; + return blank(source, both); +} + +/** + * Insert `return;` as the first statement of `name`. Returns null when absent. + * + * TWO rules make the anchor the DEFINITION rather than the first text that reads + * like one, and NEITHER is redundant -- the control fixture below carries one + * decoy of each kind, in this order, ahead of its real definition: + * + * MASKED the match is taken over `maskCommentsAndLiterals(src)`. Read raw, + * the first `function selfTest() {` in `scripts/pm/dispatch-gates.mjs` + * is a sentence in a docblock and the next is a FIXTURE STRING. + * LINE-START the match must BEGIN a line, `export` / `async` being the only + * prefixes a definition in this tree carries. Masking alone still + * prefers a mid-line named function expression -- a value in an + * object literal is not the function the dispatch calls. + * + * What the first rule costs when it is missing is not an absence: injecting into + * a template literal makes the copy a SyntaxError, and a copy that dies in the + * parser exits non-zero and prints a stack, so `mutatedSpoke` is true and the + * verdict above reads HELD -- a hold awarded to a gate for the probe having + * broken its own copy of it. `scripts/pm/dispatch-gates.mjs` carried an + * `ENTRY_BY_HAND` null for months over exactly this, a limit of the INSTRUMENT + * recorded as a property of the FILE (#14963). Anchored on the definition its + * copy parses and runs -- measured by hand, exit 1 and `selfTest() returned + * without reaching its verdict`. The row it leaves is still NOT MEASURED, but + * for a reason the probe now states itself: see that row. + */ export function injectEarlyReturn(src, name) { + const code = maskCommentsAndLiterals(src); const pats = [ - new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\([^)]*\\)\\s*(?::\\s*[A-Za-z_$][\\w$<>\\[\\]|. ]*\\s*)?\\{`), - new RegExp(`const\\s+${name}\\s*=\\s*(?:async\\s*)?\\([^)]*\\)\\s*(?::[^=]*)?=>\\s*\\{`), + new RegExp( + `^[ \\t]*(?:export\\s+(?:default\\s+)?)?(?:async\\s+)?function\\s+${name}\\s*\\([^)]*\\)` + + `\\s*(?::\\s*[A-Za-z_$][\\w$<>\\[\\]|. ]*\\s*)?\\{`, + 'm', + ), + new RegExp(`^[ \\t]*(?:export\\s+)?const\\s+${name}\\s*=\\s*(?:async\\s*)?\\([^)]*\\)\\s*(?::[^=]*)?=>\\s*\\{`, 'm'), ]; for (const re of pats) { - const m = src.match(re); + const m = code.match(re); if (!m) continue; const at = m.index + m[0].length; return `${src.slice(0, at)}\n return; /*${PROBE_MARKER}*/\n${src.slice(at)}`; @@ -436,7 +484,7 @@ const UNRUNNABLE_GATE = [ * The HELPER handshake spelling, reduced: the third of the three landed * handshake shapes, and the one the probe had never read in either direction. * Its only carrier under `scripts/` is `check-platform-checklist.mjs`, whose - * `ENTRY_BY_HAND` row is a deliberate `null` (its dispatch calls four self-test + * `ENTRY_BY_HAND` row is a deliberate `null` (its dispatch calls six self-test * functions and combines their statuses), so every probe run recorded NOT * MEASURED for it and the sweep said nothing at all about this shape -- not * held, not defeated (#15371). The other two spellings each have dozens of live @@ -567,6 +615,61 @@ const NON_DISPATCH_MENTION_GATE = [ '', ].join('\n'); +/** + * The DECOY shape, reduced: three texts that read like `function selfTest() {` + * standing AHEAD of the real definition, one for each way the anchor could take + * the wrong one, in the order they occur in `scripts/pm/dispatch-gates.mjs`. + * + * 1. a docblock sentence naming the convention -- a COMMENT; + * 2. a fixture the gate feeds its own scanner -- a TEMPLATE LITERAL; + * 3. a named function expression held as a value -- real CODE, MID-LINE. + * + * Each defeats a different half of the rule, and each fails DIFFERENTLY, which + * is why one fixture carries all three rather than three carrying one: + * + * into (1) the `return;` lands in a comment, the copy behaves exactly as the + * original, and the probe reads `mutation had no observable effect`; + * into (2) it lands inside a template literal, so the copy is a SyntaxError -- + * which exits non-zero AND prints a stack, and is therefore scored HELD. The + * flattering direction: a hold awarded to a gate for the probe breaking its + * own copy of it (#14963); + * into (3) it lands in a function nothing calls, and the probe again reads no + * observable effect. + * + * The real definition below them is HOLED -- its dispatch discards the result -- + * so the one reading that can only come from anchoring on it is DEFEATED with + * ZERO bytes printed. The fixture is spawned, so that verdict also says the + * mutated copy PARSED and RAN. + */ +const DECOY_ANCHOR_GATE = [ + '#!/usr/bin/env node', + '// The convention this tree writes: `function selfTest() {` at column 0.', + 'const FIXTURE = `', + 'function selfTest() {', + " console.log('a fixture the gate scans, not a definition');", + '}', + '`;', + 'const holder = { run: function selfTest() { return FIXTURE.length; } };', + 'function selfTest() {', + ' const failures = [];', + " if (holder.run() < 1) failures.push('the fixture text went missing');", + " if (failures.length) { console.error(failures.join(String.fromCharCode(10))); process.exit(1); }", + " console.log('fixture self-test: 1 case passes');", + '}', + "if (process.argv.includes('--self-test')) selfTest();", + '', +].join('\n'); + +/** The definition-shaped text the three decoys and the real definition share. */ +const DECOY_ANCHOR_TEXT = 'function selfTest() {'; + +/** + * The anchor an UNMASKED, UNANCHORED first match takes -- the pre-#14963 rule, + * kept here as the thing the controls below measure against rather than as a + * second implementation of anything: `injectEarlyReturn` never uses it. + */ +const NAIVE_ANCHOR = /(?:async\s+)?function\s+selfTest\s*\([^)]*\)\s*\{/; + /** * Both instruments, against both directions. Returns the failures; the caller * refuses on any. Nothing here reads the repo, so a control failure is always @@ -599,6 +702,41 @@ export function runControls() { say(DISPATCH.test(NON_DISPATCH_MENTION_GATE), 'CONTROL FIXTURE INVALID: the mention fixture does not match even UNMASKED, so the verdict above says nothing about comments being masked away'); + // ⛔ ... and the population criterion must keep reading COMMENT-masked source. + // The mask the injection ANCHOR needs blanks literals too, and every dispatch + // in this tree names the flag with a string literal -- so reading the census + // through that one would not classify a single file generously, it would empty + // the population, taking this instrument's own control fixtures (deliberately + // strings) with it. The two masks are required to answer this OPPOSITELY. + say(!DISPATCH.test(maskCommentsAndLiterals(HOLED_GATE)), + 'CONTROL FAILED: the code-only mask that the injection anchor reads still admits a dispatch to the population; the two masks no longer answer differently, and whichever of them the census ends up reading, one of the two questions is being answered with the wrong text'); + + // The ANCHOR, against every text that reads like a definition without being + // one. What is at stake is not a classification but a WRONG READING: an + // injection into a fixture string makes the copy a SyntaxError, whose non-zero + // exit and stack trace this file's verdict scores HELD (#14963). + const decoyFlags = scanSource(DECOY_ANCHOR_GATE); + const commentDecoy = DECOY_ANCHOR_GATE.indexOf(DECOY_ANCHOR_TEXT); + const literalDecoy = DECOY_ANCHOR_GATE.indexOf(DECOY_ANCHOR_TEXT, commentDecoy + 1); + const midLineDecoy = DECOY_ANCHOR_GATE.indexOf('function selfTest() { return FIXTURE.length'); + const realDef = DECOY_ANCHOR_GATE.indexOf('\nfunction selfTest() {\n const failures') + 1; + say(commentDecoy >= 0 && literalDecoy > commentDecoy && midLineDecoy > literalDecoy && realDef > midLineDecoy, + 'CONTROL FIXTURE INVALID: the three decoys no longer all stand AHEAD of the real definition, so a first-match anchor would reach the definition however it was spelled and every verdict below passes for the wrong reason'); + say(decoyFlags.comment[commentDecoy] === 1, + 'CONTROL FIXTURE INVALID: the first decoy is not comment content, so it no longer tests the comment half of the mask'); + say(decoyFlags.literal[literalDecoy] === 1, + 'CONTROL FIXTURE INVALID: the second decoy is not literal content, so it no longer tests the string/template half of the mask -- the half whose failure produces a SyntaxError and a false HELD'); + say(decoyFlags.comment[midLineDecoy] === 0 && decoyFlags.literal[midLineDecoy] === 0, + 'CONTROL FIXTURE INVALID: the third decoy is masked away as comment or literal, so it tests the mask a second time instead of the LINE-START rule it is there for'); + say(DECOY_ANCHOR_GATE.search(NAIVE_ANCHOR) === commentDecoy, + 'CONTROL FIXTURE INVALID: an unmasked first-match anchor no longer lands on a decoy, so the MASK is not what the anchor verdict below is reading'); + say(maskCommentsAndLiterals(DECOY_ANCHOR_GATE).search(NAIVE_ANCHOR) === midLineDecoy, + 'CONTROL FIXTURE INVALID: masking alone no longer lands on the mid-line decoy, so the LINE-START rule is not what the anchor verdict below is reading -- masking would be carrying it on its own'); + const decoyInjected = injectEarlyReturn(DECOY_ANCHOR_GATE, 'selfTest'); + say(decoyInjected !== null + && decoyInjected.includes(`function selfTest() {\n return; /*${PROBE_MARKER}*/\n\n const failures = [];`), + 'ANCHOR CONTROL FAILED: the early return was not injected at the REAL definition; a text that merely READS like one -- in a comment, in a fixture string, or mid-line in code -- was preferred over the function the dispatch calls'); + // Instrument 1, both directions. say(classifyFloor(maskComments(HOLED_GATE)) === 'NONE', 'POSITIVE CONTROL FAILED: a self-test deciding success by failures.length alone was not classified NONE'); @@ -637,18 +775,21 @@ export function runControls() { const sound = join(dir, 'sound-gate.mjs'); const accident = join(dir, 'accident-gate.mjs'); const unrunnable = join(dir, 'unrunnable-gate.mjs'); + const decoy = join(dir, 'decoy-anchor-gate.mjs'); const helper = join(dir, 'helper-handshake-gate.mjs'); const helperHoled = join(dir, 'helper-handshake-gate-holed.mjs'); writeFileSync(holed, HOLED_GATE); writeFileSync(sound, SOUND_GATE); writeFileSync(accident, ACCIDENT_GATE); writeFileSync(unrunnable, UNRUNNABLE_GATE); + writeFileSync(decoy, DECOY_ANCHOR_GATE); writeFileSync(helper, HELPER_HANDSHAKE_GATE); writeFileSync(helperHoled, HELPER_HANDSHAKE_GATE_HOLED); const h = probeEarlyReturn(holed, 'selfTest'); const s = probeEarlyReturn(sound, 'selfTest'); const a = probeEarlyReturn(accident, 'runSelfTest'); const u = probeEarlyReturn(unrunnable, 'selfTest'); + const dc = probeEarlyReturn(decoy, 'selfTest'); const hh = probeEarlyReturn(helper, 'selfTest'); const hhHoled = probeEarlyReturn(helperHoled, 'selfTest'); say(h.verdict === 'DEFEATED', @@ -676,6 +817,14 @@ export function runControls() { // being red, would satisfy the verdict above while testing nothing. say(u.baselineExit !== 0 && u.baselineBytes > 0 && u.baselineHead !== '', `POSITIVE CONTROL FAILED: the unrunnable fixture no longer produces the measured shape (baseline exit ${u.baselineExit}, ${u.baselineBytes} byte(s)); the NOT MEASURED verdict above would then be passing for the wrong reason`); + // The anchor, ON DISK. The static assertions above read WHERE the injection + // went; this one reads what the copy then DID -- so it also says the copy + // parsed and ran. Anchored on the fixture string instead, the copy dies in + // the parser: non-zero exit, a stack trace on stderr, verdict HELD. + say(dc.verdict === 'DEFEATED', + `POSITIVE CONTROL FAILED: a known-holed gate whose real definition is preceded by three definition-shaped decoys was read as ${dc.verdict} (${dc.why ?? ''}); anchored on the fixture string this reads HELD, a hold awarded for the probe breaking its own copy`); + say(dc.mutatedBytes === 0, + `POSITIVE CONTROL FAILED: the decoy gate printed ${dc.mutatedBytes} byte(s) when defeated; the copy anchored on the real definition returns before its verdict line and says NOTHING, so anything printed here is the copy failing rather than the gate being silent`); // The HELPER handshake spelling, both directions, differing by ONE line: the // dispatch asking `requireReachedVerdict`. Until this fixture the probe had // read that shape in NEITHER direction -- its single carrier in the tree is @@ -707,7 +856,7 @@ export function runControls() { // --------------------------------------------------------------------------- /** - * ⛔ SHRINK-ONLY. The nine files whose entry cannot be resolved mechanically, + * ⛔ SHRINK-ONLY. The ten files whose entry cannot be resolved mechanically, * resolved by READING the dispatch site (the ruling's A2.1: the grep is an * entry point, not the criterion). A `null` entry is NOT MEASURED with the * stated reason -- never a quiet pass, and never a guess. @@ -726,15 +875,24 @@ export const ENTRY_BY_HAND = Object.freeze({ // `runSelfTest()` is what the dispatch calls. Probing the inner one records a // TypeError as a handshake; probing `runSelfTest` reads the real one (#14842). 'scripts/check-workspace-manifest-cycles.mjs': 'runSelfTest', - // The dispatch calls FOUR self-test functions and combines their statuses; + // The dispatch calls SIX self-test functions and combines their statuses; // there is no single entry an early return leaves, so a one-function probe // measures a sub-battery and reads a downstream crash as a handshake. 'scripts/check-platform-checklist.mjs': null, // The self-test is an inline top-level block calling several helpers. 'scripts/check-regen-pending.mjs': null, - // Injecting into this file produces a SyntaxError (the anchor lands inside a - // template literal), so no run of it measures anything. - 'scripts/pm/dispatch-gates.mjs': null, + // Four self-test-shaped names in raw source (`selfTest`, `selfTestOnlyCallables` + // and two inside fixtures), so the entry is hand-read -- but for THIS FILE's + // reason. The row was a `null` until #14963, on the claim that injecting here + // can only produce a SyntaxError: the anchor read raw source, where a docblock + // sentence and then a FIXTURE STRING stand ahead of the real definition. That + // was the INSTRUMENT's limit recorded as this file's property. Anchored on the + // definition the copy parses and runs (measured: exit 1, `selfTest() returned + // without reaching its verdict`). The probe still reads NOT MEASURED here and + // now says why itself -- `baseline run failed (exit 1)`, because it writes its + // copy under `scripts/`, where this gate's own single-site sweep finds the + // near-duplicate and refuses (#15515). A separate card, not worked around here. + 'scripts/pm/dispatch-gates.mjs': 'selfTest', }); /**