Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
180 changes: 169 additions & 11 deletions scripts/measure-self-test-floor.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ import { tmpdir } from 'node:os';
import { fileURLToPath } from 'node:url';

import { isEntrypoint } from './invoked-as.mjs';
import { maskComments } from './js-comment-mask.mjs';
import { blank, maskComments, scanSource } from './js-comment-mask.mjs';

const ROOT = join(fileURLToPath(new URL('.', import.meta.url)), '..');

Expand Down Expand Up @@ -244,14 +244,62 @@ export function selfTestDefs(src) {
return [...names];
}

/** Insert `return;` as the first statement of `name`. Returns null when absent. */
/**
* The source a DEFINITION may be anchored in: comments AND the content of every
* string, template and regex literal blanked, every other byte -- and every
* offset and line number -- left exactly where it was, so a match found here
* slices the ORIGINAL text.
*
* ⛔ NOT for the population criterion above, which must keep reading
* `maskComments`. Every `--self-test` dispatch names the flag with a string
* literal, so this mask blanks the dispatch out of every file in the tree; a
* control below pins the two masks to their opposite answers.
*/
export function maskCommentsAndLiterals(source) {
const { comment, literal } = scanSource(source);
const both = new Uint8Array(source.length);
for (let i = 0; i < both.length; i++) both[i] = comment[i] | literal[i];
return blank(source, both);
}

/**
* Insert `return;` as the first statement of `name`. Returns null when absent.
*
* TWO rules make the anchor the DEFINITION rather than the first text that reads
* like one, and NEITHER is redundant -- the control fixture below carries one
* decoy of each kind, in this order, ahead of its real definition:
*
* MASKED the match is taken over `maskCommentsAndLiterals(src)`. Read raw,
* the first `function selfTest() {` in `scripts/pm/dispatch-gates.mjs`
* is a sentence in a docblock and the next is a FIXTURE STRING.
* LINE-START the match must BEGIN a line, `export` / `async` being the only
* prefixes a definition in this tree carries. Masking alone still
* prefers a mid-line named function expression -- a value in an
* object literal is not the function the dispatch calls.
*
* What the first rule costs when it is missing is not an absence: injecting into
* a template literal makes the copy a SyntaxError, and a copy that dies in the
* parser exits non-zero and prints a stack, so `mutatedSpoke` is true and the
* verdict above reads HELD -- a hold awarded to a gate for the probe having
* broken its own copy of it. `scripts/pm/dispatch-gates.mjs` carried an
* `ENTRY_BY_HAND` null for months over exactly this, a limit of the INSTRUMENT
* recorded as a property of the FILE (#14963). Anchored on the definition its
* copy parses and runs -- measured by hand, exit 1 and `selfTest() returned
* without reaching its verdict`. The row it leaves is still NOT MEASURED, but
* for a reason the probe now states itself: see that row.
*/
export function injectEarlyReturn(src, name) {
const code = maskCommentsAndLiterals(src);
const pats = [
new RegExp(`(?:async\\s+)?function\\s+${name}\\s*\\([^)]*\\)\\s*(?::\\s*[A-Za-z_$][\\w$<>\\[\\]|. ]*\\s*)?\\{`),
new RegExp(`const\\s+${name}\\s*=\\s*(?:async\\s*)?\\([^)]*\\)\\s*(?::[^=]*)?=>\\s*\\{`),
new RegExp(
`^[ \\t]*(?:export\\s+(?:default\\s+)?)?(?:async\\s+)?function\\s+${name}\\s*\\([^)]*\\)` +
`\\s*(?::\\s*[A-Za-z_$][\\w$<>\\[\\]|. ]*\\s*)?\\{`,
'm',
),
new RegExp(`^[ \\t]*(?:export\\s+)?const\\s+${name}\\s*=\\s*(?:async\\s*)?\\([^)]*\\)\\s*(?::[^=]*)?=>\\s*\\{`, 'm'),
];
for (const re of pats) {
const m = src.match(re);
const m = code.match(re);
if (!m) continue;
const at = m.index + m[0].length;
return `${src.slice(0, at)}\n return; /*${PROBE_MARKER}*/\n${src.slice(at)}`;
Expand Down Expand Up @@ -436,7 +484,7 @@ const UNRUNNABLE_GATE = [
* The HELPER handshake spelling, reduced: the third of the three landed
* handshake shapes, and the one the probe had never read in either direction.
* Its only carrier under `scripts/` is `check-platform-checklist.mjs`, whose
* `ENTRY_BY_HAND` row is a deliberate `null` (its dispatch calls four self-test
* `ENTRY_BY_HAND` row is a deliberate `null` (its dispatch calls six self-test
* functions and combines their statuses), so every probe run recorded NOT
* MEASURED for it and the sweep said nothing at all about this shape -- not
* held, not defeated (#15371). The other two spellings each have dozens of live
Expand Down Expand Up @@ -567,6 +615,61 @@ const NON_DISPATCH_MENTION_GATE = [
'',
].join('\n');

/**
* The DECOY shape, reduced: three texts that read like `function selfTest() {`
* standing AHEAD of the real definition, one for each way the anchor could take
* the wrong one, in the order they occur in `scripts/pm/dispatch-gates.mjs`.
*
* 1. a docblock sentence naming the convention -- a COMMENT;
* 2. a fixture the gate feeds its own scanner -- a TEMPLATE LITERAL;
* 3. a named function expression held as a value -- real CODE, MID-LINE.
*
* Each defeats a different half of the rule, and each fails DIFFERENTLY, which
* is why one fixture carries all three rather than three carrying one:
*
* into (1) the `return;` lands in a comment, the copy behaves exactly as the
* original, and the probe reads `mutation had no observable effect`;
* into (2) it lands inside a template literal, so the copy is a SyntaxError --
* which exits non-zero AND prints a stack, and is therefore scored HELD. The
* flattering direction: a hold awarded to a gate for the probe breaking its
* own copy of it (#14963);
* into (3) it lands in a function nothing calls, and the probe again reads no
* observable effect.
*
* The real definition below them is HOLED -- its dispatch discards the result --
* so the one reading that can only come from anchoring on it is DEFEATED with
* ZERO bytes printed. The fixture is spawned, so that verdict also says the
* mutated copy PARSED and RAN.
*/
const DECOY_ANCHOR_GATE = [
'#!/usr/bin/env node',
'// The convention this tree writes: `function selfTest() {` at column 0.',
'const FIXTURE = `',
'function selfTest() {',
" console.log('a fixture the gate scans, not a definition');",
'}',
'`;',
'const holder = { run: function selfTest() { return FIXTURE.length; } };',
'function selfTest() {',
' const failures = [];',
" if (holder.run() < 1) failures.push('the fixture text went missing');",
" if (failures.length) { console.error(failures.join(String.fromCharCode(10))); process.exit(1); }",
" console.log('fixture self-test: 1 case passes');",
'}',
"if (process.argv.includes('--self-test')) selfTest();",
'',
].join('\n');

/** The definition-shaped text the three decoys and the real definition share. */
const DECOY_ANCHOR_TEXT = 'function selfTest() {';

/**
* The anchor an UNMASKED, UNANCHORED first match takes -- the pre-#14963 rule,
* kept here as the thing the controls below measure against rather than as a
* second implementation of anything: `injectEarlyReturn` never uses it.
*/
const NAIVE_ANCHOR = /(?:async\s+)?function\s+selfTest\s*\([^)]*\)\s*\{/;

/**
* Both instruments, against both directions. Returns the failures; the caller
* refuses on any. Nothing here reads the repo, so a control failure is always
Expand Down Expand Up @@ -599,6 +702,41 @@ export function runControls() {
say(DISPATCH.test(NON_DISPATCH_MENTION_GATE),
'CONTROL FIXTURE INVALID: the mention fixture does not match even UNMASKED, so the verdict above says nothing about comments being masked away');

// ⛔ ... and the population criterion must keep reading COMMENT-masked source.
// The mask the injection ANCHOR needs blanks literals too, and every dispatch
// in this tree names the flag with a string literal -- so reading the census
// through that one would not classify a single file generously, it would empty
// the population, taking this instrument's own control fixtures (deliberately
// strings) with it. The two masks are required to answer this OPPOSITELY.
say(!DISPATCH.test(maskCommentsAndLiterals(HOLED_GATE)),
'CONTROL FAILED: the code-only mask that the injection anchor reads still admits a dispatch to the population; the two masks no longer answer differently, and whichever of them the census ends up reading, one of the two questions is being answered with the wrong text');

// The ANCHOR, against every text that reads like a definition without being
// one. What is at stake is not a classification but a WRONG READING: an
// injection into a fixture string makes the copy a SyntaxError, whose non-zero
// exit and stack trace this file's verdict scores HELD (#14963).
const decoyFlags = scanSource(DECOY_ANCHOR_GATE);
const commentDecoy = DECOY_ANCHOR_GATE.indexOf(DECOY_ANCHOR_TEXT);
const literalDecoy = DECOY_ANCHOR_GATE.indexOf(DECOY_ANCHOR_TEXT, commentDecoy + 1);
const midLineDecoy = DECOY_ANCHOR_GATE.indexOf('function selfTest() { return FIXTURE.length');
const realDef = DECOY_ANCHOR_GATE.indexOf('\nfunction selfTest() {\n const failures') + 1;
say(commentDecoy >= 0 && literalDecoy > commentDecoy && midLineDecoy > literalDecoy && realDef > midLineDecoy,
'CONTROL FIXTURE INVALID: the three decoys no longer all stand AHEAD of the real definition, so a first-match anchor would reach the definition however it was spelled and every verdict below passes for the wrong reason');
say(decoyFlags.comment[commentDecoy] === 1,
'CONTROL FIXTURE INVALID: the first decoy is not comment content, so it no longer tests the comment half of the mask');
say(decoyFlags.literal[literalDecoy] === 1,
'CONTROL FIXTURE INVALID: the second decoy is not literal content, so it no longer tests the string/template half of the mask -- the half whose failure produces a SyntaxError and a false HELD');
say(decoyFlags.comment[midLineDecoy] === 0 && decoyFlags.literal[midLineDecoy] === 0,
'CONTROL FIXTURE INVALID: the third decoy is masked away as comment or literal, so it tests the mask a second time instead of the LINE-START rule it is there for');
say(DECOY_ANCHOR_GATE.search(NAIVE_ANCHOR) === commentDecoy,
'CONTROL FIXTURE INVALID: an unmasked first-match anchor no longer lands on a decoy, so the MASK is not what the anchor verdict below is reading');
say(maskCommentsAndLiterals(DECOY_ANCHOR_GATE).search(NAIVE_ANCHOR) === midLineDecoy,
'CONTROL FIXTURE INVALID: masking alone no longer lands on the mid-line decoy, so the LINE-START rule is not what the anchor verdict below is reading -- masking would be carrying it on its own');
const decoyInjected = injectEarlyReturn(DECOY_ANCHOR_GATE, 'selfTest');
say(decoyInjected !== null
&& decoyInjected.includes(`function selfTest() {\n return; /*${PROBE_MARKER}*/\n\n const failures = [];`),
'ANCHOR CONTROL FAILED: the early return was not injected at the REAL definition; a text that merely READS like one -- in a comment, in a fixture string, or mid-line in code -- was preferred over the function the dispatch calls');

// Instrument 1, both directions.
say(classifyFloor(maskComments(HOLED_GATE)) === 'NONE',
'POSITIVE CONTROL FAILED: a self-test deciding success by failures.length alone was not classified NONE');
Expand Down Expand Up @@ -637,18 +775,21 @@ export function runControls() {
const sound = join(dir, 'sound-gate.mjs');
const accident = join(dir, 'accident-gate.mjs');
const unrunnable = join(dir, 'unrunnable-gate.mjs');
const decoy = join(dir, 'decoy-anchor-gate.mjs');
const helper = join(dir, 'helper-handshake-gate.mjs');
const helperHoled = join(dir, 'helper-handshake-gate-holed.mjs');
writeFileSync(holed, HOLED_GATE);
writeFileSync(sound, SOUND_GATE);
writeFileSync(accident, ACCIDENT_GATE);
writeFileSync(unrunnable, UNRUNNABLE_GATE);
writeFileSync(decoy, DECOY_ANCHOR_GATE);
writeFileSync(helper, HELPER_HANDSHAKE_GATE);
writeFileSync(helperHoled, HELPER_HANDSHAKE_GATE_HOLED);
const h = probeEarlyReturn(holed, 'selfTest');
const s = probeEarlyReturn(sound, 'selfTest');
const a = probeEarlyReturn(accident, 'runSelfTest');
const u = probeEarlyReturn(unrunnable, 'selfTest');
const dc = probeEarlyReturn(decoy, 'selfTest');
const hh = probeEarlyReturn(helper, 'selfTest');
const hhHoled = probeEarlyReturn(helperHoled, 'selfTest');
say(h.verdict === 'DEFEATED',
Expand Down Expand Up @@ -676,6 +817,14 @@ export function runControls() {
// being red, would satisfy the verdict above while testing nothing.
say(u.baselineExit !== 0 && u.baselineBytes > 0 && u.baselineHead !== '',
`POSITIVE CONTROL FAILED: the unrunnable fixture no longer produces the measured shape (baseline exit ${u.baselineExit}, ${u.baselineBytes} byte(s)); the NOT MEASURED verdict above would then be passing for the wrong reason`);
// The anchor, ON DISK. The static assertions above read WHERE the injection
// went; this one reads what the copy then DID -- so it also says the copy
// parsed and ran. Anchored on the fixture string instead, the copy dies in
// the parser: non-zero exit, a stack trace on stderr, verdict HELD.
say(dc.verdict === 'DEFEATED',
`POSITIVE CONTROL FAILED: a known-holed gate whose real definition is preceded by three definition-shaped decoys was read as ${dc.verdict} (${dc.why ?? ''}); anchored on the fixture string this reads HELD, a hold awarded for the probe breaking its own copy`);
say(dc.mutatedBytes === 0,
`POSITIVE CONTROL FAILED: the decoy gate printed ${dc.mutatedBytes} byte(s) when defeated; the copy anchored on the real definition returns before its verdict line and says NOTHING, so anything printed here is the copy failing rather than the gate being silent`);
// The HELPER handshake spelling, both directions, differing by ONE line: the
// dispatch asking `requireReachedVerdict`. Until this fixture the probe had
// read that shape in NEITHER direction -- its single carrier in the tree is
Expand Down Expand Up @@ -707,7 +856,7 @@ export function runControls() {
// ---------------------------------------------------------------------------

/**
* ⛔ SHRINK-ONLY. The nine files whose entry cannot be resolved mechanically,
* ⛔ SHRINK-ONLY. The ten files whose entry cannot be resolved mechanically,
* resolved by READING the dispatch site (the ruling's A2.1: the grep is an
* entry point, not the criterion). A `null` entry is NOT MEASURED with the
* stated reason -- never a quiet pass, and never a guess.
Expand All @@ -726,15 +875,24 @@ export const ENTRY_BY_HAND = Object.freeze({
// `runSelfTest()` is what the dispatch calls. Probing the inner one records a
// TypeError as a handshake; probing `runSelfTest` reads the real one (#14842).
'scripts/check-workspace-manifest-cycles.mjs': 'runSelfTest',
// The dispatch calls FOUR self-test functions and combines their statuses;
// The dispatch calls SIX self-test functions and combines their statuses;
// there is no single entry an early return leaves, so a one-function probe
// measures a sub-battery and reads a downstream crash as a handshake.
'scripts/check-platform-checklist.mjs': null,
// The self-test is an inline top-level block calling several helpers.
'scripts/check-regen-pending.mjs': null,
// Injecting into this file produces a SyntaxError (the anchor lands inside a
// template literal), so no run of it measures anything.
'scripts/pm/dispatch-gates.mjs': null,
// Four self-test-shaped names in raw source (`selfTest`, `selfTestOnlyCallables`
// and two inside fixtures), so the entry is hand-read -- but for THIS FILE's
// reason. The row was a `null` until #14963, on the claim that injecting here
// can only produce a SyntaxError: the anchor read raw source, where a docblock
// sentence and then a FIXTURE STRING stand ahead of the real definition. That
// was the INSTRUMENT's limit recorded as this file's property. Anchored on the
// definition the copy parses and runs (measured: exit 1, `selfTest() returned
// without reaching its verdict`). The probe still reads NOT MEASURED here and
// now says why itself -- `baseline run failed (exit 1)`, because it writes its
// copy under `scripts/`, where this gate's own single-site sweep finds the
// near-duplicate and refuses (#15515). A separate card, not worked around here.
'scripts/pm/dispatch-gates.mjs': 'selfTest',
});

/**
Expand Down
Loading