From df74ef1bf1de6bc1e8620adbdef748a3060f392c Mon Sep 17 00:00:00 2001 From: InauguralPhysicist Date: Wed, 23 Sep 2026 15:35:44 -0500 Subject: [PATCH 1/3] ci: move the GitHub-state gates out of the test suite into issue-triage (#1275) tools/issue_labels_check.sh and tools/roadmap_check.sh read live repository state, so as suite section [99zd] their verdict changed without any commit: an unrelated issue filed without labels turned PR #1273 red (#1279) and ejected #1282 from the merge queue for two hours (#1168). They now run only in .github/workflows/issue-triage.yml, which already ran both daily with its own pinned copies of each contract. It gains a path-filtered pull_request trigger (the gates, gh_probe.sh, ROADMAP.md, the workflow), so a PR that edits a gate still runs it, as an advisory check that is never required. [99zd] keeps the one part that reads only the tree, workflow_yaml_check.sh (-450 runner lines). With no GitHub caller left in CI's suite lanes, the Linux jobs drop their issues: read token and the dev image drops its pinned gh tarball. docs/CI.md's section is rewritten around where the gates run now (-194 lines of suite-caller history); the tools' headers name the one remaining caller. Floors lowered deliberately: child_exit_check 126 -> 122 (the section's six child invocations became two), docs-claims PATHS|docs/CI.md 68 -> 65. Seven section_plan skip waivers for deleted lines are removed. Co-Authored-By: Claude Opus 5.5 (1M context) --- .devcontainer/Dockerfile | 47 +-- .github/workflows/ci.yml | 18 +- .github/workflows/issue-triage.yml | 11 + CHANGELOG.md | 20 +- docs/CI.md | 216 ++----------- tests/run_all_tests.sh | 469 +---------------------------- tools/child_exit_check.sh | 2 +- tools/docs_claims_populations.txt | 4 +- tools/gh_probe.sh | 2 +- tools/issue_labels_check.sh | 8 +- tools/roadmap_check.sh | 15 +- tools/section_plan.sh | 7 - 12 files changed, 83 insertions(+), 736 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 5ef978c9..d95da638 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -41,8 +41,7 @@ ENV DEBIAN_FRONTEND=noninteractive # GREEN on the merge-gating lane (critic r8 ledger # 3, measured on CI job 106492225359). # gdb -> the JIT debugging workflow (see CLAUDE.md) -# curl -> exercising the http extension by hand; ALSO how `gh` -# is fetched below +# curl -> exercising the http extension by hand # SDL2 is deliberately absent: `make gfx` dlopen's it at runtime and compiles # against typedefs only, so no -dev headers are needed to build it. RUN apt-get update \ @@ -60,48 +59,4 @@ RUN apt-get update \ curl \ && rm -rf /var/lib/apt/lists/* -# `gh` — the GitHub CLI, the LIVE arm of suite section [99zd]. -# -# BOUGHT 2026-09-21 (round-4 blind critic, Fable; #1207): arms (b) and (c) of -# tools/roadmap_check.sh — the milestone-set mirror and the reference resolver, -# the whole point of the issue — were executed against GitHub by NO automated -# caller anywhere. This image had no `gh`, so the Linux legs SKIPPED BY NAME; -# the macOS runner has `gh` with no credentials, so it skipped too; and the -# daily audit ran only the labels gate. "CI 31/31 green" measured the -# structural arm alone while the section reported `population lines 3/3`. A -# mirror nobody checks is the failure #1207 was filed about. -# -# THE RELEASE TARBALL, not GitHub's apt repository: one pinned version with a -# checksum this Dockerfile verifies, rather than a third-party apt source whose -# key rotation is another thing that can silently change what CI runs. The -# version and the per-architecture SHA-256s are a deliberate edit; `gh --version` is asserted at -# build time so a bad extraction fails HERE and not as a mystery skip in the -# suite. -# ONE ARCHITECTURE WAS HARD-CODED, AND arm64 IS NOT HYPOTHETICAL. Bought -# 2026-09-21 (third critic, `/code-review 1226 medium`, finding 9): the -# tarball name, the checksum and the extracted directory all said -# `linux_amd64`, so on an arm64 Docker host (an Apple-silicon dev machine, a -# Graviton runner) the checksum fails and the WHOLE image build dies — not the -# gh step, the image. The architecture is now asked of dpkg and each one -# carries its own pinned checksum, both taken from the release's own -# `gh_${GH_VERSION}_checksums.txt`; an architecture with no pin here fails by -# name rather than by checksum mismatch. -ARG GH_VERSION=2.101.0 -ARG GH_SHA256_AMD64=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8 -ARG GH_SHA256_ARM64=b57e8063f18862647c9d22727c32e9da1b963f8bf9db648fe123a6975695640f -RUN set -eu \ - && arch="$(dpkg --print-architecture)" \ - && case "$arch" in \ - amd64) sha="${GH_SHA256_AMD64}" ;; \ - arm64) sha="${GH_SHA256_ARM64}" ;; \ - *) echo "gh ${GH_VERSION}: no pinned checksum for dpkg architecture '$arch' — add one from gh_${GH_VERSION}_checksums.txt rather than letting the build fail on a mismatch" >&2; exit 1 ;; \ - esac \ - && curl -fsSL -o /tmp/gh.tar.gz \ - "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_${arch}.tar.gz" \ - && echo "${sha} /tmp/gh.tar.gz" | sha256sum -c - \ - && tar -xzf /tmp/gh.tar.gz -C /tmp \ - && install -m 0755 "/tmp/gh_${GH_VERSION}_linux_${arch}/bin/gh" /usr/local/bin/gh \ - && rm -rf /tmp/gh.tar.gz "/tmp/gh_${GH_VERSION}_linux_${arch}" \ - && gh --version - WORKDIR /workspaces/EigenScript diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 70cd2f29..994ea96e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -417,16 +417,6 @@ jobs: permissions: contents: read packages: read - # THE LANE THAT HOLDS A TOKEN. Suite section [99zd] runs - # tools/roadmap_check.sh and tools/issue_labels_check.sh; their GitHub - # arms — the milestone-set mirror and the reference resolver, which are - # the whole point of #1207 — had no lane anywhere that could do anything - # but SKIP BY NAME (round-4 blind critic, Fable). `issues: read` is - # enough: milestones and issues of this repository are read through it, - # and the references arm resolves public repositories in the same - # organisation. The token is exported on the suite step below, and the - # section's own probe (tools/gh_probe.sh) makes a skip on this lane RED. - issues: read container: image: ${{ needs.dev-image.outputs.image }} credentials: @@ -485,11 +475,6 @@ jobs: if: matrix.cc == 'gcc' || github.event_name != 'pull_request' env: EIGS_SKIP_WERROR_AUDIT: 1 - # [99zd]'s live arms. Without this the roadmap gate's milestone and - # reference walks skip by name on every lane, which is exactly the - # round-3 hole. With it, [99zd] REQUIRES `milestones=gh-api:… - # refs=gh-api:… resolved=N skipped=0` and a `gh-api:` labels line. - GH_TOKEN: ${{ github.token }} run: cd tests && bash run_all_tests.sh # clang on a PR: the value of this leg is the BUILD (-Werror fires at @@ -502,7 +487,6 @@ jobs: env: EIGS_SUITE_SECTIONS: core EIGS_SKIP_WERROR_AUDIT: 1 - GH_TOKEN: ${{ github.token }} run: cd tests && bash run_all_tests.sh # Cross-repo observer corpus (#262): real observer-using programs @@ -681,7 +665,7 @@ jobs: set -u python3 -m pip install --user --break-system-packages pyyaml \ || python3 -m pip install --user pyyaml \ - || echo "PyYAML install failed; [99zd] arm (b) will SKIP BY NAME" + || echo "PyYAML install failed; the workflow-YAML load arm will SKIP BY NAME" python3 -c 'import yaml; print("PyYAML", yaml.__version__)' \ || echo "PyYAML is not importable on this runner" diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 3375f96b..1ad973d0 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -40,6 +40,17 @@ on: # behind everyone else's midnight cron on the shared runner pool. - cron: '23 5 * * *' workflow_dispatch: + # These gates read LIVE repository state, so they are not in the test suite + # (#1275): an unrelated unlabelled issue failed every PR there and ejected + # queued merges (#1279, #1168). A PR that changes a gate or ROADMAP.md still + # runs them here, as an advisory check that is never required. + pull_request: + paths: + - ROADMAP.md + - tools/issue_labels_check.sh + - tools/roadmap_check.sh + - tools/gh_probe.sh + - .github/workflows/issue-triage.yml permissions: contents: read diff --git a/CHANGELOG.md b/CHANGELOG.md index b5f046c5..b35fe1d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1386,6 +1386,17 @@ All notable changes to EigenScript are documented here. ### Changed +- **The GitHub-state gates left the test suite (#1275).** + `tools/issue_labels_check.sh` and `tools/roadmap_check.sh` read live + repository state, so as suite section `[99zd]` an unrelated issue filed + without labels turned every pull request red and ejected queued merges + (#1279, #1168). They now run only in `.github/workflows/issue-triage.yml`: + daily, on demand, and as an advisory, never-required check on pull requests + that change them, `tools/gh_probe.sh` or `ROADMAP.md`. `[99zd]` keeps the + one part that reads only the tree, `tools/workflow_yaml_check.sh`. The + Linux suite lanes no longer hold an `issues: read` token, and the dev image + no longer installs `gh`. + - **Three front-door claims that only EXECUTION could refute (#1209, #1210, #1211).** Each was stated in `docs/llms.txt` — the file every agent primes on — and each is now stated correctly with an executed fence beside it. @@ -1425,8 +1436,9 @@ All notable changes to EigenScript are documented here. first circulated is not reproducible from the API, and the figure of record is the gate's first real run after the sweep, `examined=35 missing=0`). `.github/workflows/issue-triage.yml` labels a new issue `needs-triage` when - it arrives without an `area:` and runs the audit daily. Both gates are suite - section `[99zd]` with pinned selftest case counts. + it arrives without an `area:` and runs the audit daily. Both gates ran as + suite section `[99zd]` with pinned selftest case counts until #1275 moved + them out of the suite (see below). Round 2 closed what the blind critics found in round 1. A SUCCESSFUL EXIT IS NOT A MEASUREMENT: both callers accepted `tools/issue_labels_check.sh` @@ -1512,8 +1524,8 @@ All notable changes to EigenScript are documented here. AND prints `[99zd] live arms: SKIPPED (no gh credentials on this lane)` so the log says which lane measured what. A lane that EXPORTS a token and cannot use it is red on an independent cross-check. `.devcontainer/Dockerfile` - installs `gh` from a pinned, checksummed release tarball, `ci.yml`'s - `linux / gcc` job exports `GH_TOKEN` with `issues: read`, and the daily + installed `gh` and `ci.yml`'s `linux / gcc` job exported `GH_TOKEN` with + `issues: read` (both removed again in #1275), and the daily `issue-triage.yml` audit runs the roadmap gate too, with a pin that admits no skip at all. diff --git a/docs/CI.md b/docs/CI.md index 54708d06..2c7f073a 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -209,6 +209,22 @@ Two things that were kept by memory are now kept by a gate, both bought the same day (2026-09-21, #1207/#1155 and the maintainer's "we aren't labeling issues"). +**Where they run: not in the test suite.** `tools/issue_labels_check.sh` and +`tools/roadmap_check.sh` read LIVE repository state (open issues, milestones, +other repositories), so their verdict can change without any commit. While +they ran as a suite section, an unrelated issue filed without labels turned +every pull request red and ejected queued merges (#1279, #1168). They run in +`.github/workflows/issue-triage.yml` instead (#1275): daily, on +`workflow_dispatch`, and on any pull request that changes one of them, +`tools/gh_probe.sh`, `ROADMAP.md` or the workflow itself. That PR run is +advisory and never a required check. Its audit step holds its OWN literal copy +of each gate's population regex and selftest case count, asserts the gate's +output against that copy, and asserts the gate's `--contract` equals it; a +difference is red by name and never auto-adopted, and a named skip is red +because that lane exists to make the API call. Run either gate locally with an +authenticated `gh`: `bash tools/issue_labels_check.sh`, +`bash tools/roadmap_check.sh` (each also takes `--selftest`). + **Issue labels.** The scheme on the repository is `area:` (runtime-vm, jit, concurrency, observer, memory, trace-tape, packages, http, gfx, embed, docs, ci, gates, lint-tooling, consumer, aot, stdlib), a KIND @@ -219,8 +235,7 @@ gfx, embed, docs, ci, gates, lint-tooling, consumer, aot, stdlib), a KIND a kind.** `tools/issue_labels_check.sh` enumerates every open ISSUE (never a pull request), prints `examined=N missing=M` with the offending numbers, and fails when M > 0 **or when N == 0** — an empty enumeration satisfies "nothing -is missing" without checking anything. Without `gh` it SKIPs by name; it never -turns a missing credential into a pass. `.github/workflows/issue-triage.yml` +is missing" without checking anything. `.github/workflows/issue-triage.yml` runs it daily and, on `issues: [opened, reopened]`, puts `needs-triage` on anything that arrives without an `area:` label. The backlog was essentially unlabelled before the sweep; the often-quoted "33 of 36" census is not @@ -259,108 +274,6 @@ hundred lines away, had carried `-c safe.directory='*'` for months. One workaround, every git call — and the caller that probes with the flag is what makes a gate that cannot agree with it red by name. -**`gh api --paginate` returns ONE array, so the labels gate stopped splicing -one.** `issue_labels_check.sh` carried `sed 's/^\]\[/,/' | tr -d '\n'` to join -per-page arrays. Measured with `per_page=3` over four pages: `gh` merges them -itself and there is no `][` seam at all — and on a `gh` that DID concatenate -raw bodies the seam would sit mid-line, where a `^`-anchored sed cannot reach -it. A dead repair that reads as a live one is worse than none. The splice is -gone; the Python classifier accepts EITHER a merged array or an array of pages -and refuses anything else by name, with a two-page fixture whose unlabelled -issue is on the SECOND page. `--slurp` would state the shape explicitly and is -deliberately not used: `gh` 2.45.0 (Ubuntu's package, what the dev box has) -answers `unknown flag: --slurp`, which would turn the gate into a named SKIP — -red at a live caller — on every lane whose `gh` predates the flag. - -**The caller holds its own copy of every pin.** Round 1 accepted the gate -gutted to `exit 0` — `exit=0 output=''` passed both boundaries — so round 2 had -each gate publish a contract (`--contract`: the population line it promises, -`POPULATION_RE`, and how many planted faults its selftest runs, -`SELFTEST_CASES`) and had both callers READ it. That made the thing being -policed supply the yardstick: `POPULATION_RE=examined=|.*` admitted empty -output, and a gate that deleted its plants and lowered `SELFTEST_CASES` passed -the daily lane. Round 3 keeps TWO copies, kept equal by a test. Each caller — -`[99zd]` in `tests/run_all_tests.sh` and the audit step in -`.github/workflows/issue-triage.yml` — holds the population regex and the -selftest case count as LITERALS, asserts the gate's output against its own -copy, and separately asserts that the gate's `--contract` equals that copy -verbatim. A difference is red by name ("gate contract changed; re-pin the -caller deliberately") and is never auto-adopted. Three consequences worth -knowing: - -* the population count group is `[1-9][0-9]*` and exactly ONE matching line is - required, so an empty enumeration and a duplicated line are both red; -* the source is a machine-readable token (`gh-api:` / `fixture:` / - `skipped:`), because both callers used to accept `(source: fixture ...)` as a - live measurement — their regex stopped before `(source:`. The pins admit NO - `fixture:` source anywhere. Whether they also require a LIVE `gh-api:` token - depends on the lane, and each caller decides that FOR ITSELF, never from the - gate's claim: each runs the shared probe `tools/gh_probe.sh` (the same code - the gates run), and when the probe reaches GitHub the pin requires - `milestones=gh-api:… refs=gh-api:… resolved=N skipped=0`, a `gh-api:` labels - line with no `SKIPPED BY NAME` alternative, and — when the caller's own - `python3 -c 'import yaml'` succeeds — `loader=pyyaml`. When the probe does - NOT reach GitHub the named skip is accepted and the caller prints its own - line (`[99zd] live arms: SKIPPED (no gh credentials on this lane)`), so the - log says which lanes measured what. Round 3 stated flatly that "the pinned - regexes require a LIVE source token"; that was false for two of the three - pins, which is what let a gate whose GitHub arms never ran pass on an - authenticated box; -* each caller counts its own work. Every assertion that reached a verdict the - caller accepts increments a witness, and a final check compares the witnesses - and the caller's check count with pinned literals — so deleting or - short-circuiting a check changes RESULTS instead of quietly measuring less. - -A successful exit is not a measurement, and neither is a contract the gate -wrote for itself. - -**A declared-but-empty token is a declared token.** Each caller cross-checks -the probe against an INDEPENDENT signal — does this lane DECLARE a credential? -— and a lane that declares one and cannot reach GitHub is red by name rather -than allowed its skip. Round 4 asked that question with `[ -n "$GH_TOKEN" ]`, -so a lane exporting `GH_TOKEN=""` declared nothing by it. That is not a -hypothetical shape: `env: GH_TOKEN: ${{ secrets.TYPO }}` exports an EMPTY -string, not nothing, and a secret that is missing, misspelled or scoped away -produces exactly it. With an empty token the probe reported -`gh-unauthenticated`, every GitHub-facing arm took its named skip, and the -suite caller's eleventh check printed "this lane declares no token" and passed -**11/11** on a lane that measured nothing (round-5 blind critic, Astra). -`gh_probe_token_declared` now tests PRESENCE (`${GH_TOKEN+x}`): an empty export -is a credential this lane was written to hold and cannot use, which is the -finding. An UNSET token — the dev box's keyring login, the macOS runner, the -sanitizer shards — is the only shape that still permits the named skip. - -**What the caller can and cannot prove.** A caller verifies that a gate printed -a population line it could only have produced by running its live arm ON THIS -LANE (token-pinned, against the caller's own probe), and that the gate's -selftest ran with the pinned count. A gate that FABRICATES its own output — -printing the population line and the selftest line with no work behind them — -is outside the caller's power to detect: a forged receipt reads exactly like a -true one, and a round-4 blind critic scored 10/10 against print-only stubs. That -is what the blind-critic rounds and each gate's own transverse mutations are -for. The caller's job is to make the receipt SPECIFIC enough that forging it is -a deliberate lie about a checkable thing, not to render forgery impossible. - -**Which lanes run the live arms.** `ci.yml`'s `linux / gcc` job (and `clang` on -a push) runs the full suite inside the dev image, which now installs `gh` from -a pinned, checksummed release tarball (`.devcontainer/Dockerfile`), and its -suite step exports `GH_TOKEN: ${{ github.token }}` with `issues: read` on the -job. `.github/workflows/issue-triage.yml`'s daily audit runs BOTH the labels -gate and `tools/roadmap_check.sh`, with a pin that admits no skip at all — -that lane exists to make the API call. Every other lane (macOS, the sanitizer -shards) declares no token, skips the GitHub arms BY NAME, and says so on the -caller's own line. **The sanitizer shards could hold a token and deliberately -do not**: all three run the same dev image as `linux / gcc`, so `gh` is there -and `${{ github.token }}` would work — they would simply add three more -`gh api` walks per push (a milestone read, an organisation listing and a -reference walk each) for an answer `linux / gcc` and `linux / clang` have -already produced on that same commit. The live arms are about the CONTENT of -ROADMAP.md, which is identical across shards; running them once per push is the -measurement, and running them four times is rate limit. Before round 4 no lane anywhere could do anything but skip: -the milestone mirror and the reference resolver — the whole point of #1207 — -were executed against GitHub by nothing, while `[99zd]` reported -`population lines 3/3`. - **ROADMAP.md.** `tools/roadmap_check.sh` refuses (a) any `- [ ]`/`- [x]`/`- [~]` line anywhere in the file and anything other than exactly one table, inside `## Milestones`, with five cells and a known status per row; and (b) — with @@ -391,93 +304,9 @@ by the default. Arm (a) never skips. Arm (c) also REFUSES a bare `#N` in a cell that also carries a qualified `Repo#M`: M9's row read "Tidepool#43 and #59", the bare `#59` silently resolved against EigenScript (a real, closed PR), and the row was green for a reference -it does not mean. - -**`KNOWN_REPOS` is verified once per run, and a private repository is not -evidence.** Round 4's list named `EigenKB`, which **does not exist** — a 404 on -every token, including the organisation's most privileged one — and arm (c) -mapped a repository-level 404 to "this token cannot read the repository", a -statement about the run, so `EigenKB#1` in the roadmap SKIPPED BY NAME and the -gate printed `OK … skipped=1` (round-5 blind critic, Fable). A membership list -nothing verifies certifies whatever is typed into it. Arm (c) now makes ONE -call, `gh api orgs//repos --paginate` (one page; how many rows come back -depends on the token — a repo-scoped `${{ github.token }}` sees the public -half only), and that -listing is the discriminator: name absent from a SUCCESSFUL listing → the -repository **does not exist**, red by name, `KNOWN_REPOS is stale`; name -present → `.private` decides which list it belongs on; listing fails or returns -empty → nothing is decidable and the verification **SKIPs by name** (the -reference walk still runs, exactly as before). - -**The classifier fails CLOSED, and "public" is a positive fact the listing has -to state.** Round 6 asked `[ "$priv" = "true" ]` and called everything else -public — so a listing whose rows carry no `.private` at all (a projection, a -proxy, an API change, a `jq` that answered `null`) certified every entry as -public and printed `repos=verified:13` with not one explicit `false` in it. -Measured with a 13-row null fixture: the gate printed the byte-identical OK -line and `[99zd]` read 11/11 (round-6 blind critics, Astra and Fable, -converging). Now `false` is public, `true` is private, and **anything else is -UNKNOWN VISIBILITY** — red by name ("the organisation listing carries no -visibility for `/`; refusing to certify it public") with the run's -token set to `repos=skipped:visibility-unknown:N`, which the live pin refuses -at both callers. The token the shell classifies also carries its TYPE, because -`(.private|tostring)` maps the JSON **string** `"false"` onto the boolean; a -non-boolean arrives as `non-boolean:` and cannot masquerade. - -**"Absent" means absent-or-private when the view is public-only.** A -repo-scoped `${{ github.token }}` sees only the organisation's public half, so -on that lane a listing naming no private repository at all cannot tell a -DELETED repository from one that was turned private. The verdict is red either -way; the diagnosis changes — "absent from a public-only listing: deleted, -renamed, or now private — check with a token that can see private -repositories" — because sending a maintainer to look for a deleted repository -is the same false accusation in the other direction (round-6 blind critics, -ledger 4). - -**A table row is not a separator because it contains `---`.** The row walk -skipped any row containing that substring anywhere, not only the header -separator, so a data row whose DONE clause read `never --- see the vetoes` was -never counted in `examined=` and never checked for cells, status or milestone -number (third critic, `/code-review 1226 medium`). One anchored regex, -`RC_SEP_RE`, now serves the separator COUNT, the section-placement check and -the walk — two spellings of one rule is how they disagree. - -**Which of the listing outcomes above happened is on the OK line**, as -`repos=verified:N` or `repos=skipped:` (round-5 blind -critic, Fable): until round 6 the verification left no trace there, so a run -whose listing 403'd, came back empty or was gutted printed a line -BYTE-IDENTICAL to a verified one and `[99zd]` passed 11/11 on a token-holding -lane. The contract admits both tokens; both callers require -`repos=verified:[1-9][0-9]*` once they have established for themselves that -GitHub is reachable. A `KNOWN_REPOS` entry the listing marks PRIVATE is told -apart from one that is ABSENT — "is now private (move it to PRIVATE_REPOS)" -versus "does not exist (KNOWN_REPOS is stale)" — and the selftest asserts each -plant's own diagnosis, not merely that arm (c) went red (round-5 blind critic, -Astra). A repo-scoped -`${{ github.token }}` sees only the organisation's public repositories, so for -a `PRIVATE_REPOS` entry "absent" and "private" are the same answer and both are -fine; an entry that shows up **public** is the stale direction and is red too. -`ROADMAP.md` is a PUBLIC document, so a citation its readers cannot open is not -evidence: `EigenOS`, `eigen-site`, `DeslanStudio` and `iLambdaAi` (measured -2026-09-21: `.private` is true on all four) moved out of `KNOWN_REPOS` into -`PRIVATE_REPOS`, which keeps them recognisable so that citing one is red for -its REAL reason ("private repository is not evidence in a public roadmap") -rather than red as an unrecognised name. Under round 4 those citations RESOLVED -on a maintainer's token and were counted as evidence, and would have been -`skipped=1` — and therefore red at the token-holding daily lane, for the wrong -reason — under `${{ github.token }}`. The selftest drives all of this through -an organisation-listing fixture, offline: a missing entry, a private entry, a -cited private repository, and the CONTROL that makes the discriminator load- -bearing — the same missing entry with the listing UNREADABLE is **green**, so -gutting the discriminator is red in one direction and silent in the other, and -the pair catches both. Round 7 added four more: a listing with no visibility -at all, a `.private` that is a string rather than a boolean, an entry absent -from a public-only view, and a data row whose cell contains `---`. The case -count is PUBLISHED by `--selftest`'s summary line and by `--contract`, and the -**two** callers — `[99zd]` in `tests/run_all_tests.sh` and the roadmap step in -`.github/workflows/issue-triage.yml` — each pin their own copy of it, so a -count that moves is a deliberate edit in three places rather than a silent -adoption. +it does not mean. Each gate's header documents the rest: repository visibility +(`KNOWN_REPOS` / `PRIVATE_REPOS`), what each listing outcome prints, and the +selftest's planted faults. The old file was a checkbox pile, most of it historical highlights under `## Completed`, so every counter of "roadmap items" was counting the past. @@ -510,9 +339,8 @@ runs (`python3-yaml` in `.devcontainer/Dockerfile` for every Linux leg, a setup step on the macOS lane); when it is absent anyway, the CALLER probes for PyYAML itself and allows exactly the pinned named-skip count, for that gate alone. -All three tools carry a planted-fault `--selftest` with a pinned case count and -a `--contract`, and the suite runs the live pass, the contract and the selftest -of each as `[99zd]`. +The suite runs `tools/workflow_yaml_check.sh` and its selftest as `[99zd]`, +the only part of that section that reads nothing but the tree. ## Main lane (the merge queue, then push to `main`) — the full matrix diff --git a/tests/run_all_tests.sh b/tests/run_all_tests.sh index a076f27d..420fadd2 100755 --- a/tests/run_all_tests.sh +++ b/tests/run_all_tests.sh @@ -7068,467 +7068,30 @@ else fi echo "" -# [99zd] ROADMAP.md is a MILESTONE SET, and issue labels are enforced rather -# than remembered. Both bought 2026-09-21 (#1207/#1155, and the maintainer's -# "we aren't labeling issues"): ROADMAP.md was a checkbox pile, most of it -# historical highlights under `## Completed`, so every counter of "roadmap -# items" was counting the past (ROADMAP.md's header derives both counts beside -# the commands that produce them; tools/docs_claims_check.sh executes those -# commands), and the open backlog was unlabelled — the labels gate's first real -# run after the hand sweep read `examined=35 missing=0`. -# -# A SUCCESSFUL EXIT IS NOT A MEASUREMENT (mechanical-gates §121). Round 1 of -# this section took `exit 0` as a pass: Astra removed ONLY the live-data walk -# from tools/roadmap_check.sh, left its fixture selftest intact, and this -# section read TOTAL=4 PASS=4. -# -# ROUND 2 ADDED A CONTRACT AND THEN TRUSTED IT — the wrong invariant. -# Round 2 had each gate publish `--contract` (its population regex and its -# selftest case count) and had this caller read BOTH from the gate. So the -# thing being policed supplied the yardstick: `POPULATION_RE=examined=|.*` -# admitted empty output, a contract permitting `examined=0` passed, and a gate -# with five plants deleted plus `SELFTEST_CASES=1` passed the daily workflow -# (round-2 blind critics, Astra checks 2 and 7). "One regex per gate" was the -# wrong invariant. -# -# ROUND 3 — TWO COPIES, KEPT EQUAL BY A TEST. Every pin below is a LITERAL in -# this caller. The caller asserts the gate's output against ITS OWN copy, and a -# SEPARATE check asserts the gate's `--contract` equals this copy verbatim. A -# drift is red BY NAME and is never auto-adopted; the caller is the independent -# witness, not a reader of the thing it polices. Three further round-3 rules: -# * the population count group is `[1-9][0-9]*` — zero is never a population — -# and EXACTLY ONE population line is required, so a duplicated line is red; -# * the pinned regexes require a LIVE source token (`gh-api:`), so a -# fixture-sourced run (`(source: fixture ...)`, which both callers accepted -# in round 2 because their regex stopped before `(source:`) is red by name; -# * the round-2 vacuity guard (`case "$re" in *examined=*`) is GONE. It was a -# substring test that `POPULATION_RE='examined='` satisfied. The caller no -# longer consumes the gate's regex at all, so its own pinned regex IS the -# guard. -# -# The section also counts its OWN work. Each assertion that actually reached a -# verdict the caller accepts increments a witness, and the last check compares -# the three witnesses and the section's check count with pinned literals — so -# deleting or short-circuiting a check changes RESULTS instead of silently -# measuring less. -# -# The same section also loads every workflow file (tools/workflow_yaml_check.sh): -# round 1 shipped the daily lane as unparseable YAML, so the audit this section -# guards could never have run at all. Round 3: its selftest is skip-aware, -# because two of its plants need PyYAML and no runner had it — on 538288c that -# took linux/gcc, macOS and ASan shard 2/3 red (CI run 35599371704). The -# runners now install PyYAML (.devcontainer/Dockerfile, and a macOS setup step -# in ci.yml); when it is nevertheless absent THIS caller probes for it itself -# and allows exactly the pinned named-skip count, for that gate alone. -# -# None of these tools builds anything and all belong on the PR lane. Measured -# on the dev box 2026-09-21, after round 5 added the once-per-run KNOWN_REPOS -# verification: roadmap live pass 11.9 s (one `gh api` for the milestones, one -# `gh api orgs//repos --paginate` for the repository list, and one per -# reference in the table), its selftest 7.0 s (entirely fixture-driven, no -# network); issue-labels live pass 1.5 s (one `gh api --paginate`), its -# selftest 1.7 s; workflow-yaml 1.7 s live and 4.2 s selftest. -# The GitHub-facing arms of each SKIP BY NAME without `gh` — or with `gh` -# present and UNAUTHENTICATED, which is a third state the macOS runner is in -# and round 2 reported as seven 404s. The structural arm never skips, so a -# runner with no credentials still refuses a checkbox. -# -# ROUND 4 — THE CALLER PROBES `gh` ITSELF, AND A LANE HOLDS A TOKEN. -# Round 3 trusted the gate's own SKIP TOKEN. The pinned regex admitted -# `milestones=skipped:… refs=skipped:…`, so removing ONLY the live GitHub walk -# from tools/roadmap_check.sh and dressing it as a named skip -# (`gh_authenticated() { return 1; }`) passed this section ON AN AUTHENTICATED -# BOX (round-4 blind critic, Fable) — the round-1 mutation in a new hat. Worse, -# NO automated lane could do anything but skip: no `run_all_tests.sh` step -# exported a token, the dev image had no `gh`, the macOS runner's `gh` is -# unauthenticated, and the daily audit ran only the labels gate. "CI 31/31 -# green" measured arm (a) of the roadmap gate while this section printed -# `population lines 3/3`. The milestone mirror was one nobody checked, which is -# what #1207 was filed about. -# -# So: this caller runs the SAME probe the gates run (tools/gh_probe.sh, one -# file, sourced by both), and when the probe succeeds the pins REQUIRE the live -# tokens — `milestones=gh-api:… refs=gh-api:… resolved=N skipped=0`, a -# `gh-api:` labels line with no `SKIPPED BY NAME` alternative, and -# `loader=pyyaml` whenever this caller's own `import yaml` succeeds. A -# skip-dressed live walk is then red BY NAME. When the probe fails, the named -# skip is accepted AND this caller prints its own line saying so, so the CI log -# records which lanes measured what. `.github/workflows/ci.yml`'s `linux / gcc` -# job now exports `GH_TOKEN` and its dev image installs `gh`, so the live arms -# run on every push; the daily `issue-triage.yml` audit runs the roadmap gate -# too, with a `gh-api:`-only pin. -# -# WHAT THIS CALLER CAN AND CANNOT PROVE. It verifies that a gate printed a -# population line it could only have produced by RUNNING ITS LIVE ARM ON THIS -# LANE (token-pinned, against this caller's own probe), and that the gate's -# selftest ran with the pinned count. A gate that FABRICATES its own output — -# printing the three literal lines and the selftest line with no work behind -# them — is outside this caller's power: a forged receipt reads exactly like a -# true one, and a caller measured 10/10 against one (round-4 blind critic, -# Fable, mutation M4). That is what the blind-critic rounds and each gate's own -# transverse mutations are for. The section still REQUIRES the live token, so a -# lying gate has to lie about a specific, checkable thing. -echo "[99zd] Roadmap is a milestone set, and issues are labelled (#1207/#1155)" - -# --------------------------------------------------------------------------- -# THIS CALLER'S OWN `gh` PROBE. The same file the gates source, so the two -# cannot disagree about whether GitHub is reachable — and so "the gate says it -# skipped" becomes a claim this caller can CHECK instead of one it believes. -# --------------------------------------------------------------------------- -# shellcheck source=../tools/gh_probe.sh -. "$TESTS_DIR/../tools/gh_probe.sh" -ZD_GH_STATE=$(gh_probe_state) && ZD_LIVE=1 || ZD_LIVE=0 -if [ "$ZD_LIVE" -eq 1 ]; then - echo " [99zd] live arms: REQUIRED on this lane — this caller reached GitHub itself (gh_probe_state=$ZD_GH_STATE), so a gate that skips its GitHub arm here is red by name" -else - echo " [99zd] live arms: SKIPPED (no gh credentials on this lane) — gh_probe_state=$ZD_GH_STATE; the GitHub-facing arms may skip by name here, and THIS line is how the CI log says which lanes measured what" -fi - -# --------------------------------------------------------------------------- -# THE CALLER'S OWN COPY. Literals. Not read from any gate. -# --------------------------------------------------------------------------- -# The CONTRACT pin: what the gate publishes via `--contract`, asserted -# verbatim against this copy. It ADMITS a named skip, because a lane with no -# credentials legitimately prints one. -ROADMAP_POP_RE_PINNED='^roadmap-check: OK \(examined=[1-9][0-9]* row\(s\), open=[1-9][0-9]*\) \(source: milestones=(gh-api|skipped):[^ ]+ refs=(gh-api|skipped):[^ ]+ resolved=[0-9]+ skipped=[0-9]+ repos=(verified:[0-9]+|skipped:[^ ]+)\)$' -# The LIVE pin: what this caller requires of the OUTPUT on a lane where it has -# established for itself that GitHub is reachable. No `skipped:` alternative, -# and `skipped=0` — a 403 storm that resolved nothing used to print the same -# `refs=gh-api:…` token as a walk that resolved all seven (round-4 blind -# critic, Fable, mutation M3). `resolved=[1-9][0-9]*` because zero resolved -# references is not a measurement either. -# -# ROUND 5 (blind critic Fable): `repos=verified:[1-9][0-9]*`. The KNOWN_REPOS -# verification is the one call that makes "does not exist" and "is private" -# decidable, and its outcome was nowhere on the OK line — a run whose -# organisation listing 403'd, came back empty, or was gutted printed a line -# BYTE-IDENTICAL to a verified one and passed here 11/11 on this very lane. -ROADMAP_POP_RE_LIVE='^roadmap-check: OK \(examined=[1-9][0-9]* row\(s\), open=[1-9][0-9]*\) \(source: milestones=gh-api:[^ ]+ refs=gh-api:[^ ]+ resolved=[1-9][0-9]* skipped=0 repos=verified:[1-9][0-9]*\)$' -ROADMAP_SELFTEST_EXPECTED=25 - -LABELS_POP_RE_PINNED='^issue-labels: examined=[1-9][0-9]* missing=[0-9][0-9]* \(source: gh-api:[^ )]+\)$' -LABELS_SELFTEST_EXPECTED=7 -# This gate has no structural arm, so a runner with no credentials has nothing -# to measure. It must then SAY SO — silence with rc=0 is the gutted shape. On a -# lane where THIS caller reached GitHub the skip is not accepted at all. -LABELS_SKIP_RE_PINNED='^issue-labels: SKIPPED BY NAME: ' - -WORKFLOW_POP_RE_PINNED='^workflow-yaml: OK \(examined=[1-9][0-9]* file\(s\), [1-9][0-9]* name\(s\), loader=(pyyaml|skipped:[a-z0-9-]+)\)$' -WORKFLOW_POP_RE_LIVE='^workflow-yaml: OK \(examined=[1-9][0-9]* file\(s\), [1-9][0-9]* name\(s\), loader=pyyaml\)$' -WORKFLOW_SELFTEST_EXPECTED=8 -# The ONE named-skip allowance in this section: the workflow-yaml gate's two -# loader plants, and only when PyYAML is genuinely absent. This caller decides -# that for itself rather than believing the gate — and round 4 applies the same -# probe to the gate's LIVE line, which round 3 did not: arm (b) skipping only -# on the live run, with PyYAML present, was accepted (round-4 blind critic, -# Fable, mutation M2). -WORKFLOW_SELFTEST_SKIPS_NO_PYYAML=2 -if python3 -c 'import yaml' >/dev/null 2>&1; then - WORKFLOW_ST_WANT_SKIP=0 - ZD_PYYAML=1 -else - WORKFLOW_ST_WANT_SKIP=$WORKFLOW_SELFTEST_SKIPS_NO_PYYAML - ZD_PYYAML=0 -fi -WORKFLOW_ST_WANT_PASS=$((WORKFLOW_SELFTEST_EXPECTED - WORKFLOW_ST_WANT_SKIP)) - -# --------------------------------------------------------------------------- -# THE EFFECTIVE PINS. Which of the two copies above this run asserts is decided -# by THIS caller's probes, never by the gate's own claim. -# --------------------------------------------------------------------------- -if [ "$ZD_LIVE" -eq 1 ]; then - ROADMAP_POP_RE_EFFECTIVE="$ROADMAP_POP_RE_LIVE" - LABELS_SKIP_RE_EFFECTIVE="" -else - ROADMAP_POP_RE_EFFECTIVE="$ROADMAP_POP_RE_PINNED" - LABELS_SKIP_RE_EFFECTIVE="$LABELS_SKIP_RE_PINNED" -fi -if [ "$ZD_PYYAML" -eq 1 ]; then - WORKFLOW_POP_RE_EFFECTIVE="$WORKFLOW_POP_RE_LIVE" -else - WORKFLOW_POP_RE_EFFECTIVE="$WORKFLOW_POP_RE_PINNED" -fi - -# --------------------------------------------------------------------------- -# The witnesses. A helper increments one ONLY when the caller's own pinned -# literal actually matched; the reporting branch that follows cannot fabricate -# one. So gutting a branch (the `elif false; then` a round-2 critic used) still -# ends the section red, at the accounting check below. -# --------------------------------------------------------------------------- -ZD_POP_SEEN=0 -ZD_ST_SEEN=0 -ZD_CONTRACT_SEEN=0 -ZD_CHECKS=0 -ZD_POP_EXPECTED=3 -ZD_ST_EXPECTED=3 -ZD_CONTRACT_EXPECTED=3 -ZD_CHECKS_EXPECTED=11 - -# zd_contract_check