From aaf64c3fe9dbace93048a549a926e136dc05dcd3 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 3 Aug 2026 11:54:27 +0200 Subject: [PATCH 01/24] docs(cargo-anvil): add implementation plan 0003 for benchmark regression detection Captures the design for wrapping cargo-bench-history as a cargo-anvil capability: scheduled collect+analyze with CI-artifact/cache history persistence, fail the scheduled build on an active regression (reusing native GitHub/ADO failure notifications) rather than PR comments, and a reviewed bless-via-file workflow to accept intentional changes. Cross-backend from one catalog; PR stays compile-only. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .../docs/implementation-plans/0003.md | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 crates/cargo-anvil/docs/implementation-plans/0003.md diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md new file mode 100644 index 000000000..631d0610f --- /dev/null +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -0,0 +1,163 @@ +# Implementation Plan 0003 — Benchmark regression detection via cargo-bench-history + +This plan makes automatic benchmark-regression detection a first-class +`cargo-anvil` capability, so every consuming repo gets it from one catalog +across both backends (GitHub Actions and Azure DevOps). It wraps +[`cargo-bench-history`][cbh] (cbh): a Cargo subcommand that stores each +benchmark run as an immutable record, reconstructs per-benchmark series in git +first-parent order, partitions by a hardware machine key, and reports level +shifts and drift with noise-aware, false-discovery-controlled statistics. + +The detection engine itself is **not** built here — cbh provides it. This plan +is the *wiring*: install the tool, run collect + analyze on scheduled runs, +persist history across runs on each backend, surface regressions by failing the +scheduled build, and let teams accept intentional changes through a reviewed +"bless" file. + +## Why this belongs in cargo-anvil + +cargo-anvil already carries every structural piece this needs, so the net-new +surface is small: + +- **Tool install** — `justfiles/anvil/tools.just` already generates + `anvil-tool--install` recipes; cbh becomes one more. +- **A benchmark check** — a `bench` check already exists but is **compile-only**; + this plan adds a running/analyzing variant rather than inventing a check + concept. +- **PR vs scheduled tiers** — both backends already split `pr-*` and + `scheduled-*` workflows/stages; benchmark analysis is a scheduled concern and + slots into the existing scheduled tier. +- **Cross-backend generation** — one catalog emits both backends, so a single + addition propagates to every anvil repo on GitHub *and* ADO. + +## The surfacing decision: fail the scheduled build + +Benchmarks run on **scheduled** jobs, not on PRs — running them per-PR is +expensive and, on shared runners, too noisy to gate a merge. A regression is +therefore discovered *after* the offending change merged, with no PR to comment +on. So the advisory-PR-comment convention (used by `semver-check`) does not +apply. + +Instead, **an active regression fails the scheduled build.** This reuses each +backend's native failure machinery instead of a bespoke notifier: + +- **GitHub** — a failed scheduled workflow feeds the repo's + create-issue-on-failure path; the issue is **updated in place** each run with + the live findings (benchmark, %, attributed commit, author), so concurrent + regressions and their authors surface even while the build is already red. +- **Azure DevOps** — existing failed-build notification subscriptions fire. + +This reconciles with anvil's "advisory, never fail" rule, which governs **PR +gating**: failing a *scheduled* build blocks no one's merge — it is a +"master-health" signal. On PRs the benchmark check stays **compile-only**. + +The findings (with commit/author attribution from cbh's change-point result) are +written to the build summary and to a findings file the GitHub issue path +consumes, so the one-bit build status is always backed by the full list. + +### Known limitation + +The scheduled build status is binary, so it collapses N concurrent regressions +into one red. GitHub recovers per-regression visibility through the +updated-in-place issue; **ADO is coarser** — while the build is already red, a +newly appearing second regression does not re-fire the native notification. For +v1 this is accepted (findings are in the build summary); a later ADO work-item +updater would close the gap if needed. + +## History persistence across runs + +cbh's local backend is an immutable, key-addressed directory — a natural fit for +CI artifact/cache round-tripping. Each scheduled run: + +1. checks out with **full history** (`fetch-depth: 0`) — analysis reads the + commit graph to order series and find the base merge-base; +2. **restores** the latest history (GitHub `actions/cache` — already wired into + the setup composite — or an artifact; ADO pipeline artifact or cache task); +3. runs `cbh collect` (harvesting whichever engines produced output) then + applies any pending blessings (below) then `cbh analyze`; +4. **saves** the updated history back. + +Because each run republishes the whole accumulated directory, only the latest +snapshot is ever needed. The default is a **rolling window** on CI-native +persistence: portable and zero-config, and on eviction it degrades to a harmless +cold start (advisory detection only, no gate on anyone). A **durable backend** +(cbh's Azure Blob) is an opt-in for repos wanting long history; it needs +provisioning and auth, so it is never the default. + +## Accepting intentional changes: bless via a reviewed file + +A sustained regression is re-detected on every scheduled run, so the build stays +red until the change is **fixed** or **blessed**. cbh's blessing re-baselines a +series from a commit forward via an append-only sidecar written **into the +store** (not the repo). To keep that action reviewable and to avoid handing +developers store access, blessing is expressed as a **committed file** the +scheduled job applies: + +1. The build goes red with, e.g., `emit_context/attach_emitter +50% @ bb06fd35`. +2. A developer opens a PR adding an entry to a repo config file + (`.config/bench-blessings.toml`): the benchmark-id prefix, the attributed + commit, and a human **reason**. +3. A reviewer signs off — accepting a regression becomes a deliberate, audited + decision, and the file is a living ledger of every tradeoff. +4. The next scheduled run applies new entries + (`cbh bless --context `) against the store **before** + analyze, so cbh re-baselines and the build returns to green. + +Applying blessings **inside** the scheduled job (rather than a separate +dispatch) keeps all store mutation single-writer, avoiding a read-modify-write +race on the shared history. The apply step must be **idempotent** — cbh's bless +is append-only, so already-applied entries are skipped (checked against +`cbh list blessings`) to avoid piling up sidecars. Removing an entry (or +`cbh unbless`) reverses a blessing. + +Bless targets the **attributed commit**, which has recorded data, so its +discriminants are unambiguous (a no-data bless would require hand-specifying the +target triple and machine key — avoided). + +## Sequencing + +1. **Tool + version policy.** Register `cargo-bench-history` in the tool catalog + with a pinned version and an `anvil-tool-cargo-bench-history-install` recipe. + Pure additive; no workflow change yet. +2. **The analyzing bench check (local).** A recipe that restores/creates a local + store, runs `collect` + `analyze`, writes findings (with attribution) to the + build summary path and a findings file, and **exits non-zero on an active + regression**. Behaves identically locally and in cloud (writes findings; only + the exit code gates). Keep the existing compile-only `bench` for PRs. +3. **Scheduled wiring, both backends.** Add the check to the scheduled tier; + generate history restore/save (cache/artifact) around it in the GitHub + scheduled workflow and the ADO scheduled stages; full-history checkout; + retention lease on the latest successful run. +4. **Failure surfacing.** GitHub: create/update-in-place the failure issue from + the findings file. ADO: rely on native failed-build subscriptions + build + summary. Document both. +5. **Bless application.** Define `.config/bench-blessings.toml`; add the + idempotent apply step ahead of analyze in the scheduled job; document the + red → bless-PR → green workflow. +6. **Opt-in durable backend.** Document (and gate behind config) cbh's Azure + Blob backend for repos that want history beyond the CI-native rolling window. + +## Open questions and caveats + +- **Hosted-runner machine-key density.** cbh partitions series by a hardware + fingerprint; a heterogeneous hosted pool may split a series into sparse + per-key partitions too short to analyze. Fine on self-hosted/dedicated + runners; to observe on hosted pools. (Detection quality itself is validated — + a backtest over three months of real Oxidizer history flagged three known + regressions at their exact attributed commits with zero false positives when + series were dense.) +- **Coarse attribution under sparse benchmarking.** Benches do not run every + commit, so the attributed commit is the first *benchmarked* commit after a + regression and may bundle several real commits — attribution is often a range, + not a single culprit. Bisection would narrow it but is expensive; the failure + output should state the range honestly. +- **cbh maturity.** cbh is young and little-used; its gating thresholds are + sensible defaults, not values calibrated against every consumer's data. The + pinned-tool-version policy contains the risk; findings are advisory-by-design + at the PR level and only gate the scheduled build. +- **Durable-store portability.** Truly durable, zero-config history is not + achievable portably (CI-native persistence is retention/eviction-limited; + blob needs provisioning). The rolling-window default plus opt-in blob is the + deliberate compromise. + +[cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history From 3ac9a23c6b4d8c6446bcd8e40de55c309a986f21 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 3 Aug 2026 17:02:39 +0200 Subject: [PATCH 02/24] docs(cargo-anvil): design the benchmark regression detection subsystem Add docs/design/benchmarks.md describing the capability as part of the opinionated baseline: cbh as the detection engine, regression detection as a scheduled concern, history as CI-native cross-run state (rolling-window default, opt-in durable blob), fail-the-scheduled-build surfacing via native GitHub/ADO notifications, and bless-via-reviewed-file to accept intentional changes. Register it in the design index and the check catalog, and point implementation plan 0003 at it as the design companion. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/README.md | 2 + crates/cargo-anvil/docs/design/benchmarks.md | 131 ++++++++++++++++++ crates/cargo-anvil/docs/design/checks.md | 5 +- .../docs/implementation-plans/0003.md | 4 + 4 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 crates/cargo-anvil/docs/design/benchmarks.md diff --git a/crates/cargo-anvil/docs/design/README.md b/crates/cargo-anvil/docs/design/README.md index 307558bb9..bcdc85972 100644 --- a/crates/cargo-anvil/docs/design/README.md +++ b/crates/cargo-anvil/docs/design/README.md @@ -15,6 +15,8 @@ user-visible shape of the tool. Detail lives in companion documents: - [ado.md](./ado.md) — Azure DevOps Pipelines emission, 1ESPT/msrustup composition. - [containers.md](./containers.md) — the opt-in, local-only container backend for running any `anvil-*` recipe in a pinned Linux image (Linux-on-Windows parity, distro pinning). +- [benchmarks.md](./benchmarks.md) — scheduled benchmark regression detection via + `cargo-bench-history`: history persistence, fail-the-build surfacing, and bless. - [../verification.md](../verification.md) — continuous-validation strategy: dogfooding, fixture tests, schema validation. diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md new file mode 100644 index 000000000..910165f4d --- /dev/null +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -0,0 +1,131 @@ +# cargo-anvil benchmark regression detection + +This document describes `cargo-anvil`'s support for detecting performance +regressions from a repo's benchmarks over time. It wraps +[`cargo-bench-history`][cbh] (cbh) — which stores each benchmark run as an +immutable record, reconstructs per-benchmark series in git first-parent order, +partitions series by a hardware machine key, and reports level shifts and drift +with noise-aware, false-discovery-controlled statistics — and integrates it into +the opinionated catalog across both cloud-workflow backends. + +The intended audience is `cargo-anvil` maintainers and downstream catalog +authors. The phased build-out is tracked in +[implementation-plans/0003.md](../implementation-plans/0003.md). + +## 1. Problem + +A repo's benchmarks are only useful as a regression signal if a slowdown shows +up as a break in a *trend*, well before it would reach any expensive, +bespoke-hardware load test. Turning per-run numbers into a reliable trend is the +hard part: results must be ordered by how the code evolved (not by when a +benchmark happened to run), compared only against like hardware (CI runs on a +heterogeneous, rotating pool whose machine-to-machine variance dwarfs the +measurement), and judged with noise-aware statistics (a fixed percentage +threshold on that noise fires constantly). Every anvil repo has benchmarks and +none of them gets this today — the existing `bench` check only compiles them. + +## 2. Design principles + +- **Detection is cbh's, not anvil's.** anvil installs and drives the tool; it + does not reimplement change-point detection, machine-key normalization, or + history storage. The opinionated contribution is *where* the tool runs, *how* + its history persists, and *how* a regression is surfaced and accepted. +- **Regression detection is a scheduled concern.** It obeys the catalog's + standing rule — a check belongs in scheduled iff its outcome can change without + a commit to this repo (see [checks.md §4](./checks.md)). Benchmark results + accrue over time and the trend verdict at the tip changes as history grows, so + detection runs on the scheduled tier. On pull requests the benchmark check + stays **compile-only**; running benches per-PR is too costly and, on shared + runners, too noisy to gate a merge. +- **History is cross-run state, kept in CI-native storage.** cbh's local backend + is an immutable, key-addressed directory. Between scheduled runs it round-trips + through each backend's own artifact/cache, so the subsystem stays + self-contained in the pipeline with no external store to provision by default. +- **A regression fails the scheduled build.** Because detection happens after the + offending change merged, there is no pull request to annotate, so the + advisory-PR-comment convention (see [checks.md §6](./checks.md)) does not + apply. Failing the scheduled build reuses each backend's native failure + notifications instead of a bespoke notifier. This does not contradict the + "advisory, never fail" rule, which governs *PR gating* — a scheduled build + blocks no one's merge. +- **Intentional changes are accepted through a reviewed file.** A regression is + cleared by fixing it or by *blessing* it; blessing is expressed as a committed, + reviewed entry rather than an out-of-band action, so accepting a slowdown is an + audited decision. +- **One catalog, both backends.** As with every other check, the capability is + generated for GitHub Actions and Azure DevOps from the same source, so adding + it once reaches every consuming repo. + +## 3. Place in the catalog + +The compile-only `bench` check is unchanged. A new analyzing check runs the +benchmarks, records this commit's results into the restored history, applies any +pending blessings, and analyzes the accumulated series. It lives only in the +scheduled tier and **exits non-zero when cbh reports an active regression**; +locally and in cloud it behaves identically (always writes its findings; only +the exit code gates), matching the local-vs-cloud parity every recipe keeps. + +## 4. History as cross-run state + +Each scheduled run checks out with full history (analysis reads the commit graph +to order series and locate the base merge-base), restores the latest history, +runs collect → apply-blessings → analyze, and saves the updated history back. +Because each run republishes the whole accumulated directory, only the newest +snapshot is ever needed. + +The default is a **rolling window** on CI-native persistence — portable and +zero-config, and on eviction it degrades to a harmless cold start, since +detection is advisory-by-design and gates nobody's merge. A **durable backend** +(cbh's Azure Blob) is an opt-in for repos wanting history beyond that window; it +needs provisioning and credentials, so it is never the default. The +backend-specific restore/save building blocks live in [github.md](./github.md) +and [ado.md](./ado.md). + +## 5. Surfacing: failing the scheduled build + +An active regression fails the scheduled build; the findings — each benchmark, +its magnitude, and the commit cbh attributes the change-point to — are written to +the build summary and to a findings file the backend wiring consumes. + +- **GitHub Actions** — the failure feeds the repo's create-issue-on-failure path; + the issue is **updated in place** each run from the findings file, so + concurrent regressions and the authors of their attributed commits surface even + while the build is already red. +- **Azure DevOps** — existing failed-build notification subscriptions fire; the + findings live in the build summary. + +A sustained regression re-fails every run until it is fixed or blessed, so red +stays meaningful only under the discipline that the build is always returned to +green by one of those two actions. + +## 6. Accepting intentional changes: bless + +cbh's blessing re-baselines a series from a commit forward via an append-only +sidecar written into the *history store*. To keep the store single-writer and to +make acceptance reviewable, blessing is expressed as a committed entry — the +benchmark, the attributed commit, and a human reason — that the scheduled job +applies (idempotently) before analyzing. The workflow is therefore: red build → +a reviewed pull request accepting the change → the next scheduled run applies it +and the build returns to green. The accumulated entries are an audit trail of +every deliberate tradeoff. + +## 7. Boundaries and caveats + +- **Hosted-runner machine-key density.** cbh partitions by a hardware + fingerprint; a heterogeneous hosted pool can split a series into per-key + partitions too sparse to analyze. Detection quality on *dense* series is + validated; density on hosted pools is a property to observe. Self-hosted or + dedicated runners avoid the concern. +- **Attribution is coarse under sparse benchmarking.** Benches do not run on + every commit, so the attributed commit is the first *benchmarked* one after a + regression and may bundle several changes — an honest range, not always a + single culprit. +- **The scheduled status is one bit.** It collapses several concurrent + regressions into one red; GitHub recovers per-regression detail through the + updated issue, ADO through the build summary (its native notification is + coarser while already red). +- **cbh is young.** Its gating thresholds are sensible defaults, not values + calibrated to every consumer's data; the pinned-tool-version policy contains + that risk, and the signal only gates the scheduled build. + +[cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index a1ba95c75..44cd0327c 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -311,7 +311,10 @@ What that means concretely: - **Run only in scheduled** -- the expensive whole-workspace work that doesn't fit a PR budget: the non-stacked miri profiles `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage` (in `scheduled-runtime-analysis`); full `mutants`, - `cargo-hack --feature-powerset`, `bench` (in `scheduled-exhaustive`). + `cargo-hack --feature-powerset`, and the compile-only `bench` (in + `scheduled-exhaustive`). Benchmark *regression detection* — running the + benches and analyzing the accumulated history — is also scheduled-only and is + designed in [benchmarks.md](./benchmarks.md). The single-tier-per-group rule still holds: when a check appears in both tiers it lives in two different groups (one PR group, one scheduled group). Repos that want a diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index 631d0610f..e4c269050 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -14,6 +14,10 @@ persist history across runs on each backend, surface regressions by failing the scheduled build, and let teams accept intentional changes through a reviewed "bless" file. +The design of this capability lives in +[../design/benchmarks.md](../design/benchmarks.md); this plan is the phased +build-out against it. + ## Why this belongs in cargo-anvil cargo-anvil already carries every structural piece this needs, so the net-new From 222e3c9781e84c99d8678ebda30dd7a52446ac94 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 3 Aug 2026 17:42:35 +0200 Subject: [PATCH 03/24] docs(cargo-anvil): specify benchmark history persistence in the backend design docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clarify that the cbh history store is CI-native build ARTIFACTS, not the build cache (which stays scoped to tool/dep acceleration). Add a "Benchmark regression detection" section to github.md (Actions artifacts: download-latest-from-default-branch -> collect/analyze -> upload; fail-the-build -> update-in-place tracking issue) and to ado.md (Pipeline Artifacts via the existing §4.1 job-wrapper `artifacts` contract + DownloadPipelineArtifact latestFromBranch; fail-the-build -> native failed-build notifications; one-bit-status limitation noted). Tighten benchmarks.md to name artifacts precisely. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/ado.md | 42 +++++++++++++++++++ crates/cargo-anvil/docs/design/benchmarks.md | 16 +++++--- crates/cargo-anvil/docs/design/github.md | 43 ++++++++++++++++++++ 3 files changed, 95 insertions(+), 6 deletions(-) diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 037e21a1c..3efc0d3c8 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -819,3 +819,45 @@ Adding a new advisory check is a two-step change: the recipe writes entry. There's deliberately no auto-discovery loop over the convention dir — explicit per-check entries keep stale comments deterministically clearable when a check is removed from the catalog. + +## 12. Benchmark regression detection + +The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs +`cargo-bench-history` and persists its history across scheduled runs as **cross-run +state**. It uses **pipeline artifacts** — not the §7 build cache — and reuses the +§4.1 job-wrapper `artifacts` contract to publish, so no new emission mechanism is +introduced. + +The two persistence mechanisms are separate on purpose: + +- The §7 pipeline cache is *acceleration* (evictable, keyed to tool/toolchain pins). +- The history is a small, durable, append-only store fetched as *the latest from the + default branch*; pipeline artifacts give retention plus a `latestFromBranch` + download, which fits. + +Each scheduled benchmark job: + +1. checks out with full history (`fetchDepth: 0` — the §4.1 wrapper already exposes a + `checkout` input for this; analysis reads the commit graph); +2. **restores** the history with `DownloadPipelineArtifact@2` (`buildType: specific`, + `buildVersionToDownload: latestFromBranch`, the default branch); the first run + finds none and starts empty; +3. applies any pending blessings, runs collect + analyze, writing findings to the + build summary; +4. **publishes** the updated store through the wrapper's `artifacts` parameter + (`{ name: bench-history, path: }`), which the default wrapper emits as + `PublishPipelineArtifact@1` and 1ESPT wrappers as a `pipelineArtifact` output. A + retention lease on the latest successful scheduled run keeps the chain alive. + +Surfacing is by **build failure**, not a PR comment — the regression is discovered +after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits +non-zero on an active regression, failing the stage; ADO's existing failed-build +**notification subscriptions** fire, and the findings live in the build summary. + +Known limitation: the build status is one bit, so while the pipeline is already red +from one regression a newly appearing second one does not re-fire the native +notification (the findings are still in the summary). A later work-item updater could +close this gap; it is out of scope for the first version. + +Blessings are applied from a committed `.config/bench-blessings.toml` before analyze +(step 3) — a reviewed pull request, not an out-of-band action. diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index 910165f4d..c71499324 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -39,8 +39,11 @@ none of them gets this today — the existing `bench` check only compiles them. runners, too noisy to gate a merge. - **History is cross-run state, kept in CI-native storage.** cbh's local backend is an immutable, key-addressed directory. Between scheduled runs it round-trips - through each backend's own artifact/cache, so the subsystem stays - self-contained in the pipeline with no external store to provision by default. + through each backend's native build **artifacts** — deliberately *not* the + tool/dependency cache, which is eviction-prone acceleration keyed to tool pins. + Artifacts give retention and a "fetch the latest from the default branch" + primitive, so the subsystem stays self-contained in the pipeline with no + external store to provision by default. - **A regression fails the scheduled build.** Because detection happens after the offending change merged, there is no pull request to annotate, so the advisory-PR-comment convention (see [checks.md §6](./checks.md)) does not @@ -68,10 +71,11 @@ the exit code gates), matching the local-vs-cloud parity every recipe keeps. ## 4. History as cross-run state Each scheduled run checks out with full history (analysis reads the commit graph -to order series and locate the base merge-base), restores the latest history, -runs collect → apply-blessings → analyze, and saves the updated history back. -Because each run republishes the whole accumulated directory, only the newest -snapshot is ever needed. +to order series and locate the base merge-base), restores the latest history +**artifact from the default branch**, runs collect → apply-blessings → analyze, +and publishes the updated history as this run's artifact. Because each run +republishes the whole accumulated directory, only the newest snapshot is ever +needed. The default is a **rolling window** on CI-native persistence — portable and zero-config, and on eviction it degrades to a harmless cold start, since diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 657e34676..43a069a8c 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -751,3 +751,46 @@ a matching `Upsert anvil-` / `Clear anvil-` pair with `header: anvil-`. There's deliberately no auto-discovery loop over the convention dir — explicit per-check steps keep stale comments deterministically clearable when a check is removed from the catalog. + +## 12. Benchmark regression detection + +The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs +`cargo-bench-history` and needs its history to persist across scheduled runs. +That history is **cross-run state**, so it uses GitHub **Actions artifacts** — not +the §8 build cache. The two are deliberately separate: + +- The §8 `actions/cache` is *acceleration*: evictable, keyed to the tool/toolchain + pins, scoped to make a repeat build faster. Using it as the system of record for + accumulating history would inherit its eviction and its immutable-key model. +- The history is a small, durable, append-only store that must be fetched as *the + latest from the default branch*. Artifacts give retention and an explicit + cross-run fetch, which is the right primitive. + +Each scheduled benchmark job: + +1. checks out with `fetch-depth: 0` (analysis reads the commit graph); +2. **restores** the history by downloading the `bench-history` artifact from the + most recent successful `anvil-scheduled` run on the default branch (a small step + queries the runs API for the latest success, then `actions/download-artifact` + fetches it by run id); the first run finds none and starts empty; +3. applies any pending blessings, runs collect + analyze, writing findings to the + job summary and to a findings file; +4. **saves** the updated store with `actions/upload-artifact` (name `bench-history`); + retention is set so the latest successful run's artifact outlives the gap to the + next scheduled run. + +Surfacing is by **build failure**, not a PR comment — the regression is discovered +after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits +non-zero on an active regression, failing the job. The scheduled workflow's failure +path creates-or-updates a tracking **issue** from the findings file — updated in +place each run so concurrent regressions and the authors of their attributed commits +surface even while the build is already red. This needs `issues: write`, declared on +the scheduled job only; the PR workflow keeps `contents: read`. + +Blessings are applied from a committed `.config/bench-blessings.toml` before analyze +(step 3), so accepting an intentional change is a reviewed pull request rather than an +out-of-band action. + +If a repo opts into cbh's durable Azure Blob backend instead of the artifact rolling +window, the §8 `actions/cache` may additionally host cbh's read-through object cache — +the one place the build cache touches this subsystem. From 36a74ab75cfc6ddf27480f2c61170e1202722365 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 3 Aug 2026 17:48:58 +0200 Subject: [PATCH 04/24] docs(cargo-anvil): describe desired state, not design evolution, in benchmark design Remove contrastive "artifacts not the cache / separate on purpose" framing, the "validated / to observe" and "cbh is young" status notes, and the "out of scope for the first version" roadmap line. The design docs now state the end state (history persists as build artifacts; the one-bit status limitation is a property) without justifying-against-alternatives or recording decisions. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/ado.md | 23 +++++-------- crates/cargo-anvil/docs/design/benchmarks.md | 34 +++++++++----------- crates/cargo-anvil/docs/design/github.md | 17 ++-------- 3 files changed, 26 insertions(+), 48 deletions(-) diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 3efc0d3c8..58ee317b3 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -823,17 +823,11 @@ removed from the catalog. ## 12. Benchmark regression detection The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs -`cargo-bench-history` and persists its history across scheduled runs as **cross-run -state**. It uses **pipeline artifacts** — not the §7 build cache — and reuses the -§4.1 job-wrapper `artifacts` contract to publish, so no new emission mechanism is -introduced. - -The two persistence mechanisms are separate on purpose: - -- The §7 pipeline cache is *acceleration* (evictable, keyed to tool/toolchain pins). -- The history is a small, durable, append-only store fetched as *the latest from the - default branch*; pipeline artifacts give retention plus a `latestFromBranch` - download, which fits. +`cargo-bench-history`, whose history persists across scheduled runs as **pipeline +artifacts**, reusing the §4.1 job-wrapper `artifacts` contract to publish. The +history is a small, durable, append-only store fetched as the latest from the +default branch, which pipeline artifacts' retention and `latestFromBranch` download +provide. Each scheduled benchmark job: @@ -854,10 +848,9 @@ after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exi non-zero on an active regression, failing the stage; ADO's existing failed-build **notification subscriptions** fire, and the findings live in the build summary. -Known limitation: the build status is one bit, so while the pipeline is already red -from one regression a newly appearing second one does not re-fire the native -notification (the findings are still in the summary). A later work-item updater could -close this gap; it is out of scope for the first version. +Because the build status is one bit, while the pipeline is already red from one +regression a newly appearing second one does not re-fire the native notification; +the findings remain in the build summary. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3) — a reviewed pull request, not an out-of-band action. diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index c71499324..778b01d16 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -38,19 +38,15 @@ none of them gets this today — the existing `bench` check only compiles them. stays **compile-only**; running benches per-PR is too costly and, on shared runners, too noisy to gate a merge. - **History is cross-run state, kept in CI-native storage.** cbh's local backend - is an immutable, key-addressed directory. Between scheduled runs it round-trips - through each backend's native build **artifacts** — deliberately *not* the - tool/dependency cache, which is eviction-prone acceleration keyed to tool pins. - Artifacts give retention and a "fetch the latest from the default branch" - primitive, so the subsystem stays self-contained in the pipeline with no - external store to provision by default. -- **A regression fails the scheduled build.** Because detection happens after the - offending change merged, there is no pull request to annotate, so the - advisory-PR-comment convention (see [checks.md §6](./checks.md)) does not - apply. Failing the scheduled build reuses each backend's native failure - notifications instead of a bespoke notifier. This does not contradict the - "advisory, never fail" rule, which governs *PR gating* — a scheduled build - blocks no one's merge. + is an immutable, key-addressed directory that round-trips between scheduled runs + through each backend's native build **artifacts**, whose retention and + fetch-the-latest-from-the-default-branch semantics keep the subsystem + self-contained in the pipeline with no external store to provision by default. +- **A regression fails the scheduled build.** Detection happens after the + offending change merged, so there is no pull request to annotate; the signal is + a failed scheduled build, which each backend's native failure notifications + carry. This sits within the "advisory, never fail" tenet, which governs *PR + gating* — a scheduled build blocks no one's merge. - **Intentional changes are accepted through a reviewed file.** A regression is cleared by fixing it or by *blessing* it; blessing is expressed as a committed, reviewed entry rather than an out-of-band action, so accepting a slowdown is an @@ -117,9 +113,9 @@ every deliberate tradeoff. - **Hosted-runner machine-key density.** cbh partitions by a hardware fingerprint; a heterogeneous hosted pool can split a series into per-key - partitions too sparse to analyze. Detection quality on *dense* series is - validated; density on hosted pools is a property to observe. Self-hosted or - dedicated runners avoid the concern. + partitions too sparse to analyze. Whether a hosted pool stays dense enough + depends on its hardware homogeneity; self-hosted or dedicated runners avoid the + concern. - **Attribution is coarse under sparse benchmarking.** Benches do not run on every commit, so the attributed commit is the first *benchmarked* one after a regression and may bundle several changes — an honest range, not always a @@ -128,8 +124,8 @@ every deliberate tradeoff. regressions into one red; GitHub recovers per-regression detail through the updated issue, ADO through the build summary (its native notification is coarser while already red). -- **cbh is young.** Its gating thresholds are sensible defaults, not values - calibrated to every consumer's data; the pinned-tool-version policy contains - that risk, and the signal only gates the scheduled build. +- **Uncalibrated thresholds.** cbh's gating thresholds are defaults rather than + values calibrated to every consumer's data; pinning the tool version contains + the resulting risk, and the signal only gates the scheduled build. [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 43a069a8c..4aac15f95 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -755,16 +755,9 @@ clearable when a check is removed from the catalog. ## 12. Benchmark regression detection The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs -`cargo-bench-history` and needs its history to persist across scheduled runs. -That history is **cross-run state**, so it uses GitHub **Actions artifacts** — not -the §8 build cache. The two are deliberately separate: - -- The §8 `actions/cache` is *acceleration*: evictable, keyed to the tool/toolchain - pins, scoped to make a repeat build faster. Using it as the system of record for - accumulating history would inherit its eviction and its immutable-key model. -- The history is a small, durable, append-only store that must be fetched as *the - latest from the default branch*. Artifacts give retention and an explicit - cross-run fetch, which is the right primitive. +`cargo-bench-history`, whose history persists across scheduled runs as GitHub +**Actions artifacts**: a small, durable, append-only store fetched as the latest +from the default branch, which artifacts' retention and cross-run download provide. Each scheduled benchmark job: @@ -790,7 +783,3 @@ the scheduled job only; the PR workflow keeps `contents: read`. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3), so accepting an intentional change is a reviewed pull request rather than an out-of-band action. - -If a repo opts into cbh's durable Azure Blob backend instead of the artifact rolling -window, the §8 `actions/cache` may additionally host cbh's read-through object cache — -the one place the build cache touches this subsystem. From b5f208bbe7ddc01cb8f9b1b408c5da77e0f6bc49 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 3 Aug 2026 18:26:25 +0200 Subject: [PATCH 05/24] docs(cargo-anvil): reflect the scheduled-benchmarks group in diagrams and catalog Give benchmark regression detection its own scheduled group (isolating the history artifact round-trip and fail-on-regression from the other exhaustive work). Update the checks.md tier/group/check flowchart and its scheduled-tier table (now 5 groups), the github.md and ado.md scheduled-pipeline diagrams and emitted-artifact file trees, and name the group + check (bench-history) in benchmarks.md. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/ado.md | 9 ++++++++- crates/cargo-anvil/docs/design/benchmarks.md | 14 ++++++++------ crates/cargo-anvil/docs/design/checks.md | 6 +++++- crates/cargo-anvil/docs/design/github.md | 9 ++++++++- 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 58ee317b3..96eb034df 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -170,6 +170,12 @@ flowchart LR sadv_setup ==> sadv_setup_just sexh_setup ==> sexh_setup_just + sched_stages --> sbench_s["stage: scheduled_benchmarks
linux + windows jobs"]:::stage + sbench_s ==> sbench_step[".pipelines/anvil/
steps/scheduled-benchmarks.yml"]:::step + sbench_step ==> sbench_setup[".pipelines/anvil/
steps/setup.yml"]:::step + sbench_step ==> sbench_just["just anvil-scheduled-benchmarks"]:::recipe + sbench_setup ==> sbench_setup_just["just anvil-setup"]:::recipe + classDef trigger fill:#fff4d6,stroke:#b08800,stroke-width:1px; classDef root fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef impl fill:#dff0d8,stroke:#28a745,stroke-width:1px; @@ -215,7 +221,8 @@ Note the ADO topology differs from GitHub Actions in two places: ├── scheduled-test.yml owned ├── scheduled-advisories.yml owned ├── scheduled-runtime-analysis.yml owned - └── scheduled-exhaustive.yml owned + ├── scheduled-exhaustive.yml owned + └── scheduled-benchmarks.yml owned ``` All files are regular owned files tracked by the sidecar `.anvil.lock` manifest diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index 778b01d16..98f0625eb 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -57,12 +57,14 @@ none of them gets this today — the existing `bench` check only compiles them. ## 3. Place in the catalog -The compile-only `bench` check is unchanged. A new analyzing check runs the -benchmarks, records this commit's results into the restored history, applies any -pending blessings, and analyzes the accumulated series. It lives only in the -scheduled tier and **exits non-zero when cbh reports an active regression**; -locally and in cloud it behaves identically (always writes its findings; only -the exit code gates), matching the local-vs-cloud parity every recipe keeps. +The compile-only `bench` check is unchanged. A new analyzing check — +`bench-history` — runs the benchmarks, records this commit's results into the +restored history, applies any pending blessings, and analyzes the accumulated +series. It lives in its own scheduled group, `scheduled-benchmarks`, so its +history round-trip and failure semantics stay isolated, and it **exits non-zero +when cbh reports an active regression**; locally and in cloud it behaves +identically (always writes its findings; only the exit code gates), matching the +local-vs-cloud parity every recipe keeps. ## 4. History as cross-run state diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index 44cd0327c..f53dd7d32 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -52,6 +52,7 @@ flowchart LR sched --> s_adv[anvil-scheduled-advisories]:::group sched --> s_runtime[anvil-scheduled-runtime-analysis]:::group sched --> s_exh[anvil-scheduled-exhaustive]:::group + sched --> s_bench[anvil-scheduled-benchmarks]:::group pr_fast --> fmt[fmt]:::check pr_fast --> clippy[clippy]:::check @@ -99,6 +100,8 @@ flowchart LR s_exh --> cargo_hack[cargo-hack]:::check s_exh --> bench[bench]:::check + s_bench --> bench_history[bench-history]:::check + classDef tier fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef group fill:#f6f8fa,stroke:#586069,stroke-width:1px; classDef check fill:#f3e8ff,stroke:#6f42c1,stroke-width:1px,font-size:10px; @@ -117,7 +120,7 @@ flowchart LR The three `pr-slow*` groups are independent: failures in `pr-test` don't block `pr-runtime-analysis` or `pr-mutants` from running, and overall PR wall-clock is `max(pr-test, pr-runtime-analysis, pr-mutants)` per leg rather than the sum. Locally, `just anvil-pr-slow` is an umbrella recipe that runs all three sub-recipes sequentially so adopters who want "run everything slow" don't have to type three commands. -### scheduled tier (4 groups) +### scheduled tier (5 groups) | Group | OS scope | Purpose | |----------------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------| @@ -125,6 +128,7 @@ The three `pr-slow*` groups are independent: failures in `pr-test` don't block ` | `scheduled-advisories` | Same default as `pr-fast` | Re-runs every check whose outcome can change without a commit to this repo: `deny`, `audit`, `aprz` (external databases), `clippy` (lint set evolves with toolchain). Cross-OS because clippy compiles per host. | | `scheduled-runtime-analysis` | Same default as `pr-runtime-analysis` | Whole-workspace runtime correctness under profiles too expensive (or too non-deterministic) for PR: `miri` (stacked borrows, full-workspace re-run of the PR-tier impact-scoped check), `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage`. Each profile is a separate cloud-workflow job so they fan out in parallel rather than serializing into a single multi-hour run. OS scope matches `pr-runtime-analysis` -- if miri-under-stacked-borrows is worth running on a given OS leg in PR, the harder miri profiles are worth running there too: their job is precisely to surface UB that the more permissive stacked-borrows model misses. Adopters who can't afford the full matrix (each profile costs hours per leg) override the matrix in their root workflow / pipeline. | | `scheduled-exhaustive` | Linux x86_64 + Windows x86_64 | The expensive whole-workspace permutations that don't fit the PR budget: full `cargo mutants`, `cargo-hack --feature-powerset`, and `cargo bench --no-run` plus a single-iteration smoke run per bench target. Cross-OS to match `oxidizer`'s policy and to give cargo-hack / bench compile coverage for cfg-gated code. **x86_64-only**: same `cargo-mutants` / `winapi` constraint as `pr-mutants`. Adopters who can't afford the full matrix (mutants-full can run for hours per leg) override the matrix in their root workflow / pipeline. | +| `scheduled-benchmarks` | Same default as `scheduled-exhaustive` | Runs the benchmarks and analyzes the accumulated history with `cargo-bench-history` to detect performance regressions, restoring and publishing that history as a build artifact and failing on an active regression. Its own group so the history round-trip and fail-on-regression semantics stay isolated from the other exhaustive work. See [benchmarks.md](./benchmarks.md). | **Backend asymmetry on ARM coverage.** The GitHub backend ships a four-leg default matrix (Linux/Windows × x86_64/aarch64) because GH has Microsoft-hosted ARM runners diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 4aac15f95..0a1d3ec46 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -168,6 +168,12 @@ flowchart LR sadv_setup ==> sadv_setup_just sexh_setup ==> sexh_setup_just + sched_impl --> sbench_job["scheduled-benchmarks
matrix: linux, windows"]:::job + sbench_job ==> sbench_act[".github/actions/
anvil-scheduled-benchmarks"]:::action + sbench_act ==> sbench_setup[".github/actions/
anvil-setup"]:::action + sbench_act ==> sbench_just["just anvil-scheduled-benchmarks"]:::recipe + sbench_setup ==> sbench_setup_just["just anvil-setup"]:::recipe + classDef trigger fill:#fff4d6,stroke:#b08800,stroke-width:1px; classDef root fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef impl fill:#dff0d8,stroke:#28a745,stroke-width:1px; @@ -193,7 +199,8 @@ Every PR-tier group job declares `needs: [impact-linux, impact-windows]` so it c │ ├── anvil-scheduled-test/action.yml owned │ ├── anvil-scheduled-advisories/action.yml owned │ ├── anvil-scheduled-runtime-analysis/action.yml owned -│ └── anvil-scheduled-exhaustive/action.yml owned +│ ├── anvil-scheduled-exhaustive/action.yml owned +│ └── anvil-scheduled-benchmarks/action.yml owned └── workflows/ ├── anvil-pr-impl.yml owned (reusable workflow doing the wiring) ├── anvil-scheduled-impl.yml owned (reusable workflow for the scheduled tier) From 96e71357146ead08f4441cbc816c885eb9983c0b Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Tue, 4 Aug 2026 10:00:15 +0200 Subject: [PATCH 06/24] docs(cargo-anvil): confine design to the design docs; make 0003 a pure plan MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Strip the surfacing/persistence/bless/caveats prose from implementation plan 0003 (all of it now lives in design/benchmarks.md and the backend §12 sections) and rewrite it as sequencing only: principles plus six phases that reference the design rather than restating it. Add a "History horizon" boundary to benchmarks.md §7 so the design fully owns the caveats. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/benchmarks.md | 3 + .../docs/implementation-plans/0003.md | 238 ++++++------------ 2 files changed, 78 insertions(+), 163 deletions(-) diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index 98f0625eb..de65b3172 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -129,5 +129,8 @@ every deliberate tradeoff. - **Uncalibrated thresholds.** cbh's gating thresholds are defaults rather than values calibrated to every consumer's data; pinning the tool version contains the resulting risk, and the signal only gates the scheduled build. +- **History horizon.** CI-native artifacts hold a rolling window, not unbounded + history; a repo that needs a longer horizon opts into the durable backend + (§4). [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index e4c269050..375c9abfd 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -1,167 +1,79 @@ # Implementation Plan 0003 — Benchmark regression detection via cargo-bench-history -This plan makes automatic benchmark-regression detection a first-class -`cargo-anvil` capability, so every consuming repo gets it from one catalog -across both backends (GitHub Actions and Azure DevOps). It wraps -[`cargo-bench-history`][cbh] (cbh): a Cargo subcommand that stores each -benchmark run as an immutable record, reconstructs per-benchmark series in git -first-parent order, partitions by a hardware machine key, and reports level -shifts and drift with noise-aware, false-discovery-controlled statistics. - -The detection engine itself is **not** built here — cbh provides it. This plan -is the *wiring*: install the tool, run collect + analyze on scheduled runs, -persist history across runs on each backend, surface regressions by failing the -scheduled build, and let teams accept intentional changes through a reviewed -"bless" file. - -The design of this capability lives in -[../design/benchmarks.md](../design/benchmarks.md); this plan is the phased -build-out against it. - -## Why this belongs in cargo-anvil - -cargo-anvil already carries every structural piece this needs, so the net-new -surface is small: - -- **Tool install** — `justfiles/anvil/tools.just` already generates - `anvil-tool--install` recipes; cbh becomes one more. -- **A benchmark check** — a `bench` check already exists but is **compile-only**; - this plan adds a running/analyzing variant rather than inventing a check - concept. -- **PR vs scheduled tiers** — both backends already split `pr-*` and - `scheduled-*` workflows/stages; benchmark analysis is a scheduled concern and - slots into the existing scheduled tier. -- **Cross-backend generation** — one catalog emits both backends, so a single - addition propagates to every anvil repo on GitHub *and* ADO. - -## The surfacing decision: fail the scheduled build - -Benchmarks run on **scheduled** jobs, not on PRs — running them per-PR is -expensive and, on shared runners, too noisy to gate a merge. A regression is -therefore discovered *after* the offending change merged, with no PR to comment -on. So the advisory-PR-comment convention (used by `semver-check`) does not -apply. - -Instead, **an active regression fails the scheduled build.** This reuses each -backend's native failure machinery instead of a bespoke notifier: - -- **GitHub** — a failed scheduled workflow feeds the repo's - create-issue-on-failure path; the issue is **updated in place** each run with - the live findings (benchmark, %, attributed commit, author), so concurrent - regressions and their authors surface even while the build is already red. -- **Azure DevOps** — existing failed-build notification subscriptions fire. - -This reconciles with anvil's "advisory, never fail" rule, which governs **PR -gating**: failing a *scheduled* build blocks no one's merge — it is a -"master-health" signal. On PRs the benchmark check stays **compile-only**. - -The findings (with commit/author attribution from cbh's change-point result) are -written to the build summary and to a findings file the GitHub issue path -consumes, so the one-bit build status is always backed by the full list. - -### Known limitation - -The scheduled build status is binary, so it collapses N concurrent regressions -into one red. GitHub recovers per-regression visibility through the -updated-in-place issue; **ADO is coarser** — while the build is already red, a -newly appearing second regression does not re-fire the native notification. For -v1 this is accepted (findings are in the build summary); a later ADO work-item -updater would close the gap if needed. - -## History persistence across runs - -cbh's local backend is an immutable, key-addressed directory — a natural fit for -CI artifact/cache round-tripping. Each scheduled run: - -1. checks out with **full history** (`fetch-depth: 0`) — analysis reads the - commit graph to order series and find the base merge-base; -2. **restores** the latest history (GitHub `actions/cache` — already wired into - the setup composite — or an artifact; ADO pipeline artifact or cache task); -3. runs `cbh collect` (harvesting whichever engines produced output) then - applies any pending blessings (below) then `cbh analyze`; -4. **saves** the updated history back. - -Because each run republishes the whole accumulated directory, only the latest -snapshot is ever needed. The default is a **rolling window** on CI-native -persistence: portable and zero-config, and on eviction it degrades to a harmless -cold start (advisory detection only, no gate on anyone). A **durable backend** -(cbh's Azure Blob) is an opt-in for repos wanting long history; it needs -provisioning and auth, so it is never the default. - -## Accepting intentional changes: bless via a reviewed file - -A sustained regression is re-detected on every scheduled run, so the build stays -red until the change is **fixed** or **blessed**. cbh's blessing re-baselines a -series from a commit forward via an append-only sidecar written **into the -store** (not the repo). To keep that action reviewable and to avoid handing -developers store access, blessing is expressed as a **committed file** the -scheduled job applies: - -1. The build goes red with, e.g., `emit_context/attach_emitter +50% @ bb06fd35`. -2. A developer opens a PR adding an entry to a repo config file - (`.config/bench-blessings.toml`): the benchmark-id prefix, the attributed - commit, and a human **reason**. -3. A reviewer signs off — accepting a regression becomes a deliberate, audited - decision, and the file is a living ledger of every tradeoff. -4. The next scheduled run applies new entries - (`cbh bless --context `) against the store **before** - analyze, so cbh re-baselines and the build returns to green. - -Applying blessings **inside** the scheduled job (rather than a separate -dispatch) keeps all store mutation single-writer, avoiding a read-modify-write -race on the shared history. The apply step must be **idempotent** — cbh's bless -is append-only, so already-applied entries are skipped (checked against -`cbh list blessings`) to avoid piling up sidecars. Removing an entry (or -`cbh unbless`) reverses a blessing. - -Bless targets the **attributed commit**, which has recorded data, so its -discriminants are unambiguous (a no-data bless would require hand-specifying the -target triple and machine key — avoided). - -## Sequencing - -1. **Tool + version policy.** Register `cargo-bench-history` in the tool catalog - with a pinned version and an `anvil-tool-cargo-bench-history-install` recipe. - Pure additive; no workflow change yet. -2. **The analyzing bench check (local).** A recipe that restores/creates a local - store, runs `collect` + `analyze`, writes findings (with attribution) to the - build summary path and a findings file, and **exits non-zero on an active - regression**. Behaves identically locally and in cloud (writes findings; only - the exit code gates). Keep the existing compile-only `bench` for PRs. -3. **Scheduled wiring, both backends.** Add the check to the scheduled tier; - generate history restore/save (cache/artifact) around it in the GitHub - scheduled workflow and the ADO scheduled stages; full-history checkout; - retention lease on the latest successful run. -4. **Failure surfacing.** GitHub: create/update-in-place the failure issue from - the findings file. ADO: rely on native failed-build subscriptions + build - summary. Document both. -5. **Bless application.** Define `.config/bench-blessings.toml`; add the - idempotent apply step ahead of analyze in the scheduled job; document the - red → bless-PR → green workflow. -6. **Opt-in durable backend.** Document (and gate behind config) cbh's Azure - Blob backend for repos that want history beyond the CI-native rolling window. - -## Open questions and caveats - -- **Hosted-runner machine-key density.** cbh partitions series by a hardware - fingerprint; a heterogeneous hosted pool may split a series into sparse - per-key partitions too short to analyze. Fine on self-hosted/dedicated - runners; to observe on hosted pools. (Detection quality itself is validated — - a backtest over three months of real Oxidizer history flagged three known - regressions at their exact attributed commits with zero false positives when - series were dense.) -- **Coarse attribution under sparse benchmarking.** Benches do not run every - commit, so the attributed commit is the first *benchmarked* commit after a - regression and may bundle several real commits — attribution is often a range, - not a single culprit. Bisection would narrow it but is expensive; the failure - output should state the range honestly. -- **cbh maturity.** cbh is young and little-used; its gating thresholds are - sensible defaults, not values calibrated against every consumer's data. The - pinned-tool-version policy contains the risk; findings are advisory-by-design - at the PR level and only gate the scheduled build. -- **Durable-store portability.** Truly durable, zero-config history is not - achievable portably (CI-native persistence is retention/eviction-limited; - blob needs provisioning). The rolling-window default plus opt-in blob is the - deliberate compromise. +The design for this capability is [../design/benchmarks.md](../design/benchmarks.md) +— with the catalog placement in [../design/checks.md](../design/checks.md) and the +backend wiring in [../design/github.md §12](../design/github.md) and +[../design/ado.md §12](../design/ado.md). This plan does not restate *what* the +capability is; it sequences *how* and *in what order* it lands. + +The detection engine is [`cargo-bench-history`][cbh]; anvil supplies only the +wiring. The approach was de-risked ahead of this plan by a backtest over three +months of real benchmark history — cbh flagged three known regressions at their +exact attributed commits with no false positives on dense series — so the plan +proceeds to wiring rather than re-validating detection. + +## Sequencing principles + +1. **Additive first.** Tool registration and the local recipe land before any + cloud-workflow change, so they review without touching backend generation. +2. **Local before cloud.** The check runs and gates locally before any scheduled + wiring exists (anvil's local-vs-cloud parity); the cloud phases only add + persistence, surfacing, and scheduling around an already-working recipe. +3. **Both backends per concern.** Persistence and failure surfacing each land as a + single commit spanning GitHub and ADO, so the two never drift. +4. **Snapshots travel with each phase** (see [../verification.md](../verification.md)): + the catalog and emitted-file snapshots grow with the new tool, group, + actions/steps, and workflows as each phase adds them. + +## Phase 1 — Tool registration + +Pin `cargo-bench-history` in `versions.just` and add an +`anvil-tool-cargo-bench-history-install` recipe to `tools.just`. No group +references it yet. Pure additive; catalog snapshots gain the recipe. + +## Phase 2 — The `bench-history` check recipe (local) + +Add the `bench-history` check recipe (`checks/bench-history.just`) and the +`scheduled-benchmarks` group recipe. The recipe resolves a local history +directory, applies any pending blessings, runs `cbh collect` then `cbh analyze`, +writes findings to the build-summary path and a findings file, and exits non-zero +on an active regression — behaving identically with no cloud context. Catalog +snapshots gain the group and check. + +## Phase 3 — Scheduled wiring and history persistence (both backends) + +Emit the `scheduled-benchmarks` job (GitHub reusable scheduled workflow) and stage +(ADO scheduled stages), each running `anvil-setup` then +`just anvil-scheduled-benchmarks`, and add the artifact round-trip: + +- **GitHub** — `fetch-depth: 0` checkout; a step resolving the latest successful + `anvil-scheduled` run on the default branch and downloading its `bench-history` + artifact; `actions/upload-artifact` with retention at the end. +- **ADO** — full-history checkout via the §4.1 wrapper `checkout` input; + `DownloadPipelineArtifact@2` (`buildVersionToDownload: latestFromBranch`); + publish through the wrapper's `artifacts` contract. + +Emitted-file snapshots gain the actions/steps and workflow/stage entries. + +## Phase 4 — Failure surfacing + +- **GitHub** — a failure-path step in the scheduled workflow that creates or + updates the tracking issue in place from the findings file; `issues: write` + scoped to the scheduled job. +- **ADO** — reliance on native failed-build notification subscriptions plus the + build summary; no new emission, documented in the scheduled step. + +## Phase 5 — Bless application + +Define the `.config/bench-blessings.toml` schema and the idempotent apply step that +runs ahead of analyze in the scheduled job, skipping entries already present per +`cbh list blessings`. A fixture repo exercises red → bless → green. + +## Phase 6 — Opt-in durable backend + +Gate cbh's Azure Blob backend behind repo config for repos wanting history beyond +the artifact rolling window, and document the provisioning and credential +requirements. [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history + From eb0dd1669cc6c68a4ce0e1d987e3230f90ca4bb6 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Tue, 4 Aug 2026 17:47:53 +0200 Subject: [PATCH 07/24] docs(cargo-anvil): make artifacts the sole store; add local behavior; blob a non-goal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Artifacts are the only supported history store: state it as such in benchmarks.md §4 and reduce cbh's own durable backends (e.g. Azure Blob) to an explicit non-goal; reframe the §8 history-horizon caveat accordingly and drop plan Phase 6. Add a §6 "Local behavior" section: the recipe is identical locally but has no shared history, so local analysis is a friendly no-op; regression detection is a scheduled/shared concern and the self-contained failure surface (finding + cbh's trend chart) is the interface, with local reproduction a documented manual escape hatch. Note the chart in §5 so the surface is self-contained. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/benchmarks.md | 43 +++++++++++++------ .../docs/implementation-plans/0003.md | 6 --- 2 files changed, 31 insertions(+), 18 deletions(-) diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index de65b3172..e97b7524a 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -59,7 +59,7 @@ none of them gets this today — the existing `bench` check only compiles them. The compile-only `bench` check is unchanged. A new analyzing check — `bench-history` — runs the benchmarks, records this commit's results into the -restored history, applies any pending blessings, and analyzes the accumulated +history, applies any pending blessings, and analyzes the accumulated series. It lives in its own scheduled group, `scheduled-benchmarks`, so its history round-trip and failure semantics stay isolated, and it **exits non-zero when cbh reports an active regression**; locally and in cloud it behaves @@ -75,19 +75,20 @@ and publishes the updated history as this run's artifact. Because each run republishes the whole accumulated directory, only the newest snapshot is ever needed. -The default is a **rolling window** on CI-native persistence — portable and +The persistence is a **rolling window** on CI-native artifacts — portable and zero-config, and on eviction it degrades to a harmless cold start, since -detection is advisory-by-design and gates nobody's merge. A **durable backend** -(cbh's Azure Blob) is an opt-in for repos wanting history beyond that window; it -needs provisioning and credentials, so it is never the default. The -backend-specific restore/save building blocks live in [github.md](./github.md) -and [ado.md](./ado.md). +detection is advisory-by-design and gates nobody's merge. The backend-specific +restore/save building blocks live in [github.md](./github.md) and +[ado.md](./ado.md). cbh's own durable backends (for example Azure Blob) are +outside anvil's scope: the artifact rolling window is the supported store. ## 5. Surfacing: failing the scheduled build An active regression fails the scheduled build; the findings — each benchmark, its magnitude, and the commit cbh attributes the change-point to — are written to -the build summary and to a findings file the backend wiring consumes. +the build summary and to a findings file the backend wiring consumes. The emitted +findings include cbh's topology-accurate trend chart, so the reviewer's surface is +self-contained — enough to decide *fix or bless* without reproducing the run. - **GitHub Actions** — the failure feeds the repo's create-issue-on-failure path; the issue is **updated in place** each run from the findings file, so @@ -100,7 +101,25 @@ A sustained regression re-fails every run until it is fixed or blessed, so red stays meaningful only under the discipline that the build is always returned to green by one of those two actions. -## 6. Accepting intentional changes: bless +## 6. Local behavior + +The `bench-history` recipe is the same command locally and in cloud, but its +*input* differs: the history lives only in the CI artifact store, so a local run +has no shared trend to analyze. Locally the recipe runs the benches, records them +into a gitignored local store, and analyzes that store — which on a fresh checkout +is empty, so analysis is a clean no-op reporting that there is no local history +yet. A developer thus gets the current run's numbers (and a single-machine local +trend if they run it repeatedly), not the shared regression signal. + +Regression detection is therefore a scheduled, shared concern, and the failure +surface (§5) — cbh's finding and trend chart in the issue or build summary — is +the interface a developer acts on. Reproducing a CI finding locally is not a +first-class workflow: it would require downloading that run's `bench-history` +artifact into the local store and running cbh's `examine` with the run's machine +key (a developer's machine has a different hardware fingerprint). That remains a +documented manual escape hatch rather than a generated recipe. + +## 7. Accepting intentional changes: bless cbh's blessing re-baselines a series from a commit forward via an append-only sidecar written into the *history store*. To keep the store single-writer and to @@ -111,7 +130,7 @@ a reviewed pull request accepting the change → the next scheduled run applies and the build returns to green. The accumulated entries are an audit trail of every deliberate tradeoff. -## 7. Boundaries and caveats +## 8. Boundaries and caveats - **Hosted-runner machine-key density.** cbh partitions by a hardware fingerprint; a heterogeneous hosted pool can split a series into per-key @@ -130,7 +149,7 @@ every deliberate tradeoff. values calibrated to every consumer's data; pinning the tool version contains the resulting risk, and the signal only gates the scheduled build. - **History horizon.** CI-native artifacts hold a rolling window, not unbounded - history; a repo that needs a longer horizon opts into the durable backend - (§4). + history; there is no anvil-supported durable store, so the horizon is the + artifact retention. [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index 375c9abfd..019d88fa3 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -69,11 +69,5 @@ Define the `.config/bench-blessings.toml` schema and the idempotent apply step t runs ahead of analyze in the scheduled job, skipping entries already present per `cbh list blessings`. A fixture repo exercises red → bless → green. -## Phase 6 — Opt-in durable backend - -Gate cbh's Azure Blob backend behind repo config for repos wanting history beyond -the artifact rolling window, and document the provisioning and credential -requirements. - [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history From 6871c2373fa5c378af467b2f1fde54b6716323e1 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Tue, 4 Aug 2026 17:55:14 +0200 Subject: [PATCH 08/24] docs(cargo-anvil): add scheduled-benchmarks to the local recipe surface List the anvil-scheduled-benchmarks group recipe (anvil-bench-history) in local.md's groups.just and the anvil-scheduled tier aggregator, and note that the group runs the same recipe locally but needs the shared CI history for regression analysis (pointing to benchmarks.md section 6 rather than duplicating it). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/local.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/cargo-anvil/docs/design/local.md b/crates/cargo-anvil/docs/design/local.md index a946e4d32..7da2745b4 100644 --- a/crates/cargo-anvil/docs/design/local.md +++ b/crates/cargo-anvil/docs/design/local.md @@ -124,8 +124,13 @@ anvil-pr-mutants: anvil-mutants-diff anvil-scheduled-test: anvil-llvm-cov anvil-doc-test anvil-examples anvil-scheduled-advisories: anvil-deny anvil-audit anvil-aprz anvil-clippy anvil-scheduled-exhaustive: anvil-mutants-full anvil-cargo-hack anvil-bench +anvil-scheduled-benchmarks: anvil-bench-history ``` +The `scheduled-benchmarks` group runs the same recipe locally, but benchmark +regression analysis needs the shared CI history that a local checkout does not +have; its local behavior is described in [benchmarks.md §6](./benchmarks.md). + ### tiers.just Three tier aggregators. Each tier is a recipe that depends on the appropriate set of groups @@ -134,7 +139,7 @@ in a deterministic order: ```just anvil-pr: anvil-pr-validate-prereqs anvil-pr-fast anvil-pr-slow anvil-scheduled: anvil-scheduled-validate-prereqs anvil-scheduled-test anvil-scheduled-advisories \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive anvil-scheduled-benchmarks anvil-full: anvil-pr anvil-scheduled ``` From f18f4e75c03df6993fa34a0693da8c6b27e4b5cb Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Tue, 4 Aug 2026 19:27:05 +0200 Subject: [PATCH 09/24] feat(cargo-anvil): detect benchmark regressions on the scheduled tier Wire cargo-bench-history into the catalog as a new `bench-history` check in its own `scheduled-benchmarks` group. The check runs the workspace benchmarks, records them into a history that round-trips through CI-native build artifacts, applies committed blessings, analyzes the accumulated series, and exits non-zero on an active regression. Both backends carry the history and the failure surface: GitHub restores from the newest scheduled run carrying the leg artifact and files a create-or-update tracking issue on failure; ADO restores via DownloadPipelineArtifact and publishes through the job wrapper, relying on native failed-build notifications plus the build summary. Restore and publish are both indifferent to the run outcome, so samples taken while the pipeline is red survive. The ADO job wrapper gains a `fetchDepth` parameter, since the analysis walks the commit graph and needs a full checkout. Also fix the actionlint schema test, which silently skipped whenever the tool was installed because its fixture was not a git repository. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 31 +- .../anvil-scheduled-benchmarks/action.yml | 55 +++ .github/workflows/anvil-scheduled-impl.yml | 102 ++++ .github/workflows/anvil-scheduled.yml | 6 + crates/cargo-anvil/README.md | 1 + crates/cargo-anvil/docs/design/README.md | 1 + crates/cargo-anvil/docs/design/ado.md | 46 +- crates/cargo-anvil/docs/design/benchmarks.md | 30 +- crates/cargo-anvil/docs/design/checks.md | 14 +- crates/cargo-anvil/docs/design/github.md | 31 +- crates/cargo-anvil/docs/design/local.md | 5 +- .../docs/implementation-plans/0003.md | 5 +- crates/cargo-anvil/src/anvil/artifacts/ado.rs | 71 ++- .../cargo-anvil/src/anvil/artifacts/github.rs | 28 ++ .../src/anvil/artifacts/justfile.rs | 36 ++ crates/cargo-anvil/src/lib.rs | 1 + crates/cargo-anvil/src/run.rs | 5 + .../templates/ado/scheduled-stages.yml | 43 ++ .../ado/steps/bench-history-restore.yml | 39 ++ .../ado/steps/bench-history-summary.yml | 27 ++ .../cargo-anvil/templates/ado/steps/job.yml | 10 + .../github/scheduled-impl-workflow.yml | 102 ++++ .../github/scheduled-root-workflow.yml | 6 + .../justfiles/anvil/checks/bench-history.just | 205 ++++++++ .../anvil/groups/scheduled-benchmarks.just | 27 ++ .../templates/justfiles/anvil/mod.just | 2 + .../templates/justfiles/anvil/tiers.just | 9 +- .../templates/justfiles/anvil/tools.just | 8 + .../templates/justfiles/anvil/versions.just | 1 + crates/cargo-anvil/tests/schemas.rs | 13 + .../snapshots/snapshots__ado_backend.snap | 439 +++++++++++++++++- .../snapshots/snapshots__github_backend.snap | 421 ++++++++++++++++- .../snapshots/snapshots__local_only.snap | 256 +++++++++- justfiles/anvil/checks/bench-history.just | 205 ++++++++ .../anvil/groups/scheduled-benchmarks.just | 27 ++ justfiles/anvil/mod.just | 2 + justfiles/anvil/tiers.just | 9 +- justfiles/anvil/tools.just | 8 + justfiles/anvil/versions.just | 1 + 39 files changed, 2265 insertions(+), 63 deletions(-) create mode 100644 .github/actions/anvil-scheduled-benchmarks/action.yml create mode 100644 crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml create mode 100644 crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml create mode 100644 crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just create mode 100644 crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just create mode 100644 justfiles/anvil/checks/bench-history.just create mode 100644 justfiles/anvil/groups/scheduled-benchmarks.just diff --git a/.anvil.lock b/.anvil.lock index 4a6ea11e0..d7a232915 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.3.0" -catalog_checksum = "sha256:69b8997e0614880b267e672a05b4bd6404d4532494feae72c94d39b813f190c3" +catalog_checksum = "sha256:f1e7a171ef90c49a946b978d6841816cbff6eb431d767aa4512c5c352d70c115" [[file]] path = ".github/actions/anvil-impact/action.yml" @@ -27,6 +27,10 @@ checksum = "sha256:3fff3e46365b6684a7855c9bb61bd4599e2b15452d8d06c7b7ec3f304ea60 path = ".github/actions/anvil-scheduled-advisories/action.yml" checksum = "sha256:2bdbacd4cfa1626ae6ea9e99921a827bc14b5cd805ce00f49b1326ac00cea98b" +[[file]] +path = ".github/actions/anvil-scheduled-benchmarks/action.yml" +checksum = "sha256:3ee15d8a912d93b544d1f08a168479213b6f2764f8f5534365f1c35a78ed4d3e" + [[file]] path = ".github/actions/anvil-scheduled-exhaustive/action.yml" checksum = "sha256:0e61edc0c6dfcd2cf3a6ad3c498804da681032d0a4da7bd3dd2419dc77e11a7f" @@ -53,11 +57,11 @@ checksum = "sha256:cace8cf38c1ce2dc85d054c39a14de67d100b9bb10c1a3ec8c70ad5bcad32 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:37467df6bf06ed5cbd7178266c08905172edd9cb66ef49057e83bdfccf6d4f40" +checksum = "sha256:fae2a31ee25a226e36a1ade3667e17643d28d769c711dcfface8991c8f4b9580" [[file]] path = ".github/workflows/anvil-scheduled.yml" -checksum = "sha256:b5e0a82b87adcd3be7b4522716af733a3746c3d24def2aa97c422f916cbe0c37" +checksum = "sha256:0db8c05cc5f3131925eceeeaec5d13af093b03e4b294b1532cafce2c6a410569" [[file]] path = "justfiles/anvil/checks/aprz.just" @@ -67,6 +71,10 @@ checksum = "sha256:9823027a0379b45d8be7ee2e822d4de39590159a2b196456caefdbb70c500 path = "justfiles/anvil/checks/audit.just" checksum = "sha256:c5735ada01381c6f6524178d42cbd500d7adfbefd4b9da8f82ab354280e5de2e" +[[file]] +path = "justfiles/anvil/checks/bench-history.just" +checksum = "sha256:89efe8f601f409805eb2cefeeb17b05c285e5823cd349f419dddbd1fe821c731" + [[file]] path = "justfiles/anvil/checks/bench.just" checksum = "sha256:a8fa651a85f36a6ee8519c4b1271eb79664ca447313612a582c71a54186b5285" @@ -239,6 +247,10 @@ checksum = "sha256:c309a7b9f1f2df31a988d5acd51508dc9f44cee5077789892ddd3c9f566ba path = "justfiles/anvil/groups/scheduled-advisories.just" checksum = "sha256:2f0ed3b45c1431f5f5e425696385cbf06ec8f4e99d7232dc2d2794e33325ac5c" +[[file]] +path = "justfiles/anvil/groups/scheduled-benchmarks.just" +checksum = "sha256:e53308fe8486ffd586f515510e54e0603161612fb1c8bf0a47f60257b1258682" + [[file]] path = "justfiles/anvil/groups/scheduled-exhaustive.just" checksum = "sha256:61fa2fc759fd979232dffd4d379be9cf40353ce27a285992b19c45f1a6bb7624" @@ -257,7 +269,7 @@ checksum = "sha256:6f8ed97d9d60f6844fe37427b0e282b701b960219adf7b2a29fa9814c3a18 [[file]] path = "justfiles/anvil/mod.just" -checksum = "sha256:bca3f9ea7628843c5c201cacb0b6ab9fc6259a725b9a63f4a782ecfe281911c8" +checksum = "sha256:e738a34c28e9b13f5844cf719603c0ba87c8809f6e04151051ccf05801274c0b" [[file]] path = "justfiles/anvil/runner.just" @@ -265,15 +277,15 @@ checksum = "sha256:458c343288ac34c284c8d0c716306ae669d0b2bd0555256297dd041d38de7 [[file]] path = "justfiles/anvil/tiers.just" -checksum = "sha256:5cbb45c94bdc0807ac90a21a1f6d742391cd467a0e1a88a66cbc6761e9f2c6a8" +checksum = "sha256:5a1bd55fb37a95f8396ea6a0d652d3ed43053549b4c3898bc9a4a2c0e10d9f16" [[file]] path = "justfiles/anvil/tools.just" -checksum = "sha256:1c22f38c093e836c83e5ef4da35b04a1ff6a8a0d0355310f5605cf52bfb95cd7" +checksum = "sha256:9d158a76330d1d635603ed846c2c36def066bd7976ccc210ccf7b8bed7577da1" [[file]] path = "justfiles/anvil/versions.just" -checksum = "sha256:403b9abf954d0f7479144d6890e1db36dfa1b199704741c7087b48355ea593ba" +checksum = "sha256:2759cc21cc3c5a8736b857f872d5c0f81fef0a53fdfc6221b10f8e174f888879" [[region]] host = ".delta.toml" @@ -320,6 +332,11 @@ host = "crates/cargo-coverage-gate/Cargo.toml" id = "anvil-lints" checksum = "sha256:2dd7c0f21339fd17092b8dedfe924aa86732c3520baab84f914c2d8f4103ac40" +[[region]] +host = "crates/cargo-each/Cargo.toml" +id = "anvil-lints" +checksum = "sha256:2dd7c0f21339fd17092b8dedfe924aa86732c3520baab84f914c2d8f4103ac40" + [[region]] host = "crates/cargo-heather/Cargo.toml" id = "anvil-lints" diff --git a/.github/actions/anvil-scheduled-benchmarks/action.yml b/.github/actions/anvil-scheduled-benchmarks/action.yml new file mode 100644 index 000000000..805281f0b --- /dev/null +++ b/.github/actions/anvil-scheduled-benchmarks/action.yml @@ -0,0 +1,55 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# The token scheduled-benchmarks is substituted by cargo-anvil at emit time with +# the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +name: anvil-scheduled-benchmarks +description: Run the scheduled-benchmarks check group. +inputs: + include_modified: + description: | + Pre-formatted --package args (e.g. "--package alpha@1.0.0 --package + beta@0.2.0") for the modified tier, or the sentinel "--skip" when + nothing modified. Packages are version-qualified cargo specs so they + resolve uniquely even when a like-named crate is also a transitive + dependency. Local invocations leave it unset; recipes default to + --workspace. + default: "" + required: false + include_affected: + description: | + Same shape as include_modified, but for the affected tier + (modified ∪ rev-deps). + default: "" + required: false + include_required: + description: | + Same shape as include_modified, but for the required tier + (affected ∪ workspace-internal transitive deps). + default: "" + required: false + free-disk-space: + description: Remove unused toolchains from GitHub-hosted runners before setup. + default: "false" + required: false +runs: + using: composite + steps: + - uses: ./.github/actions/anvil-setup + with: + group: scheduled-benchmarks + free-disk-space: ${{ inputs.free-disk-space }} + - name: Run just anvil-scheduled-benchmarks + shell: bash + env: + ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} + ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} + ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} + # Some checks (e.g. cargo-aprz, run only by groups that include it) + # hit the GitHub API; pass the built-in token so they use the + # authenticated quota (1000 vs 60 req/hr). Harmless for groups whose + # checks never read it. + GITHUB_TOKEN: ${{ github.token }} + run: just anvil-scheduled-benchmarks diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 01858bc48..1918e6df8 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -114,3 +114,105 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/anvil-scheduled-exhaustive + + scheduled-benchmarks: + # Benchmark regression detection. x86_64-only, matching + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # Only this job files the regression tracking issue. + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. + fetch-depth: 0 + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: bench-history-${{ matrix.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that + # carries the artifact. Restoring from the latest *successful* + # run would drop every sample collected while the pipeline was + # red from a regression — precisely the window that matters. + # The in-progress run of this very workflow has not uploaded + # yet, so it simply fails the download and the loop moves on. + for run_id in $(gh run list --workflow anvil-scheduled.yml \ + --branch "$DEFAULT_BRANCH" --limit 10 \ + --json databaseId --jq '.[].databaseId'); do + if gh run download "$run_id" --name "$ARTIFACT" \ + --dir target/anvil/bench-history 2>/dev/null; then + echo "restored benchmark history from run $run_id" + exit 0 + fi + done + echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + - uses: ./.github/actions/anvil-scheduled-benchmarks + - name: Save benchmark history + # always(): the run's own samples belong in the history even + # when the analysis flagged a regression and failed the job. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ matrix.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused + # or infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: File a benchmark regression issue + # Surfacing is by failed build; the issue carries the per-finding + # detail the one-bit build status cannot, and is updated in place + # so a regression appearing while the build is already red still + # reaches the author of its attributed commit. + if: failure() + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TITLE: Benchmark regressions detected (${{ matrix.os }}) + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + findings=target/anvil/bench/findings-summary.md + if [ ! -f "$findings" ]; then + echo "the job failed before producing findings; nothing to file" + exit 0 + fi + body=$(mktemp) + { + printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" + printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' + cat "$findings" + } > "$body" + number=$(gh issue list --state open --limit 100 --json number,title \ + --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --body-file "$body" + echo "updated issue #$number" + else + gh issue create --title "$TITLE" --body-file "$body" + fi diff --git a/.github/workflows/anvil-scheduled.yml b/.github/workflows/anvil-scheduled.yml index ecee2c3ea..31823ed0b 100644 --- a/.github/workflows/anvil-scheduled.yml +++ b/.github/workflows/anvil-scheduled.yml @@ -18,4 +18,10 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # The scheduled-benchmarks job files the regression tracking issue. + # A reusable workflow cannot grant itself more than the caller does, + # so the grant is repeated here and narrowed to that one job inside. + issues: write secrets: inherit diff --git a/crates/cargo-anvil/README.md b/crates/cargo-anvil/README.md index 2bfeb3b2f..ab98c6a7b 100644 --- a/crates/cargo-anvil/README.md +++ b/crates/cargo-anvil/README.md @@ -273,6 +273,7 @@ schedule against the default branch, not on PRs: scheduled-exhaustivemutants-full cargo-hackfeature powerset benchcompile-only + scheduled-benchmarksbench-historyregression detection over the accumulated benchmark history diff --git a/crates/cargo-anvil/docs/design/README.md b/crates/cargo-anvil/docs/design/README.md index bcdc85972..c4a77dbeb 100644 --- a/crates/cargo-anvil/docs/design/README.md +++ b/crates/cargo-anvil/docs/design/README.md @@ -385,6 +385,7 @@ pipeline. | `pr-test`, `pr-runtime-analysis`, `scheduled-test` | All legs above | Where compile-time and runtime OS / arch bugs actually surface. The three `pr-slow*` groups run as parallel cloud-workflow jobs (split out from a former single `pr-slow`) for shorter wall-clock per leg. | | `pr-mutants` | GH: Linux x86_64 + Windows x86_64 + Linux aarch64 (windows-arm self-skips). ADO: Linux x86_64 + Windows x86_64 | Diff-scoped mutation testing. cargo-mutants doesn't build on `aarch64-pc-windows-msvc`; the recipe self-skips so the windows-arm leg is a no-op. | | `scheduled-exhaustive` | Linux x86_64 + Windows x86_64 | Full `cargo-mutants` / `cargo-hack` / `bench`. cargo-mutants doesn't build on `aarch64-pc-windows-msvc`; rather than splitting the matrix to add an ARM-Linux leg for cargo-hack and bench, the whole group is x86-only. Adopters with ARM-specific concerns extend the matrix in their root workflow. | +| `scheduled-benchmarks` | Linux x86_64 + Windows x86_64 | Benchmark regression detection over the accumulated history. Matches `scheduled-exhaustive`; each leg carries its own history artifact because the series are partitioned per machine. See [benchmarks.md](./benchmarks.md). | macOS is not in the default matrix — adopters who need it fork the owned reusable workflow (GH) or override `testPools` (ADO). The GH-side knob set is intentionally diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 96eb034df..66994147f 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -211,9 +211,11 @@ Note the ADO topology differs from GitHub Actions in two places: ├── impact.yml owned (cargo-delta impact step; omitted if .delta.toml disabled) ├── job.yml owned-but-user-customizable │ (per-job wrapper; takes `name`, - │ `pool`, `steps`, `artifacts`; - │ users edit to inject 1ESPT - │ `templateContext:` etc.) + │ `pool`, `steps`, `fetchDepth`, + │ `artifacts`; users edit to inject + │ 1ESPT `templateContext:` etc.) + ├── bench-history-restore.yml owned (restore the benchmark history artifact) + ├── bench-history-summary.yml owned (attach benchmark findings to the build summary) ├── pr-fast.yml owned (one step template per group) ├── pr-test.yml owned ├── pr-runtime-analysis.yml owned @@ -233,8 +235,8 @@ customized by adopters whose ADO instance requires extension templates (1ES PT, SubstratePT, M365PT). Once a user edits it, the standard dirty-file flow kicks in — subsequent anvil updates Propose into a `.proposed` sibling rather than overwriting. The stages templates address the wrapper only via its -parameter contract (`name`, `pool`, `steps`, `artifacts`), so the wrapper can -diverge arbitrarily without blocking stage-shape updates. See §4.1. +parameter contract (`name`, `pool`, `steps`, `fetchDepth`, `artifacts`), so the +wrapper can diverge arbitrarily without blocking stage-shape updates. See §4.1. ## 3. Root pipelines @@ -377,6 +379,7 @@ The contract is intentionally small and stable: | `name` | `string` | yes | Job name; ADO derives the display name from it. | | `pool` | `object` | yes | Pool block, passed verbatim to ADO's `pool:` key. `linuxPool` and `windowsPool` at the stage level are object parameters, so users can override their shape (e.g. `{ name, os, image }` for 1ESPT). | | `steps` | `stepList` | yes | Body of the job. Templated step lists are fine — the wrapper splices them in via `${{ each step in parameters.steps }}: - ${{ step }}`. | +| `fetchDepth` | `string` | no | When set, the job checks out explicitly at this depth (`'0'` for full history) instead of taking the implicit default checkout. 1ESPT wrappers map it onto `templateContext.inputs` instead. | | `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. The stages templates don't need to know which backend they're targeting. | The default wrapper anvil ships is six lines of logic: @@ -386,11 +389,15 @@ parameters: - { name: name, type: string } - { name: pool, type: object } - { name: steps, type: stepList } + - { name: fetchDepth, type: string, default: '' } - { name: artifacts, type: object, default: [] } jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: + - ${{ if ne(parameters.fetchDepth, '') }}: + - checkout: self + fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: @@ -412,7 +419,7 @@ jobs: inputs: - input: checkout repository: self - fetchDepth: 0 + fetchDepth: ${{ parameters.fetchDepth }} outputs: - ${{ each artifact in parameters.artifacts }}: - output: pipelineArtifact @@ -832,23 +839,26 @@ removed from the catalog. The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs `cargo-bench-history`, whose history persists across scheduled runs as **pipeline artifacts**, reusing the §4.1 job-wrapper `artifacts` contract to publish. The -history is a small, durable, append-only store fetched as the latest from the -default branch, which pipeline artifacts' retention and `latestFromBranch` download -provide. +history is partitioned per machine, so each leg of the group's matrix carries its +own artifact (`bench-history-`). Each scheduled benchmark job: -1. checks out with full history (`fetchDepth: 0` — the §4.1 wrapper already exposes a - `checkout` input for this; analysis reads the commit graph); -2. **restores** the history with `DownloadPipelineArtifact@2` (`buildType: specific`, - `buildVersionToDownload: latestFromBranch`, the default branch); the first run +1. checks out with full history (the §4.1 wrapper's `fetchDepth` parameter; analysis + reads the commit graph); +2. **restores** the history with `DownloadPipelineArtifact@2` + (`buildVersionToDownload: latestFromBranch`, the default branch); the first run finds none and starts empty; -3. applies any pending blessings, runs collect + analyze, writing findings to the - build summary; +3. applies any pending blessings, runs collect + analyze, writing findings to a + findings file which a following step attaches to the build summary; 4. **publishes** the updated store through the wrapper's `artifacts` parameter - (`{ name: bench-history, path: }`), which the default wrapper emits as - `PublishPipelineArtifact@1` and 1ESPT wrappers as a `pipelineArtifact` output. A - retention lease on the latest successful scheduled run keeps the chain alive. + (`{ name: bench-history-, path: }`), which the default wrapper emits + as `PublishPipelineArtifact@1` and 1ESPT wrappers as a `pipelineArtifact` output. + +The restore admits failed and partially succeeded builds, which is what keeps the +chain intact across a regression: a flagged regression fails the stage, so a +success-only restore would discard every sample taken while the pipeline stayed red. +The publish likewise runs whatever the job's outcome. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index e97b7524a..0ed58047a 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -66,14 +66,27 @@ when cbh reports an active regression**; locally and in cloud it behaves identically (always writes its findings; only the exit code gates), matching the local-vs-cloud parity every recipe keeps. +The check is **unscoped**: it ignores the impact-analysis include contract every +other check honors. A series is only comparable when the same suite is measured at +every commit, so measuring an impact-scoped subset would punch holes in the history +that detection cannot tell apart from a benchmark being deleted. + ## 4. History as cross-run state Each scheduled run checks out with full history (analysis reads the commit graph -to order series and locate the base merge-base), restores the latest history -**artifact from the default branch**, runs collect → apply-blessings → analyze, -and publishes the updated history as this run's artifact. Because each run -republishes the whole accumulated directory, only the newest snapshot is ever -needed. +to order series and locate the base merge-base), restores the history published by +the most recent run on the **default branch that carries one**, runs collect → +apply-blessings → analyze, and publishes the updated history as this run's +artifact. Because each run republishes the whole accumulated directory, only the +newest snapshot is ever needed. + +Both the restore and the publish are indifferent to the run's outcome. A flagged +regression fails the build, so restoring only from green runs — or publishing only +on success — would throw away exactly the samples taken while the pipeline stayed +red, which is the stretch of history a reviewer most needs. + +Every series is partitioned by a machine key, so each leg of the group's matrix +carries its own artifact rather than sharing one. The persistence is a **rolling window** on CI-native artifacts — portable and zero-config, and on eviction it degrades to a harmless cold start, since @@ -130,13 +143,18 @@ a reviewed pull request accepting the change → the next scheduled run applies and the build returns to green. The accumulated entries are an audit trail of every deliberate tradeoff. +Idempotence comes from reconciling the committed entries against the blessings +already recorded in the store, so re-running the job never re-appends a sidecar +that is already in effect. + ## 8. Boundaries and caveats - **Hosted-runner machine-key density.** cbh partitions by a hardware fingerprint; a heterogeneous hosted pool can split a series into per-key partitions too sparse to analyze. Whether a hosted pool stays dense enough depends on its hardware homogeneity; self-hosted or dedicated runners avoid the - concern. + concern. An adopter who knows their pool is uniform enough can substitute a + stable pool label for the fingerprint, trading partition fidelity for density. - **Attribution is coarse under sparse benchmarking.** Benches do not run on every commit, so the attributed commit is the first *benchmarked* one after a regression and may bundle several changes — an honest range, not always a diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index f53dd7d32..74d45b384 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -271,6 +271,15 @@ The `miri` row above is the one place the catalog deliberately duplicates a chec | `cargo-hack` powerset | `cargo hack --workspace --feature-powerset --depth 2 check` | oxidizer, oxidizer-github | | `bench` | `cargo bench --workspace --all-features --no-run` ＋ a single-iteration smoke benchmark for each bench target | oxidizer | +### `scheduled-benchmarks` + +| Check | Invocation | Source | +|----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| +| `bench-history` | `cargo bench-history` collect → apply-blessings → analyze over the history restored from the previous scheduled run; exits non-zero on an active regression. | new | + +Unlike the compile-only `bench` above, this check actually *runs* the benchmarks and +judges the resulting trend. See [benchmarks.md](./benchmarks.md). + ## 3. Per-check vs grouped cloud workflows execution Each *group* is one cloud-workflow job. Within a job, the checks belonging to the group run sequentially @@ -353,7 +362,7 @@ Bucket assignments per check: | modified | `fmt`, `cargo-sort`, `license-headers`, `ensure-no-cyclic-deps`, `ensure-no-default-features`, `readme-check`, `spellcheck` | | affected | `clippy`*, `llvm-cov`, `doc-test`, `examples`, `mutants` (diff and full), `miri`, `careful`, `loom`, `bolero`, `semver-check`, `external-types`, `bench` | | required | `doc-build`, `udeps`, `cargo-hack` (feature powerset) | -| unscoped | `pr-title`, `deny`, `audit`, `aprz`, `mutants-full`, `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage` | +| unscoped | `pr-title`, `deny`, `audit`, `aprz`, `mutants-full`, `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage`, `bench-history` | \* cargo-delta's README recommends `clippy` with the modified tier. anvil deliberately runs it on the affected set instead: a change in a crate's API can introduce clippy lints @@ -369,7 +378,8 @@ deps), `cargo udeps` (unused-deps detection needs the resolved graph), `cargo ha `unscoped` is for checks that have nothing to do with workspace-member identity: `deny`/`audit` read `Cargo.lock`, `pr-title` reads PR metadata, `aprz` consults an -external risk DB. These ignore the env vars and always run. +external risk DB, and `bench-history` needs the same suite measured at every commit +for its series to stay comparable. These ignore the env vars and always run. The sentinel `--skip` is a magic string that cannot be a valid cargo argument, so there is no collision with real package names. Recipes test for it with diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 0a1d3ec46..22279c607 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -763,29 +763,38 @@ clearable when a check is removed from the catalog. The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs `cargo-bench-history`, whose history persists across scheduled runs as GitHub -**Actions artifacts**: a small, durable, append-only store fetched as the latest -from the default branch, which artifacts' retention and cross-run download provide. +**Actions artifacts**. The history is partitioned per machine, so each leg of the +group's matrix carries its own artifact (`bench-history-`) — which also keeps +the names distinct within a run, as artifact upload requires. Each scheduled benchmark job: 1. checks out with `fetch-depth: 0` (analysis reads the commit graph); -2. **restores** the history by downloading the `bench-history` artifact from the - most recent successful `anvil-scheduled` run on the default branch (a small step - queries the runs API for the latest success, then `actions/download-artifact` - fetches it by run id); the first run finds none and starts empty; +2. **restores** the history by walking back from the newest `anvil-scheduled` run + on the default branch and taking the first that carries the leg's artifact; the + first run finds none and starts empty; 3. applies any pending blessings, runs collect + analyze, writing findings to the job summary and to a findings file; -4. **saves** the updated store with `actions/upload-artifact` (name `bench-history`); - retention is set so the latest successful run's artifact outlives the gap to the - next scheduled run. +4. **saves** the updated store with `actions/upload-artifact`, whatever the job's + outcome, so the samples collected while the pipeline is red are not lost. + Retention is set so the latest artifact outlives the gap to the next scheduled + run. + +Restoring from the newest run that *carries* the artifact rather than the newest +*successful* one is what keeps the chain intact across a regression: a flagged +regression fails the job, so a success-only restore would discard every sample +taken while the pipeline stayed red. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits non-zero on an active regression, failing the job. The scheduled workflow's failure path creates-or-updates a tracking **issue** from the findings file — updated in place each run so concurrent regressions and the authors of their attributed commits -surface even while the build is already red. This needs `issues: write`, declared on -the scheduled job only; the PR workflow keeps `contents: read`. +surface even while the build is already red. This needs `issues: write` and, for the +restore step's runs/artifacts queries, `actions: read`. A reusable workflow cannot +grant itself more than its caller, so the root workflow passes both through and the +impl workflow narrows them to the benchmark job; the PR workflow keeps +`contents: read`. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3), so accepting an intentional change is a reviewed pull request rather than an diff --git a/crates/cargo-anvil/docs/design/local.md b/crates/cargo-anvil/docs/design/local.md index 7da2745b4..4ef9fca83 100644 --- a/crates/cargo-anvil/docs/design/local.md +++ b/crates/cargo-anvil/docs/design/local.md @@ -123,6 +123,8 @@ anvil-pr-mutants: anvil-mutants-diff anvil-scheduled-test: anvil-llvm-cov anvil-doc-test anvil-examples anvil-scheduled-advisories: anvil-deny anvil-audit anvil-aprz anvil-clippy +anvil-scheduled-runtime-analysis: anvil-miri anvil-miri-tree-borrows \ + anvil-miri-strict-provenance anvil-miri-race-coverage anvil-scheduled-exhaustive: anvil-mutants-full anvil-cargo-hack anvil-bench anvil-scheduled-benchmarks: anvil-bench-history ``` @@ -139,7 +141,8 @@ in a deterministic order: ```just anvil-pr: anvil-pr-validate-prereqs anvil-pr-fast anvil-pr-slow anvil-scheduled: anvil-scheduled-validate-prereqs anvil-scheduled-test anvil-scheduled-advisories \ - anvil-scheduled-exhaustive anvil-scheduled-benchmarks + anvil-scheduled-runtime-analysis anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks anvil-full: anvil-pr anvil-scheduled ``` diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index 019d88fa3..c5de5de11 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -66,8 +66,9 @@ Emitted-file snapshots gain the actions/steps and workflow/stage entries. ## Phase 5 — Bless application Define the `.config/bench-blessings.toml` schema and the idempotent apply step that -runs ahead of analyze in the scheduled job, skipping entries already present per -`cbh list blessings`. A fixture repo exercises red → bless → green. +runs ahead of analyze in the scheduled job, reconciling the committed entries against +what `cbh list blessings` reports as already recorded. A fixture repo exercises +red → bless → green. [cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index 591e57609..46ce0ed1f 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -22,6 +22,12 @@ const ADVISORY_COMMENTS_STEP: &str = include_str!("../../../templates/ado/steps/ /// Embedded body of the dirty-file job wrapper. const JOB_WRAPPER: &str = include_str!("../../../templates/ado/steps/job.yml"); +/// Embedded body of the benchmark-history restore step template. +const BENCH_HISTORY_RESTORE_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-restore.yml"); + +/// Embedded body of the benchmark-findings build-summary step template. +const BENCH_HISTORY_SUMMARY_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-summary.yml"); + /// Embedded body of the PR-tier stages template. const PR_STAGES: &str = include_str!("../../../templates/ado/pr-stages.yml"); @@ -54,6 +60,7 @@ const GROUPS: &[&str] = &[ "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ]; /// Embedded template for one per-group step. `__GROUP__` is substituted with @@ -100,6 +107,28 @@ pub fn job_wrapper() -> Artifact { Artifact::backend_file(Backend::Ado, ".pipelines/anvil/steps/job.yml", JOB_WRAPPER) } +/// `.pipelines/anvil/steps/bench-history-restore.yml` — restores the +/// benchmark history the previous scheduled run published. +#[must_use] +pub fn bench_history_restore() -> Artifact { + Artifact::backend_file( + Backend::Ado, + ".pipelines/anvil/steps/bench-history-restore.yml", + BENCH_HISTORY_RESTORE_STEP, + ) +} + +/// `.pipelines/anvil/steps/bench-history-summary.yml` — attaches the +/// benchmark findings to the build summary. +#[must_use] +pub fn bench_history_summary() -> Artifact { + Artifact::backend_file( + Backend::Ado, + ".pipelines/anvil/steps/bench-history-summary.yml", + BENCH_HISTORY_SUMMARY_STEP, + ) +} + /// `.pipelines/anvil/pr.yml` — the PR-tier stages template. #[must_use] pub fn pr_stages() -> Artifact { @@ -164,12 +193,20 @@ pub(crate) const GROUP_STEPS: &[(&str, &str)] = &[ ".pipelines/anvil/steps/scheduled-runtime-analysis.yml", ), ("scheduled-exhaustive", ".pipelines/anvil/steps/scheduled-exhaustive.yml"), + ("scheduled-benchmarks", ".pipelines/anvil/steps/scheduled-benchmarks.yml"), ]; /// All ADO backend artifacts in emission order. #[must_use] pub(crate) fn all() -> Vec { - let mut out = vec![setup_step(), impact_step(), advisory_comments(), job_wrapper()]; + let mut out = vec![ + setup_step(), + impact_step(), + advisory_comments(), + job_wrapper(), + bench_history_restore(), + bench_history_summary(), + ]; for (group, path) in GROUP_STEPS { out.push(Artifact::backend_file(Backend::Ado, path, render_group_step(group))); } @@ -258,6 +295,7 @@ mod tests { "name: steps", "type: stepList", "name: artifacts", + "name: fetchDepth", "PublishPipelineArtifact@1", ] { assert!(JOB_WRAPPER.contains(needle), "wrapper missing '{needle}'"); @@ -343,6 +381,7 @@ mod tests { "stage: scheduled_advisories", "stage: scheduled_runtime_analysis", "stage: scheduled_exhaustive", + "stage: scheduled_benchmarks", ] { assert!(SCHEDULED_STAGES.contains(needle), "scheduled stages missing '{needle}'"); } @@ -354,6 +393,36 @@ mod tests { ); } + #[test] + fn scheduled_benchmarks_stage_round_trips_the_history_artifact() { + // Analysis walks the commit graph, so both legs check out fully. + assert_eq!( + SCHEDULED_STAGES.matches("fetchDepth: '0'").count(), + 2, + "both benchmark legs must check out the full history" + ); + // Per-leg artifact names: the history is partitioned per machine. + for needle in ["bench-history-linux", "bench-history-windows"] { + assert_eq!( + SCHEDULED_STAGES.matches(needle).count(), + 2, + "the restore and publish sides must agree on the artifact name '{needle}'" + ); + } + assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-restore.yml")); + assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-summary.yml")); + // Take the newest run carrying the artifact whatever its outcome: + // restoring only from green runs would drop every sample collected + // while the pipeline was red from a regression. + assert!(BENCH_HISTORY_RESTORE_STEP.contains("buildVersionToDownload: latestFromBranch")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("allowFailedBuilds: true")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("allowPartiallySucceededBuilds: true")); + // A missing artifact is a cold start, not a failure. + assert!(BENCH_HISTORY_RESTORE_STEP.contains("continueOnError: true")); + assert!(BENCH_HISTORY_SUMMARY_STEP.contains("##vso[task.uploadsummary]")); + assert!(BENCH_HISTORY_SUMMARY_STEP.contains("condition: succeededOrFailed()")); + } + #[test] fn custom_stages_stubs_are_empty_and_take_pool_parameters() { // The extension stubs must emit a valid empty stages list (so the diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index 783dad67b..af9964f29 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -45,6 +45,7 @@ const GROUPS: &[&str] = &[ "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ]; /// Embedded template for one per-group composite action. `__GROUP__` is @@ -123,6 +124,7 @@ pub(crate) const GROUP_ACTIONS: &[(&str, &str)] = &[ ".github/actions/anvil-scheduled-runtime-analysis/action.yml", ), ("scheduled-exhaustive", ".github/actions/anvil-scheduled-exhaustive/action.yml"), + ("scheduled-benchmarks", ".github/actions/anvil-scheduled-benchmarks/action.yml"), ]; /// All GitHub backend artifacts in emission order. @@ -244,6 +246,7 @@ mod tests { "scheduled-advisories:", "scheduled-runtime-analysis:", "scheduled-exhaustive:", + "scheduled-benchmarks:", ] { assert!( SCHEDULED_IMPL_WORKFLOW.contains(needle), @@ -258,6 +261,31 @@ mod tests { ); } + #[test] + fn scheduled_benchmarks_job_round_trips_the_history_artifact() { + // Analysis walks the commit graph, so the leg needs full history. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); + // Per-leg artifact names: the history is partitioned per machine, + // and upload-artifact rejects a name reused within one run. + assert_eq!( + SCHEDULED_IMPL_WORKFLOW.matches("bench-history-${{ matrix.os }}").count(), + 2, + "the restore and save steps must agree on the per-leg artifact name" + ); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions/upload-artifact@")); + // Saving on failure too: the samples collected while the pipeline + // is red from a regression are the ones that matter. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh run download")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh issue create")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh issue edit")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("issues: write")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("GITHUB_STEP_SUMMARY")); + // The reusable workflow cannot grant itself more than the caller, + // so the root workflow must pass the same permission through. + assert!(SCHEDULED_ROOT_WORKFLOW.contains("issues: write")); + assert!(SCHEDULED_ROOT_WORKFLOW.contains("actions: read")); + } + #[test] fn root_workflows_call_reusable_workflows() { assert!(PR_ROOT_WORKFLOW.contains("uses: ./.github/workflows/anvil-pr-impl.yml")); diff --git a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs index f2ee74d17..1c6a13d11 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs @@ -102,6 +102,7 @@ const CHECK_FILES: &[(&str, &str)] = split_recipe_files!( "aprz", "audit", "bench", + "bench-history", "bolero", "careful", "cargo-hack", @@ -146,6 +147,7 @@ const GROUP_FILES: &[(&str, &str)] = split_recipe_files!( "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ] ); @@ -313,6 +315,7 @@ mod tests { "anvil-scheduled-test:", "anvil-scheduled-advisories:", "anvil-scheduled-exhaustive:", + "anvil-scheduled-benchmarks:", ] { assert!(groups.contains(needle), "groups tree missing '{needle}'"); } @@ -331,6 +334,7 @@ mod tests { "anvil-scheduled-advisories: anvil-scheduled-advisories-validate-prereqs", "anvil-scheduled-runtime-analysis: anvil-scheduled-runtime-analysis-validate-prereqs", "anvil-scheduled-exhaustive: anvil-scheduled-exhaustive-validate-prereqs", + "anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs", ] { assert!( groups.contains(needle), @@ -339,6 +343,36 @@ mod tests { } } + #[test] + fn bench_history_gates_on_active_regressions_and_applies_blessings() { + let (_, body) = CHECK_FILES + .iter() + .find(|(path, _)| *path == "justfiles/anvil/checks/bench-history.just") + .expect("bench-history check template is registered"); + + // cargo-bench-history never fails on findings, so the recipe must + // read the JSON report and gate on active regressions itself. + for needle in [ + "cargo bench-history collect", + "--skip-existing", + "cargo bench-history analyze", + "_anvil-bench-history-bless", + "cargo bench-history bless", + "cargo bench-history list blessings", + ".config/bench-blessings.toml", + "$_.direction -eq 'regression' -and $_.active", + ] { + assert!(body.contains(needle), "bench-history template missing '{needle}'"); + } + + // The series is only comparable when the same suite is measured at + // every commit, so the recipe must not honour impact scoping. + assert!( + !body.contains("ANVIL_INCLUDE_AFFECTED"), + "bench-history must measure the whole workspace, not an impact-scoped subset" + ); + } + #[test] fn tiers_just_template_has_three_tiers() { for needle in [ @@ -370,6 +404,7 @@ mod tests { "anvil-scheduled-advisories", "anvil-scheduled-runtime-analysis", "anvil-scheduled-exhaustive", + "anvil-scheduled-benchmarks", ] { assert!(TIERS_JUST.contains(needle), "scheduled tier must reference group '{needle}'"); } @@ -409,6 +444,7 @@ mod tests { "import 'container/container.just'", "import 'groups/pr-fast.just'", "import 'groups/scheduled-exhaustive.just'", + "import 'groups/scheduled-benchmarks.just'", "import 'runner.just'", "import 'tiers.just'", "import 'tools.just'", diff --git a/crates/cargo-anvil/src/lib.rs b/crates/cargo-anvil/src/lib.rs index 734de6abc..7f06e307f 100644 --- a/crates/cargo-anvil/src/lib.rs +++ b/crates/cargo-anvil/src/lib.rs @@ -272,6 +272,7 @@ //! scheduled-exhaustivemutants-full //! cargo-hackfeature powerset //! benchcompile-only +//! scheduled-benchmarksbench-historyregression detection over the accumulated benchmark history //! //! //! diff --git a/crates/cargo-anvil/src/run.rs b/crates/cargo-anvil/src/run.rs index cede567ac..573d65096 100644 --- a/crates/cargo-anvil/src/run.rs +++ b/crates/cargo-anvil/src/run.rs @@ -527,6 +527,7 @@ mod tests { "justfiles/anvil/checks/miri.just", "justfiles/anvil/groups/pr-fast.just", "justfiles/anvil/groups/scheduled-exhaustive.just", + "justfiles/anvil/groups/scheduled-benchmarks.just", "justfiles/anvil/tiers.just", "justfiles/anvil/tools.just", "justfiles/anvil/versions.just", @@ -878,6 +879,7 @@ mod tests { ".github/actions/anvil-scheduled-advisories/action.yml", ".github/actions/anvil-scheduled-runtime-analysis/action.yml", ".github/actions/anvil-scheduled-exhaustive/action.yml", + ".github/actions/anvil-scheduled-benchmarks/action.yml", ".github/workflows/anvil-pr-impl.yml", ".github/workflows/anvil-scheduled-impl.yml", ".github/workflows/anvil-pr.yml", @@ -931,6 +933,9 @@ mod tests { ".pipelines/anvil/steps/scheduled-advisories.yml", ".pipelines/anvil/steps/scheduled-runtime-analysis.yml", ".pipelines/anvil/steps/scheduled-exhaustive.yml", + ".pipelines/anvil/steps/scheduled-benchmarks.yml", + ".pipelines/anvil/steps/bench-history-restore.yml", + ".pipelines/anvil/steps/bench-history-summary.yml", ".pipelines/anvil/pr.yml", ".pipelines/anvil/scheduled.yml", ".pipelines/anvil-pr.yml", diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index 2f788a752..669683e64 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -106,3 +106,46 @@ stages: pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-exhaustive.yml + + - stage: scheduled_benchmarks + displayName: anvil scheduled-benchmarks + dependsOn: [] + jobs: + # Benchmark regression detection. OS scope matches + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + # The restore step runs first; the wrapper's `artifacts` contract + # publishes the updated store at the end of the job -- including + # when the analysis failed the job, so the samples collected while + # the pipeline is red are not lost. + - template: steps/job.yml + parameters: + name: linux + pool: ${{ parameters.linuxPool }} + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. + fetchDepth: '0' + artifacts: + - name: bench-history-linux + path: target/anvil/bench-history + steps: + - template: steps/bench-history-restore.yml + parameters: + artifact: bench-history-linux + - template: steps/scheduled-benchmarks.yml + - template: steps/bench-history-summary.yml + - template: steps/job.yml + parameters: + name: windows + pool: ${{ parameters.windowsPool }} + fetchDepth: '0' + artifacts: + - name: bench-history-windows + path: target/anvil/bench-history + steps: + - template: steps/bench-history-restore.yml + parameters: + artifact: bench-history-windows + - template: steps/scheduled-benchmarks.yml + - template: steps/bench-history-summary.yml diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml new file mode 100644 index 000000000..4d97099d8 --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -0,0 +1,39 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Restores the benchmark history the previous scheduled run published. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history +steps: + # The store has to exist even when there is nothing to restore, so the + # first ever run analyzes an empty history instead of erroring. + - pwsh: New-Item -ItemType Directory -Force -Path '${{ parameters.path }}' | Out-Null + displayName: Prepare benchmark history store + - task: DownloadPipelineArtifact@2 + displayName: Restore ${{ parameters.artifact }} + # The first run has no artifact to restore; a missing artifact is a + # cold start, not a failure. + continueOnError: true + inputs: + buildType: specific + project: $(System.TeamProjectId) + definition: $(System.DefinitionId) + buildVersionToDownload: latestFromBranch + branchName: $(Build.SourceBranch) + # Take the newest run that carries the artifact whatever its + # outcome. Restoring only from green runs would drop every sample + # collected while the pipeline was red from a regression -- + # precisely the window that matters. + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + artifactName: ${{ parameters.artifact }} + targetPath: ${{ parameters.path }} diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml new file mode 100644 index 000000000..7f67525cc --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Attaches the benchmark findings to the build summary, so a failed +# scheduled build carries the per-finding detail its one-bit status +# cannot. Runs whether or not the analysis flagged anything. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: findings + type: string + default: target/anvil/bench/findings.md +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $findings = '${{ parameters.findings }}' + if (-not (Test-Path -LiteralPath $findings)) { + Write-Host "anvil: no $findings to attach" + exit 0 + } + $full = (Resolve-Path -LiteralPath $findings).Path + Write-Host "##vso[task.uploadsummary]$full" + displayName: Publish benchmark findings + condition: succeededOrFailed() diff --git a/crates/cargo-anvil/templates/ado/steps/job.yml b/crates/cargo-anvil/templates/ado/steps/job.yml index 028b1ccd2..e7275899d 100644 --- a/crates/cargo-anvil/templates/ado/steps/job.yml +++ b/crates/cargo-anvil/templates/ado/steps/job.yml @@ -17,6 +17,10 @@ # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# - fetchDepth (string) Optional. When set, the job checks out explicitly at +# this depth ('0' for full history) instead of taking +# the implicit default checkout. 1ESPT wrappers map it +# onto templateContext.inputs instead. # - artifacts (object) Optional list of pipeline artifacts to publish. # Each item: { name: string, path: string }. # The default wrapper appends one @@ -33,6 +37,9 @@ parameters: type: object - name: steps type: stepList + - name: fetchDepth + type: string + default: '' - name: artifacts type: object default: [] @@ -41,6 +48,9 @@ jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: + - ${{ if ne(parameters.fetchDepth, '') }}: + - checkout: self + fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index 01858bc48..1918e6df8 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -114,3 +114,105 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/anvil-scheduled-exhaustive + + scheduled-benchmarks: + # Benchmark regression detection. x86_64-only, matching + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # Only this job files the regression tracking issue. + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. + fetch-depth: 0 + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: bench-history-${{ matrix.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that + # carries the artifact. Restoring from the latest *successful* + # run would drop every sample collected while the pipeline was + # red from a regression — precisely the window that matters. + # The in-progress run of this very workflow has not uploaded + # yet, so it simply fails the download and the loop moves on. + for run_id in $(gh run list --workflow anvil-scheduled.yml \ + --branch "$DEFAULT_BRANCH" --limit 10 \ + --json databaseId --jq '.[].databaseId'); do + if gh run download "$run_id" --name "$ARTIFACT" \ + --dir target/anvil/bench-history 2>/dev/null; then + echo "restored benchmark history from run $run_id" + exit 0 + fi + done + echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + - uses: ./.github/actions/anvil-scheduled-benchmarks + - name: Save benchmark history + # always(): the run's own samples belong in the history even + # when the analysis flagged a regression and failed the job. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ matrix.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused + # or infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: File a benchmark regression issue + # Surfacing is by failed build; the issue carries the per-finding + # detail the one-bit build status cannot, and is updated in place + # so a regression appearing while the build is already red still + # reaches the author of its attributed commit. + if: failure() + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TITLE: Benchmark regressions detected (${{ matrix.os }}) + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + findings=target/anvil/bench/findings-summary.md + if [ ! -f "$findings" ]; then + echo "the job failed before producing findings; nothing to file" + exit 0 + fi + body=$(mktemp) + { + printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" + printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' + cat "$findings" + } > "$body" + number=$(gh issue list --state open --limit 100 --json number,title \ + --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --body-file "$body" + echo "updated issue #$number" + else + gh issue create --title "$TITLE" --body-file "$body" + fi diff --git a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml index ecee2c3ea..31823ed0b 100644 --- a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml @@ -18,4 +18,10 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # The scheduled-benchmarks job files the regression tracking issue. + # A reusable workflow cannot grant itself more than the caller does, + # so the grant is repeated here and narrowed to that one job inside. + issues: write secrets: inherit diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just new file mode 100644 index 000000000..b1d7e3c85 --- /dev/null +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores the +# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs. Depending on a TOML parser here would mean a + # second tool pin for three string fields. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = ($rawLine -split '#', 2)[0].Trim() + if (-not $line) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Stored commits are abbreviated, and a stored blessing names + # either the concrete benchmark it resolved to (once a run exists + # at that commit) or the prefix filter it was issued with. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and + (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + & cargo bench-history bless --local="$store" --context $resolved @key $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs diff --git a/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just b/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just new file mode 100644 index 000000000..4ca0c121a --- /dev/null +++ b/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs diff --git a/crates/cargo-anvil/templates/justfiles/anvil/mod.just b/crates/cargo-anvil/templates/justfiles/anvil/mod.just index 2a3741105..85afe11a4 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/mod.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/mod.just @@ -39,6 +39,7 @@ import 'helpers.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -76,6 +77,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'runner.just' import 'tiers.just' import 'tools.just' diff --git a/crates/cargo-anvil/templates/justfiles/anvil/tiers.just b/crates/cargo-anvil/templates/justfiles/anvil/tiers.just index 05038e0ea..f964a72c7 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/tiers.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/tiers.just @@ -33,7 +33,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full tier: PR + scheduled, end-to-end. Useful before tagging a release. [group("anvil")] @@ -74,7 +75,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -82,7 +84,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] diff --git a/crates/cargo-anvil/templates/justfiles/anvil/tools.just b/crates/cargo-anvil/templates/justfiles/anvil/tools.just index 2bfad3528..1a73d8dc2 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/tools.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/tools.just @@ -532,6 +532,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the diff --git a/crates/cargo-anvil/templates/justfiles/anvil/versions.just b/crates/cargo-anvil/templates/justfiles/anvil/versions.just index c7a32d8f1..38882231b 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/versions.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/versions.just @@ -52,6 +52,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.0.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.9" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/schemas.rs b/crates/cargo-anvil/tests/schemas.rs index 4377e163c..85804bd8e 100644 --- a/crates/cargo-anvil/tests/schemas.rs +++ b/crates/cargo-anvil/tests/schemas.rs @@ -97,6 +97,18 @@ fn taplo_validates_emitted_toml_files() { #[test] fn actionlint_validates_emitted_workflows() { let tmp = run_with_backend("github"); + // actionlint refuses to run outside a git repository ("no project was + // found in any parent directories"), so the generated tree has to look + // like one before the workflows can be validated. + let Some(init) = try_run(Command::new("git").args(["init", "--quiet"]).current_dir(tmp.path())) else { + eprintln!("skipping: git not installed"); + return; + }; + assert!( + init.status.success(), + "git init failed in the fixture:\n{}", + String::from_utf8_lossy(&init.stderr) + ); let mut cmd = Command::new("actionlint"); cmd.current_dir(tmp.path()); let Some(out) = try_run(&mut cmd) else { @@ -141,6 +153,7 @@ fn just_lists_emitted_recipes() { ("anvil-scheduled", "# Run all scheduled checks."), ("anvil-scheduled-advisories", "# Run the scheduled advisory checks."), ("anvil-scheduled-exhaustive", "# Run the scheduled exhaustive checks."), + ("anvil-scheduled-benchmarks", "# Run the scheduled benchmark regression detection."), ("anvil-scheduled-runtime-analysis", "# Run the scheduled runtime analysis."), ("anvil-scheduled-test", "# Run the scheduled tests."), ] { diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index 4724e150c..d6d9f5a2d 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -449,6 +449,49 @@ stages: steps: - template: steps/scheduled-exhaustive.yml + - stage: scheduled_benchmarks + displayName: anvil scheduled-benchmarks + dependsOn: [] + jobs: + # Benchmark regression detection. OS scope matches + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + # The restore step runs first; the wrapper's `artifacts` contract + # publishes the updated store at the end of the job -- including + # when the analysis failed the job, so the samples collected while + # the pipeline is red are not lost. + - template: steps/job.yml + parameters: + name: linux + pool: ${{ parameters.linuxPool }} + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. + fetchDepth: '0' + artifacts: + - name: bench-history-linux + path: target/anvil/bench-history + steps: + - template: steps/bench-history-restore.yml + parameters: + artifact: bench-history-linux + - template: steps/scheduled-benchmarks.yml + - template: steps/bench-history-summary.yml + - template: steps/job.yml + parameters: + name: windows + pool: ${{ parameters.windowsPool }} + fetchDepth: '0' + artifacts: + - name: bench-history-windows + path: target/anvil/bench-history + steps: + - template: steps/bench-history-restore.yml + parameters: + artifact: bench-history-windows + - template: steps/scheduled-benchmarks.yml + - template: steps/bench-history-summary.yml + === .pipelines/anvil/steps/advisory-comments.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -547,6 +590,76 @@ steps: } } +=== .pipelines/anvil/steps/bench-history-restore.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Restores the benchmark history the previous scheduled run published. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history +steps: + # The store has to exist even when there is nothing to restore, so the + # first ever run analyzes an empty history instead of erroring. + - pwsh: New-Item -ItemType Directory -Force -Path '${{ parameters.path }}' | Out-Null + displayName: Prepare benchmark history store + - task: DownloadPipelineArtifact@2 + displayName: Restore ${{ parameters.artifact }} + # The first run has no artifact to restore; a missing artifact is a + # cold start, not a failure. + continueOnError: true + inputs: + buildType: specific + project: $(System.TeamProjectId) + definition: $(System.DefinitionId) + buildVersionToDownload: latestFromBranch + branchName: $(Build.SourceBranch) + # Take the newest run that carries the artifact whatever its + # outcome. Restoring only from green runs would drop every sample + # collected while the pipeline was red from a regression -- + # precisely the window that matters. + allowPartiallySucceededBuilds: true + allowFailedBuilds: true + artifactName: ${{ parameters.artifact }} + targetPath: ${{ parameters.path }} + +=== .pipelines/anvil/steps/bench-history-summary.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Attaches the benchmark findings to the build summary, so a failed +# scheduled build carries the per-finding detail its one-bit status +# cannot. Runs whether or not the analysis flagged anything. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: findings + type: string + default: target/anvil/bench/findings.md +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $findings = '${{ parameters.findings }}' + if (-not (Test-Path -LiteralPath $findings)) { + Write-Host "anvil: no $findings to attach" + exit 0 + } + $full = (Resolve-Path -LiteralPath $findings).Path + Write-Host "##vso[task.uploadsummary]$full" + displayName: Publish benchmark findings + condition: succeededOrFailed() + === .pipelines/anvil/steps/impact.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -665,6 +778,10 @@ steps: # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# - fetchDepth (string) Optional. When set, the job checks out explicitly at +# this depth ('0' for full history) instead of taking +# the implicit default checkout. 1ESPT wrappers map it +# onto templateContext.inputs instead. # - artifacts (object) Optional list of pipeline artifacts to publish. # Each item: { name: string, path: string }. # The default wrapper appends one @@ -681,6 +798,9 @@ parameters: type: object - name: steps type: stepList + - name: fetchDepth + type: string + default: '' - name: artifacts type: object default: [] @@ -689,6 +809,9 @@ jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: + - ${{ if ne(parameters.fetchDepth, '') }}: + - checkout: self + fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: @@ -999,6 +1122,66 @@ steps: ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} +=== .pipelines/anvil/steps/scheduled-benchmarks.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# The token scheduled-benchmarks is substituted by cargo-anvil at emit time with +# the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +parameters: + - name: include_modified + type: string + default: '' + - name: include_affected + type: string + default: '' + - name: include_required + type: string + default: '' +steps: + - template: setup.yml + parameters: + group: scheduled-benchmarks + # ADO has no PR-title predefined variable: `System.PullRequest.Title` + # does NOT exist, so `$(System.PullRequest.Title)` would expand to the + # literal unexpanded macro text (non-empty) and make anvil-pr-title + # validate that string. Resolve the real title from the REST API on PR + # builds and publish it as the PR_TITLE pipeline variable. Only + # anvil-pr-title (in pr-fast) consults it; other groups ignore it, but we + # resolve uniformly to keep group.yml the same across groups. Best-effort: + # on a non-PR build, a fork PR with a restricted token, or any API error, + # PR_TITLE is left empty and anvil-pr-title skips. + - pwsh: | + $ErrorActionPreference = 'Stop' + $prId = $env:SYSTEM_PULLREQUEST_PULLREQUESTID + if (-not $prId) { + Write-Host 'anvil: not a PR build; leaving PR_TITLE empty' + Write-Host '##vso[task.setvariable variable=PR_TITLE]' + exit 0 + } + $uri = "$($env:SYSTEM_COLLECTIONURI)$($env:SYSTEM_TEAMPROJECTID)/_apis/git/repositories/$($env:BUILD_REPOSITORY_ID)/pullRequests/${prId}?api-version=7.0" + try { + $resp = Invoke-RestMethod -Uri $uri -Headers @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + Write-Host "##vso[task.setvariable variable=PR_TITLE]$($resp.title)" + } catch { + Write-Host "anvil: could not resolve PR title ($_); leaving PR_TITLE empty" + Write-Host '##vso[task.setvariable variable=PR_TITLE]' + } + displayName: anvil-scheduled-benchmarks (resolve PR title) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - bash: just anvil-scheduled-benchmarks + displayName: anvil-scheduled-benchmarks + env: + # Resolved by the preceding step (empty on non-PR / fork / API + # failure, in which case anvil-pr-title skips). + PR_TITLE: $(PR_TITLE) + ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} + ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} + ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} + === .pipelines/anvil/steps/scheduled-exhaustive.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -1588,6 +1771,213 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores the +# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs. Depending on a TOML parser here would mean a + # second tool pin for three string fields. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = ($rawLine -split '#', 2)[0].Trim() + if (-not $line) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Stored commits are abbreviated, and a stored blessing names + # either the concrete benchmark it resolved to (once a run exists + # at that commit) or the prefix filter it was issued with. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and + (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + & cargo bench-history bless --local="$store" --context $resolved @key $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4640,6 +5030,35 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -5011,6 +5430,7 @@ import 'helpers.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -5048,6 +5468,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'runner.just' import 'tiers.just' import 'tools.just' @@ -5140,7 +5561,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full tier: PR + scheduled, end-to-end. Useful before tagging a release. [group("anvil")] @@ -5181,7 +5603,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -5189,7 +5612,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -5759,6 +6183,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -6004,6 +6436,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.0.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.9" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index d2cd52c11..fd0f66142 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -429,6 +429,63 @@ runs: GITHUB_TOKEN: ${{ github.token }} run: just anvil-scheduled-advisories +=== .github/actions/anvil-scheduled-benchmarks/action.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# The token scheduled-benchmarks is substituted by cargo-anvil at emit time with +# the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +name: anvil-scheduled-benchmarks +description: Run the scheduled-benchmarks check group. +inputs: + include_modified: + description: | + Pre-formatted --package args (e.g. "--package alpha@1.0.0 --package + beta@0.2.0") for the modified tier, or the sentinel "--skip" when + nothing modified. Packages are version-qualified cargo specs so they + resolve uniquely even when a like-named crate is also a transitive + dependency. Local invocations leave it unset; recipes default to + --workspace. + default: "" + required: false + include_affected: + description: | + Same shape as include_modified, but for the affected tier + (modified ∪ rev-deps). + default: "" + required: false + include_required: + description: | + Same shape as include_modified, but for the required tier + (affected ∪ workspace-internal transitive deps). + default: "" + required: false + free-disk-space: + description: Remove unused toolchains from GitHub-hosted runners before setup. + default: "false" + required: false +runs: + using: composite + steps: + - uses: ./.github/actions/anvil-setup + with: + group: scheduled-benchmarks + free-disk-space: ${{ inputs.free-disk-space }} + - name: Run just anvil-scheduled-benchmarks + shell: bash + env: + ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} + ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} + ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} + # Some checks (e.g. cargo-aprz, run only by groups that include it) + # hit the GitHub API; pass the built-in token so they use the + # authenticated quota (1000 vs 60 req/hr). Harmless for groups whose + # checks never read it. + GITHUB_TOKEN: ${{ github.token }} + run: just anvil-scheduled-benchmarks + === .github/actions/anvil-scheduled-exhaustive/action.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -1191,6 +1248,108 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/anvil-scheduled-exhaustive + scheduled-benchmarks: + # Benchmark regression detection. x86_64-only, matching + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # Only this job files the regression tracking issue. + issues: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. + fetch-depth: 0 + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: bench-history-${{ matrix.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that + # carries the artifact. Restoring from the latest *successful* + # run would drop every sample collected while the pipeline was + # red from a regression — precisely the window that matters. + # The in-progress run of this very workflow has not uploaded + # yet, so it simply fails the download and the loop moves on. + for run_id in $(gh run list --workflow anvil-scheduled.yml \ + --branch "$DEFAULT_BRANCH" --limit 10 \ + --json databaseId --jq '.[].databaseId'); do + if gh run download "$run_id" --name "$ARTIFACT" \ + --dir target/anvil/bench-history 2>/dev/null; then + echo "restored benchmark history from run $run_id" + exit 0 + fi + done + echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + - uses: ./.github/actions/anvil-scheduled-benchmarks + - name: Save benchmark history + # always(): the run's own samples belong in the history even + # when the analysis flagged a regression and failed the job. + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ matrix.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused + # or infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: File a benchmark regression issue + # Surfacing is by failed build; the issue carries the per-finding + # detail the one-bit build status cannot, and is updated in place + # so a regression appearing while the build is already red still + # reaches the author of its attributed commit. + if: failure() + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + TITLE: Benchmark regressions detected (${{ matrix.os }}) + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + findings=target/anvil/bench/findings-summary.md + if [ ! -f "$findings" ]; then + echo "the job failed before producing findings; nothing to file" + exit 0 + fi + body=$(mktemp) + { + printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" + printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' + cat "$findings" + } > "$body" + number=$(gh issue list --state open --limit 100 --json number,title \ + --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") + if [ -n "$number" ]; then + gh issue edit "$number" --body-file "$body" + echo "updated issue #$number" + else + gh issue create --title "$TITLE" --body-file "$body" + fi + === .github/workflows/anvil-scheduled.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -1212,6 +1371,12 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + # The scheduled-benchmarks job files the regression tracking issue. + # A reusable workflow cannot grant itself more than the caller does, + # so the grant is repeated here and narrowed to that one job inside. + issues: write secrets: inherit === Cargo.toml === @@ -1488,6 +1653,213 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores the +# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs. Depending on a TOML parser here would mean a + # second tool pin for three string fields. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = ($rawLine -split '#', 2)[0].Trim() + if (-not $line) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Stored commits are abbreviated, and a stored blessing names + # either the concrete benchmark it resolved to (once a run exists + # at that commit) or the prefix filter it was issued with. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and + (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + & cargo bench-history bless --local="$store" --context $resolved @key $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4540,6 +4912,35 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4911,6 +5312,7 @@ import 'helpers.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -4948,6 +5350,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'runner.just' import 'tiers.just' import 'tools.just' @@ -5040,7 +5443,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full tier: PR + scheduled, end-to-end. Useful before tagging a release. [group("anvil")] @@ -5081,7 +5485,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -5089,7 +5494,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -5659,6 +6065,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -5904,6 +6318,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.0.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.9" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index 6ff24216d..fe69ac49c 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -296,6 +296,213 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores the +# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs. Depending on a TOML parser here would mean a + # second tool pin for three string fields. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = ($rawLine -split '#', 2)[0].Trim() + if (-not $line) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Stored commits are abbreviated, and a stored blessing names + # either the concrete benchmark it resolved to (once a run exists + # at that commit) or the prefix filter it was issued with. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and + (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + & cargo bench-history bless --local="$store" --context $resolved @key $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -3348,6 +3555,35 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -3719,6 +3955,7 @@ import 'helpers.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -3756,6 +3993,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'runner.just' import 'tiers.just' import 'tools.just' @@ -3848,7 +4086,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full tier: PR + scheduled, end-to-end. Useful before tagging a release. [group("anvil")] @@ -3889,7 +4128,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -3897,7 +4137,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -4467,6 +4708,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -4712,6 +4961,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.0.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.9" cargo_check_external_types_version := "0.5.0" diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just new file mode 100644 index 000000000..b1d7e3c85 --- /dev/null +++ b/justfiles/anvil/checks/bench-history.just @@ -0,0 +1,205 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores the +# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs. Depending on a TOML parser here would mean a + # second tool pin for three string fields. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = ($rawLine -split '#', 2)[0].Trim() + if (-not $line) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Stored commits are abbreviated, and a stored blessing names + # either the concrete benchmark it resolved to (once a run exists + # at that commit) or the prefix filter it was issued with. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and + (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + & cargo bench-history bless --local="$store" --context $resolved @key $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs diff --git a/justfiles/anvil/groups/scheduled-benchmarks.just b/justfiles/anvil/groups/scheduled-benchmarks.just new file mode 100644 index 000000000..4ca0c121a --- /dev/null +++ b/justfiles/anvil/groups/scheduled-benchmarks.just @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# Managed by cargo-anvil. Update the corresponding template in the cargo-anvil +# crate unless the change is repository-specific. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs diff --git a/justfiles/anvil/mod.just b/justfiles/anvil/mod.just index 2a3741105..85afe11a4 100644 --- a/justfiles/anvil/mod.just +++ b/justfiles/anvil/mod.just @@ -39,6 +39,7 @@ import 'helpers.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -76,6 +77,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'runner.just' import 'tiers.just' import 'tools.just' diff --git a/justfiles/anvil/tiers.just b/justfiles/anvil/tiers.just index 05038e0ea..f964a72c7 100644 --- a/justfiles/anvil/tiers.just +++ b/justfiles/anvil/tiers.just @@ -33,7 +33,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full tier: PR + scheduled, end-to-end. Useful before tagging a release. [group("anvil")] @@ -74,7 +75,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -82,7 +84,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] diff --git a/justfiles/anvil/tools.just b/justfiles/anvil/tools.just index 2bfad3528..1a73d8dc2 100644 --- a/justfiles/anvil/tools.just +++ b/justfiles/anvil/tools.just @@ -532,6 +532,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the diff --git a/justfiles/anvil/versions.just b/justfiles/anvil/versions.just index c7a32d8f1..38882231b 100644 --- a/justfiles/anvil/versions.just +++ b/justfiles/anvil/versions.just @@ -52,6 +52,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.0.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.9" cargo_check_external_types_version := "0.5.0" From 2d73e3a432f5b4a7571c1c688d79b41a845efe22 Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Tue, 4 Aug 2026 19:38:20 +0200 Subject: [PATCH 10/24] fix(cargo-anvil): regenerate the README for the new catalog row Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cargo-anvil/README.md b/crates/cargo-anvil/README.md index 7fe8f9d6d..5a8c0fa50 100644 --- a/crates/cargo-anvil/README.md +++ b/crates/cargo-anvil/README.md @@ -428,7 +428,7 @@ And `docs/verification.md` for the continuous-validation strategy. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGkYW0CYXSEGxYc2fK81jTWG7kWg0hlspxYGx-DzHaE-xjXG1cDT7T4wIbxYXKEG-W4Zsmp4-oGGxu6bPxIizmpG08WGD3cTJMxGw_Y1RjIWIdKYWSBg2tjYXJnby1hbnZpbGUwLjMuMGtjYXJnb19hbnZpbA + [__cargo_doc2readme_dependencies_info]: ggGkYW0CYXSEGxYc2fK81jTWG7kWg0hlspxYGx-DzHaE-xjXG1cDT7T4wIbxYXKEG924U18WOtymGxfBoJCeze0eGzTQ9xSQumaFG5sJwoDpJBBrYWSBg2tjYXJnby1hbnZpbGUwLjMuMGtjYXJnb19hbnZpbA [__link0]: https://crates.io/crates/cargo-delta [__link1]: https://crates.io/crates/cargo-spellcheck [__link2]: https://crates.io/crates/cargo-coverage-gate From 2a883f4dc80e2eea73ce96511b4b090d45aee177 Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Fri, 7 Aug 2026 12:38:57 +0200 Subject: [PATCH 11/24] refactor(cargo-anvil): defer benchmark failure reporting to the scheduled publisher PR #65 adds a generic publish-failure job that upserts one incident issue for any failing scheduled group. The benchmark-specific issue step added here would have filed a second notification for the same event, so it is removed: the group now just fails and leaves its findings on the build summary, and reporting happens the same way it does for every other scheduled failure. The job keeps `actions: read` for the history-artifact restore; `issues: write` is no longer needed on either the impl or the root workflow. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 6 +-- .github/workflows/anvil-scheduled-impl.yml | 35 --------------- .github/workflows/anvil-scheduled.yml | 9 ++-- crates/cargo-anvil/docs/design/benchmarks.md | 25 +++++------ crates/cargo-anvil/docs/design/github.md | 18 ++++---- .../docs/implementation-plans/0003.md | 9 ++-- .../cargo-anvil/src/anvil/artifacts/github.rs | 12 ++--- .../github/scheduled-impl-workflow.yml | 35 --------------- .../github/scheduled-root-workflow.yml | 9 ++-- .../snapshots/snapshots__github_backend.snap | 44 ++----------------- 10 files changed, 47 insertions(+), 155 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index 672981697..c88965610 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.3.0" -catalog_checksum = "sha256:5f5d64906e7b2e67f86b7ff5b0b39873f81ecd20a48878d9b5d8b4edc850fe7c" +catalog_checksum = "sha256:76a789ad185047f4e8b6a6e67dbdc5ce7fae9a1165bd13763583245594ddc273" [[file]] path = ".anvil/container/Containerfile" @@ -89,11 +89,11 @@ checksum = "sha256:cb7996cc978eb3f6db6572a78eec1341bfbebb592c90f78f69d622eaacf6d [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:83a6dd5375db294df79cbe01c2f4888e8b036e8475cdf8c9e41326c0b3c3e3f8" +checksum = "sha256:e04d9046a042050a90ea4621acb7b26b6234877a422729a552a2285f932e6822" [[file]] path = ".github/workflows/anvil-scheduled.yml" -checksum = "sha256:e24ebbc326844561e76508450acf74a79133f56b3558f34549c0bf28b29ea9cc" +checksum = "sha256:c6eaa527bb6678b28ceae9f0fd7e8677fca7d49eeefd70ef65d60806e32c8012" [[file]] path = "justfiles/anvil/checks/aprz.just" diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 91b3562a0..faee8be8f 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -128,8 +128,6 @@ jobs: contents: read # Restoring the history reads the Actions runs/artifacts API. actions: read - # Only this job files the regression tracking issue. - issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -183,36 +181,3 @@ jobs: if [ -f target/anvil/bench/findings.md ]; then cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" fi - - name: File a benchmark regression issue - # Surfacing is by failed build; the issue carries the per-finding - # detail the one-bit build status cannot, and is updated in place - # so a regression appearing while the build is already red still - # reaches the author of its attributed commit. - if: failure() - continue-on-error: true - shell: bash - env: - GH_TOKEN: ${{ github.token }} - TITLE: Benchmark regressions detected (${{ matrix.os }}) - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -euo pipefail - findings=target/anvil/bench/findings-summary.md - if [ ! -f "$findings" ]; then - echo "the job failed before producing findings; nothing to file" - exit 0 - fi - body=$(mktemp) - { - printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" - printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' - cat "$findings" - } > "$body" - number=$(gh issue list --state open --limit 100 --json number,title \ - --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") - if [ -n "$number" ]; then - gh issue edit "$number" --body-file "$body" - echo "updated issue #$number" - else - gh issue create --title "$TITLE" --body-file "$body" - fi diff --git a/.github/workflows/anvil-scheduled.yml b/.github/workflows/anvil-scheduled.yml index 750681311..9968c7e01 100644 --- a/.github/workflows/anvil-scheduled.yml +++ b/.github/workflows/anvil-scheduled.yml @@ -18,10 +18,9 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read - # Restoring the history reads the Actions runs/artifacts API. + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. A reusable workflow cannot + # grant itself more than its caller does, so the grant is repeated + # here and narrowed to that one job inside. actions: read - # The scheduled-benchmarks job files the regression tracking issue. - # A reusable workflow cannot grant itself more than the caller does, - # so the grant is repeated here and narrowed to that one job inside. - issues: write secrets: inherit diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index 0ed58047a..a2f334829 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -99,16 +99,15 @@ outside anvil's scope: the artifact rolling window is the supported store. An active regression fails the scheduled build; the findings — each benchmark, its magnitude, and the commit cbh attributes the change-point to — are written to -the build summary and to a findings file the backend wiring consumes. The emitted -findings include cbh's topology-accurate trend chart, so the reviewer's surface is -self-contained — enough to decide *fix or bless* without reproducing the run. +the build summary and to a findings file. The emitted findings include cbh's +topology-accurate trend chart, so the reviewer's surface is self-contained — +enough to decide *fix or bless* without reproducing the run. -- **GitHub Actions** — the failure feeds the repo's create-issue-on-failure path; - the issue is **updated in place** each run from the findings file, so - concurrent regressions and the authors of their attributed commits surface even - while the build is already red. -- **Azure DevOps** — existing failed-build notification subscriptions fire; the - findings live in the build summary. +The failure itself is reported by whatever mechanism the backend already uses for +a failed scheduled build: notification subscriptions on Azure DevOps, the +scheduled-failure issue publisher on GitHub Actions. This subsystem contributes +the *detail* on the build summary rather than a notification channel of its own, so +a regression reaches a human exactly the way every other scheduled failure does. A sustained regression re-fails every run until it is fixed or blessed, so red stays meaningful only under the discipline that the build is always returned to @@ -125,7 +124,7 @@ yet. A developer thus gets the current run's numbers (and a single-machine local trend if they run it repeatedly), not the shared regression signal. Regression detection is therefore a scheduled, shared concern, and the failure -surface (§5) — cbh's finding and trend chart in the issue or build summary — is +surface (§5) — cbh's finding and trend chart on the build summary — is the interface a developer acts on. Reproducing a CI finding locally is not a first-class workflow: it would require downloading that run's `bench-history` artifact into the local store and running cbh's `examine` with the run's machine @@ -160,9 +159,9 @@ that is already in effect. regression and may bundle several changes — an honest range, not always a single culprit. - **The scheduled status is one bit.** It collapses several concurrent - regressions into one red; GitHub recovers per-regression detail through the - updated issue, ADO through the build summary (its native notification is - coarser while already red). + regressions into one red, and a second regression appearing while the build is + already red re-fires no notification. The per-regression detail is recovered + from the build summary, which lists every finding on every run. - **Uncalibrated thresholds.** cbh's gating thresholds are defaults rather than values calibrated to every consumer's data; pinning the tool version contains the resulting risk, and the signal only gates the scheduled build. diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index d9651e5f3..cdb39d5b4 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -774,6 +774,11 @@ Each scheduled benchmark job: Retention is set so the latest artifact outlives the gap to the next scheduled run. +The restore step queries the runs and artifacts APIs, so the job needs +`actions: read`. A reusable workflow cannot grant itself more than its caller, so +the root workflow passes it through and the impl workflow narrows it to the +benchmark job; the PR workflow keeps `contents: read`. + Restoring from the newest run that *carries* the artifact rather than the newest *successful* one is what keeps the chain intact across a regression: a flagged regression fails the job, so a success-only restore would discard every sample @@ -781,14 +786,11 @@ taken while the pipeline stayed red. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits -non-zero on an active regression, failing the job. The scheduled workflow's failure -path creates-or-updates a tracking **issue** from the findings file — updated in -place each run so concurrent regressions and the authors of their attributed commits -surface even while the build is already red. This needs `issues: write` and, for the -restore step's runs/artifacts queries, `actions: read`. A reusable workflow cannot -grant itself more than its caller, so the root workflow passes both through and the -impl workflow narrows them to the benchmark job; the PR workflow keeps -`contents: read`. +non-zero on an active regression, failing the job; the repo's scheduled-failure +issue publisher then reports it like any other scheduled failure. The per-finding +detail — each benchmark, its magnitude, its attributed commit, and cbh's trend +chart — is written to the job summary, so the failed run carries everything a +reviewer needs to decide *fix or bless*. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3), so accepting an intentional change is a reviewed pull request rather than an diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index c5de5de11..805a6cba3 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -57,11 +57,10 @@ Emitted-file snapshots gain the actions/steps and workflow/stage entries. ## Phase 4 — Failure surfacing -- **GitHub** — a failure-path step in the scheduled workflow that creates or - updates the tracking issue in place from the findings file; `issues: write` - scoped to the scheduled job. -- **ADO** — reliance on native failed-build notification subscriptions plus the - build summary; no new emission, documented in the scheduled step. +Write the findings to each backend's build summary and rely on the repo's existing +failed-scheduled-build reporting to notify a human — ADO notification subscriptions, +the GitHub scheduled-failure issue publisher. No notification channel of this +subsystem's own. ## Phase 5 — Bless application diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index 5e1472043..8b62dab31 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -286,13 +286,13 @@ mod tests { // Saving on failure too: the samples collected while the pipeline // is red from a regression are the ones that matter. assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh run download")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh issue create")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh issue edit")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("issues: write")); assert!(SCHEDULED_IMPL_WORKFLOW.contains("GITHUB_STEP_SUMMARY")); - // The reusable workflow cannot grant itself more than the caller, - // so the root workflow must pass the same permission through. - assert!(SCHEDULED_ROOT_WORKFLOW.contains("issues: write")); + // Notifying a human is the generic scheduled-failure publisher's + // job; this group only has to fail and leave its findings behind. + assert!(!SCHEDULED_IMPL_WORKFLOW.contains("gh issue create")); + // Restoring the history reads the runs/artifacts API, and a + // reusable workflow cannot grant itself more than its caller. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions: read")); assert!(SCHEDULED_ROOT_WORKFLOW.contains("actions: read")); } diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index 91b3562a0..faee8be8f 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -128,8 +128,6 @@ jobs: contents: read # Restoring the history reads the Actions runs/artifacts API. actions: read - # Only this job files the regression tracking issue. - issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -183,36 +181,3 @@ jobs: if [ -f target/anvil/bench/findings.md ]; then cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" fi - - name: File a benchmark regression issue - # Surfacing is by failed build; the issue carries the per-finding - # detail the one-bit build status cannot, and is updated in place - # so a regression appearing while the build is already red still - # reaches the author of its attributed commit. - if: failure() - continue-on-error: true - shell: bash - env: - GH_TOKEN: ${{ github.token }} - TITLE: Benchmark regressions detected (${{ matrix.os }}) - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -euo pipefail - findings=target/anvil/bench/findings-summary.md - if [ ! -f "$findings" ]; then - echo "the job failed before producing findings; nothing to file" - exit 0 - fi - body=$(mktemp) - { - printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" - printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' - cat "$findings" - } > "$body" - number=$(gh issue list --state open --limit 100 --json number,title \ - --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") - if [ -n "$number" ]; then - gh issue edit "$number" --body-file "$body" - echo "updated issue #$number" - else - gh issue create --title "$TITLE" --body-file "$body" - fi diff --git a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml index 750681311..9968c7e01 100644 --- a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml @@ -18,10 +18,9 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read - # Restoring the history reads the Actions runs/artifacts API. + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. A reusable workflow cannot + # grant itself more than its caller does, so the grant is repeated + # here and narrowed to that one job inside. actions: read - # The scheduled-benchmarks job files the regression tracking issue. - # A reusable workflow cannot grant itself more than the caller does, - # so the grant is repeated here and narrowed to that one job inside. - issues: write secrets: inherit diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 3d2ccb6b2..606057326 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -2461,8 +2461,6 @@ jobs: contents: read # Restoring the history reads the Actions runs/artifacts API. actions: read - # Only this job files the regression tracking issue. - issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -2516,39 +2514,6 @@ jobs: if [ -f target/anvil/bench/findings.md ]; then cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" fi - - name: File a benchmark regression issue - # Surfacing is by failed build; the issue carries the per-finding - # detail the one-bit build status cannot, and is updated in place - # so a regression appearing while the build is already red still - # reaches the author of its attributed commit. - if: failure() - continue-on-error: true - shell: bash - env: - GH_TOKEN: ${{ github.token }} - TITLE: Benchmark regressions detected (${{ matrix.os }}) - RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - run: | - set -euo pipefail - findings=target/anvil/bench/findings-summary.md - if [ ! -f "$findings" ]; then - echo "the job failed before producing findings; nothing to file" - exit 0 - fi - body=$(mktemp) - { - printf 'Detected by `cargo-bench-history` in [this scheduled run](%s).\n\n' "$RUN_URL" - printf 'Fix the regression, or accept it by adding an entry to `.config/bench-blessings.toml`.\n\n' - cat "$findings" - } > "$body" - number=$(gh issue list --state open --limit 100 --json number,title \ - --jq "[.[] | select(.title == \"$TITLE\")] | .[0].number // empty") - if [ -n "$number" ]; then - gh issue edit "$number" --body-file "$body" - echo "updated issue #$number" - else - gh issue create --title "$TITLE" --body-file "$body" - fi === .github/workflows/anvil-scheduled.yml === # Copyright (c) Microsoft Corporation. @@ -2571,12 +2536,11 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml permissions: contents: read - # Restoring the history reads the Actions runs/artifacts API. + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. A reusable workflow cannot + # grant itself more than its caller does, so the grant is repeated + # here and narrowed to that one job inside. actions: read - # The scheduled-benchmarks job files the regression tracking issue. - # A reusable workflow cannot grant itself more than the caller does, - # so the grant is repeated here and narrowed to that one job inside. - issues: write secrets: inherit === Cargo.toml === From b7322c03d1e6779c37363703f2b33ad41cf9e28c Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Fri, 21 Aug 2026 09:54:56 +0200 Subject: [PATCH 12/24] fix(cargo-anvil): address review of the benchmark regression capability Correctness: - Keep the ADO job-wrapper contract frozen. The `fetchDepth` parameter would have broken every adopter who forked `steps/job.yml`, failing template expansion for the whole scheduled pipeline. The benchmark group now leads its own step list with an explicit checkout, which needs nothing from the wrapper. Publishing moves out of the wrapper's `artifacts` contract for the same reason, since it needs a condition the contract cannot express. - Distinguish an absent history artifact from a failed restore on both backends. Previously any failure -- token, API, corrupt payload -- was read as a cold start, and the run then published an empty store over the accumulated chain while reporting clean. Absence is now positively identified; anything else fails the job, and the publish is guarded on the restore having reached a known state. - Walk back to a run that actually carries the artifact on ADO too; `latestFromBranch` resolves one build and does not walk. - Compare persisted blessing identity exactly. `StartsWith` matched when the stored entry was narrower than the requested one, so a broader committed blessing was skipped while the build stayed red. - Treat `#` as a comment only at line start, so a reason citing an issue number is no longer silently truncated. - Identify the workflow by its runtime name rather than a literal filename, which a rename would have silently reset. - Check out with `lfs: true`; benchmark inputs can be LFS-tracked. Behavior: - Gate only under CI. The recipe reports identically everywhere, but failing a local pre-release `anvil-full` on laptop measurement noise would invite silencing it with a committed blessing. - Plumb the machine-key override through the GitHub wiring, so the documented answer to the top caveat is reachable there and not only on ADO. Verification: - Functional coverage for the recipe: the active-regression gate, inactive and improvement findings, empty history, tool-failure propagation, the blessing reconciliation boundary, and parser rejection. Docs now describe what shipped: anvil generates no failure notifier, restore is not limited to successful runs, and a repo without benchmarks is a supported case rather than one the design assumed away. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 6 +- .../action.yml.anvil-proposed | 55 ---- .github/workflows/anvil-scheduled-impl.yml | 85 +++++- crates/cargo-anvil/docs/design/ado.md | 30 +- crates/cargo-anvil/docs/design/benchmarks.md | 30 +- crates/cargo-anvil/docs/design/github.md | 12 +- .../docs/implementation-plans/0003.md | 28 +- crates/cargo-anvil/src/anvil/artifacts/ado.rs | 55 +++- .../cargo-anvil/src/anvil/artifacts/github.rs | 29 +- .../templates/ado/scheduled-stages.yml | 34 ++- .../ado/steps/bench-history-publish.yml | 31 ++ .../ado/steps/bench-history-restore.yml | 100 ++++-- .../cargo-anvil/templates/ado/steps/job.yml | 10 - .../github/scheduled-impl-workflow.yml | 85 +++++- .../justfiles/anvil/checks/bench-history.just | 61 +++- crates/cargo-anvil/tests/recipe_contracts.rs | 286 +++++++++++++++++- .../snapshots/snapshots__ado_backend.snap | 238 +++++++++++---- .../snapshots/snapshots__github_backend.snap | 146 +++++++-- .../snapshots/snapshots__local_only.snap | 61 +++- justfiles/anvil/checks/bench-history.just | 61 +++- 20 files changed, 1124 insertions(+), 319 deletions(-) delete mode 100644 .github/actions/anvil-scheduled-benchmarks/action.yml.anvil-proposed create mode 100644 crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml diff --git a/.anvil.lock b/.anvil.lock index 677ffea49..9099d8ddc 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.4.0" -catalog_checksum = "sha256:3fa597bf04cc6fc187b4f09c9265a03f239c9dfaa880a5a40f1ceca5780c5edd" +catalog_checksum = "sha256:9cffc53cd58efdf370c607161c95d9e65815667b71be95b9bcb838297386e15b" [[file]] path = ".anvil/container/Containerfile" @@ -89,7 +89,7 @@ checksum = "sha256:cb7996cc978eb3f6db6572a78eec1341bfbebb592c90f78f69d622eaacf6d [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:843de9eee526bacb9c07ec873bf7083b183b1a875a65cadec3ecf9b0e92644df" +checksum = "sha256:084dbdf183797db31c21660485524344e1805d27d2f3a8b504c2ea8c296369fe" [[file]] path = ".github/workflows/anvil-scheduled.yml" @@ -105,7 +105,7 @@ checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536 [[file]] path = "justfiles/anvil/checks/bench-history.just" -checksum = "sha256:b01da5abc438c46f5c0ef2a7a9eaef3280d544f048d7eb418bab0e4341ce4b7c" +checksum = "sha256:222f3e22c442e8a4dcb155b4cd29052089afa45b823c30b5a88c7d4535ba7c9a" [[file]] path = "justfiles/anvil/checks/bench.just" diff --git a/.github/actions/anvil-scheduled-benchmarks/action.yml.anvil-proposed b/.github/actions/anvil-scheduled-benchmarks/action.yml.anvil-proposed deleted file mode 100644 index 01d76fdfc..000000000 --- a/.github/actions/anvil-scheduled-benchmarks/action.yml.anvil-proposed +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md -# The token scheduled-benchmarks is substituted by cargo-anvil at emit time with -# the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). -name: anvil-scheduled-benchmarks -description: Run the scheduled-benchmarks check group. -inputs: - include_modified: - description: | - Pre-formatted --package args (e.g. "--package alpha@1.0.0 --package - beta@0.2.0") for the modified tier, or the sentinel "--skip" when - nothing modified. Packages are version-qualified cargo specs so they - resolve uniquely even when a like-named crate is also a transitive - dependency. Local invocations leave it unset; recipes default to - --workspace. - default: "" - required: false - include_affected: - description: | - Same shape as include_modified, but for the affected tier - (modified ∪ rev-deps). - default: "" - required: false - include_required: - description: | - Same shape as include_modified, but for the required tier - (affected ∪ workspace-internal transitive deps). - default: "" - required: false - free-disk-space: - description: Remove unused toolchains from GitHub-hosted runners before setup. - default: "false" - required: false -runs: - using: composite - steps: - - uses: ./.github/actions/anvil-setup - with: - group: scheduled-benchmarks - free-disk-space: ${{ inputs.free-disk-space }} - - name: Run just anvil-scheduled-benchmarks - shell: bash - env: - ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} - ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} - ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} - # Some checks (e.g. cargo-aprz, run only by groups that include it) - # hit the GitHub API; pass the built-in token so they use the - # authenticated quota (1000 vs 60 req/hr). Harmless for groups whose - # checks never read it. - GITHUB_TOKEN: ${{ github.token }} - run: just anvil-scheduled-benchmarks diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 1b5b099d1..d643e0380 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -24,6 +24,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -141,38 +149,83 @@ jobs: with: # The analysis orders each series by first-parent commit # topology and locates the merge-base, so it needs the whole - # commit graph. + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. fetch-depth: 0 + lfs: true - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} ARTIFACT: bench-history-${{ matrix.os }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename + # must not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" run: | set -euo pipefail mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that - # carries the artifact. Restoring from the latest *successful* - # run would drop every sample collected while the pipeline was - # red from a regression — precisely the window that matters. - # The in-progress run of this very workflow has not uploaded - # yet, so it simply fails the download and the loop moves on. - for run_id in $(gh run list --workflow anvil-scheduled.yml \ - --branch "$DEFAULT_BRANCH" --limit 10 \ + # carries this leg's artifact. Restoring from the latest + # *successful* run would drop every sample collected while the + # pipeline was red from a regression — precisely the window + # that matters. + # + # Absence and failure are kept distinct. A run is only a + # candidate once the artifacts API confirms the artifact exists + # and has not expired; a download that then fails is an + # operational error (token, API, corrupt payload) and fails the + # job rather than being silently downgraded to a cold start. + # That distinction is what stops one transient failure from + # publishing an empty store over a good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do - if gh run download "$run_id" --name "$ARTIFACT" \ - --dir target/anvil/bench-history 2>/dev/null; then - echo "restored benchmark history from run $run_id" - exit 0 + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 done - echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + + # No run in the window carried the artifact. That is a genuine + # cold start (first run, or the chain lapsed), so it is surfaced + # on the summary rather than only in this log — "history quietly + # restarted" must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks + env: + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - name: Save benchmark history - # always(): the run's own samples belong in the history even - # when the analysis flagged a regression and failed the job. - if: always() + # always(): the run's own samples belong in the history even when + # the analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the + # restore failed operationally the store is not a continuation of + # the chain, and publishing it would overwrite good history with a + # truncated snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bench-history-${{ matrix.os }} diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 99ccfc687..7531da874 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -211,9 +211,9 @@ Note the ADO topology differs from GitHub Actions in two places: ├── impact.yml owned (cargo-delta impact step) ├── job.yml owned-but-user-customizable │ (per-job wrapper; takes `name`, - │ `pool`, `steps`, `fetchDepth`, - │ `artifacts`; users edit to inject - │ 1ESPT `templateContext:` etc.) + │ `pool`, `steps`, `artifacts`; + │ users edit to inject 1ESPT + │ `templateContext:` etc.) ├── bench-history-restore.yml owned (restore the benchmark history artifact) ├── bench-history-summary.yml owned (attach benchmark findings to the build summary) ├── pr-fast.yml owned (one step template per group) @@ -235,8 +235,8 @@ customized by adopters whose ADO instance requires extension templates (1ES PT, SubstratePT, M365PT). Once a user edits it, the standard dirty-file flow kicks in — subsequent anvil updates Propose into a `.proposed` sibling rather than overwriting. The stages templates address the wrapper only via its -parameter contract (`name`, `pool`, `steps`, `fetchDepth`, `artifacts`), so the -wrapper can diverge arbitrarily without blocking stage-shape updates. See §4.1. +parameter contract (`name`, `pool`, `steps`, `artifacts`), so the wrapper can +diverge arbitrarily without blocking stage-shape updates. See §4.1. ## 3. Root pipelines @@ -379,9 +379,14 @@ The contract is intentionally small and stable: | `name` | `string` | yes | Job name; ADO derives the display name from it. | | `pool` | `object` | yes | Pool block, passed verbatim to ADO's `pool:` key. `linuxPool` and `windowsPool` at the stage level are object parameters, so users can override their shape (e.g. `{ name, os, image }` for 1ESPT). | | `steps` | `stepList` | yes | Body of the job. Templated step lists are fine — the wrapper splices them in via `${{ each step in parameters.steps }}: - ${{ step }}`. | -| `fetchDepth` | `string` | no | When set, the job checks out explicitly at this depth (`'0'` for full history) instead of taking the implicit default checkout. 1ESPT wrappers map it onto `templateContext.inputs` instead. | | `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. The stages templates don't need to know which backend they're targeting. | +A job that needs a non-default checkout (depth, LFS) puts an explicit `checkout` +step at the head of its own `steps` list rather than growing this contract. The +contract stays frozen because a forked wrapper cannot be expected to declare a +parameter added after the fork, and a stages template binding one would fail +expansion for the entire pipeline. + The default wrapper anvil ships is six lines of logic: ```yaml @@ -389,15 +394,11 @@ parameters: - { name: name, type: string } - { name: pool, type: object } - { name: steps, type: stepList } - - { name: fetchDepth, type: string, default: '' } - { name: artifacts, type: object, default: [] } jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: - - ${{ if ne(parameters.fetchDepth, '') }}: - - checkout: self - fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: @@ -416,10 +417,6 @@ jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} templateContext: - inputs: - - input: checkout - repository: self - fetchDepth: ${{ parameters.fetchDepth }} outputs: - ${{ each artifact in parameters.artifacts }}: - output: pipelineArtifact @@ -839,8 +836,9 @@ own artifact (`bench-history-`). Each scheduled benchmark job: -1. checks out with full history (the §4.1 wrapper's `fetchDepth` parameter; analysis - reads the commit graph); +1. checks out with full history and LFS via an explicit `checkout` step at the head + of its own step list (analysis reads the commit graph; benchmark inputs may be + LFS-tracked); 2. **restores** the history with `DownloadPipelineArtifact@2` (`buildVersionToDownload: latestFromBranch`, the default branch); the first run finds none and starts empty; diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index a2f334829..417dcdea8 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -21,8 +21,12 @@ hard part: results must be ordered by how the code evolved (not by when a benchmark happened to run), compared only against like hardware (CI runs on a heterogeneous, rotating pool whose machine-to-machine variance dwarfs the measurement), and judged with noise-aware statistics (a fixed percentage -threshold on that noise fires constantly). Every anvil repo has benchmarks and -none of them gets this today — the existing `bench` check only compiles them. +threshold on that noise fires constantly). A repo with benchmarks gets none of +this today — the existing `bench` check only compiles them. + +A repo without benchmarks is a first-class case, not an oversight: the recipe +runs, records nothing, and analyzes an empty history to a clean no-op, so the +capability costs such a repo a green job and nothing else. ## 2. Design principles @@ -103,11 +107,13 @@ the build summary and to a findings file. The emitted findings include cbh's topology-accurate trend chart, so the reviewer's surface is self-contained — enough to decide *fix or bless* without reproducing the run. -The failure itself is reported by whatever mechanism the backend already uses for -a failed scheduled build: notification subscriptions on Azure DevOps, the -scheduled-failure issue publisher on GitHub Actions. This subsystem contributes -the *detail* on the build summary rather than a notification channel of its own, so -a regression reaches a human exactly the way every other scheduled failure does. +The failure itself is reported by whatever mechanism the repo already has for a +failed scheduled build: notification subscriptions on Azure DevOps, GitHub's own +scheduled-run failure notifications (or an adopter-supplied publisher) on GitHub +Actions. anvil generates no notifier of its own for this. The subsystem +contributes the *detail* on the build summary rather than a notification channel, +so a regression reaches a human exactly the way every other scheduled failure +does. A sustained regression re-fails every run until it is fixed or blessed, so red stays meaningful only under the discipline that the build is always returned to @@ -123,6 +129,13 @@ is empty, so analysis is a clean no-op reporting that there is no local history yet. A developer thus gets the current run's numbers (and a single-machine local trend if they run it repeatedly), not the shared regression signal. +For the same reason the *gate* is CI-only by default. The recipe reports +identically in both places, but only exits non-zero under CI (or with an explicit +local opt-in): a laptop's measurement noise is not the shared trend, and failing +a pre-release `anvil-full` on thermal throttling would invite silencing it with a +committed blessing — polluting a reviewed, audited file with one machine's +artifacts. + Regression detection is therefore a scheduled, shared concern, and the failure surface (§5) — cbh's finding and trend chart on the build summary — is the interface a developer acts on. Reproducing a CI finding locally is not a @@ -153,7 +166,8 @@ that is already in effect. partitions too sparse to analyze. Whether a hosted pool stays dense enough depends on its hardware homogeneity; self-hosted or dedicated runners avoid the concern. An adopter who knows their pool is uniform enough can substitute a - stable pool label for the fingerprint, trading partition fidelity for density. + stable pool label for the fingerprint, trading partition fidelity for density; + both backends expose that as an input on their scheduled wiring. - **Attribution is coarse under sparse benchmarking.** Benches do not run on every commit, so the attributed commit is the first *benchmarked* one after a regression and may bundle several changes — an honest range, not always a diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index cdb39d5b4..bfeb398af 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -786,11 +786,13 @@ taken while the pipeline stayed red. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits -non-zero on an active regression, failing the job; the repo's scheduled-failure -issue publisher then reports it like any other scheduled failure. The per-finding -detail — each benchmark, its magnitude, its attributed commit, and cbh's trend -chart — is written to the job summary, so the failed run carries everything a -reviewer needs to decide *fix or bless*. +non-zero on an active regression, failing the job. anvil does not generate a +notifier for this: what reaches a human is whatever failure reporting the repo +already has — GitHub's own scheduled-run failure notifications by default, or an +adopter-supplied publisher. The per-finding detail — each benchmark, its +magnitude, its attributed commit, and cbh's trend chart — is written to the job +summary, so the failed run carries everything a reviewer needs to decide +*fix or bless*. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3), so accepting an intentional change is a reviewed pull request rather than an diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md index 805a6cba3..7aa2ffc91 100644 --- a/crates/cargo-anvil/docs/implementation-plans/0003.md +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -44,23 +44,29 @@ snapshots gain the group and check. Emit the `scheduled-benchmarks` job (GitHub reusable scheduled workflow) and stage (ADO scheduled stages), each running `anvil-setup` then -`just anvil-scheduled-benchmarks`, and add the artifact round-trip: - -- **GitHub** — `fetch-depth: 0` checkout; a step resolving the latest successful - `anvil-scheduled` run on the default branch and downloading its `bench-history` - artifact; `actions/upload-artifact` with retention at the end. -- **ADO** — full-history checkout via the §4.1 wrapper `checkout` input; - `DownloadPipelineArtifact@2` (`buildVersionToDownload: latestFromBranch`); - publish through the wrapper's `artifacts` contract. +`just anvil-scheduled-benchmarks`, and add the artifact round-trip. Each leg +carries its own artifact, and both backends walk back from the newest run to the +first that actually carries it — restoring only from *successful* runs would +discard the samples taken while the pipeline was red, which is the stretch that +matters. A restore that fails operationally must fail the job rather than +publishing a truncated store over the chain. + +- **GitHub** — `fetch-depth: 0` + `lfs: true` checkout; a step resolving the run + via the runs/artifacts API and downloading with `gh run download`; + `actions/upload-artifact` with retention, guarded on the restore outcome. +- **ADO** — full-history + LFS checkout as an explicit `checkout` step in the + group's own step list (the §4.1 wrapper contract stays frozen, since adopters + fork that file); a restore step resolving the build through the build/artifacts + REST API; a publish step carrying the same guard. Emitted-file snapshots gain the actions/steps and workflow/stage entries. ## Phase 4 — Failure surfacing Write the findings to each backend's build summary and rely on the repo's existing -failed-scheduled-build reporting to notify a human — ADO notification subscriptions, -the GitHub scheduled-failure issue publisher. No notification channel of this -subsystem's own. +failed-scheduled-build reporting to notify a human — ADO notification +subscriptions, GitHub's own scheduled-run failure notifications or an +adopter-supplied publisher. anvil generates no notifier of its own. ## Phase 5 — Bless application diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index 41b35327c..bff0e4069 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -25,6 +25,9 @@ const JOB_WRAPPER: &str = include_str!("../../../templates/ado/steps/job.yml"); /// Embedded body of the benchmark-history restore step template. const BENCH_HISTORY_RESTORE_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-restore.yml"); +/// Embedded body of the benchmark-history publish step template. +const BENCH_HISTORY_PUBLISH_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-publish.yml"); + /// Embedded body of the benchmark-findings build-summary step template. const BENCH_HISTORY_SUMMARY_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-summary.yml"); @@ -129,6 +132,17 @@ pub fn bench_history_summary() -> Artifact { ) } +/// `.pipelines/anvil/steps/bench-history-publish.yml` — publishes the +/// updated benchmark history as this leg's artifact. +#[must_use] +pub fn bench_history_publish() -> Artifact { + Artifact::backend_file( + Backend::Ado, + ".pipelines/anvil/steps/bench-history-publish.yml", + BENCH_HISTORY_PUBLISH_STEP, + ) +} + /// `.pipelines/anvil/pr.yml` — the PR-tier stages template. #[must_use] pub fn pr_stages() -> Artifact { @@ -206,6 +220,7 @@ pub(crate) fn all() -> Vec { job_wrapper(), bench_history_restore(), bench_history_summary(), + bench_history_publish(), ]; for (group, path) in GROUP_STEPS { out.push(Artifact::backend_file(Backend::Ado, path, render_group_step(group))); @@ -295,7 +310,6 @@ mod tests { "name: steps", "type: stepList", "name: artifacts", - "name: fetchDepth", "PublishPipelineArtifact@1", ] { assert!(JOB_WRAPPER.contains(needle), "wrapper missing '{needle}'"); @@ -395,12 +409,22 @@ mod tests { #[test] fn scheduled_benchmarks_stage_round_trips_the_history_artifact() { - // Analysis walks the commit graph, so both legs check out fully. + // The checkout leads the group's own step list rather than going + // through a wrapper parameter: job.yml is the file adopters fork, + // so binding a parameter their copy lacks would fail expansion for + // the whole pipeline. + assert!( + !JOB_WRAPPER.contains("fetchDepth"), + "the job wrapper contract must stay frozen; put checkout in the group's step list" + ); assert_eq!( - SCHEDULED_STAGES.matches("fetchDepth: '0'").count(), + SCHEDULED_STAGES.matches("- checkout: self").count(), 2, - "both benchmark legs must check out the full history" + "both benchmark legs check out explicitly" ); + assert_eq!(SCHEDULED_STAGES.matches("fetchDepth: 0").count(), 2); + // Benchmark inputs can be LFS-tracked. + assert_eq!(SCHEDULED_STAGES.matches("lfs: true").count(), 2); // Per-leg artifact names: the history is partitioned per machine. for needle in ["bench-history-linux", "bench-history-windows"] { assert_eq!( @@ -411,14 +435,23 @@ mod tests { } assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-restore.yml")); assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-summary.yml")); - // Take the newest run carrying the artifact whatever its outcome: - // restoring only from green runs would drop every sample collected + assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-publish.yml")); + // Take the newest build carrying the artifact whatever its outcome: + // restoring only from green builds would drop every sample collected // while the pipeline was red from a regression. - assert!(BENCH_HISTORY_RESTORE_STEP.contains("buildVersionToDownload: latestFromBranch")); - assert!(BENCH_HISTORY_RESTORE_STEP.contains("allowFailedBuilds: true")); - assert!(BENCH_HISTORY_RESTORE_STEP.contains("allowPartiallySucceededBuilds: true")); - // A missing artifact is a cold start, not a failure. - assert!(BENCH_HISTORY_RESTORE_STEP.contains("continueOnError: true")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("queryOrder=finishTimeDescending")); + // Absence and operational failure must stay distinguishable, or one + // transient error publishes an empty store over a good history. + assert!(BENCH_HISTORY_RESTORE_STEP.contains("if ($status -eq 404) { continue }")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("ANVIL_BENCH_RESTORE]restored")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("ANVIL_BENCH_RESTORE]cold-start")); + assert!( + !BENCH_HISTORY_RESTORE_STEP.contains("continueOnError"), + "a blanket continueOnError would read every failure as a cold start" + ); + // The publish is guarded on the restore having reached a known state. + assert!(BENCH_HISTORY_PUBLISH_STEP.contains("ne(variables['ANVIL_BENCH_RESTORE'], '')")); + assert!(BENCH_HISTORY_PUBLISH_STEP.contains("succeededOrFailed()")); assert!(BENCH_HISTORY_SUMMARY_STEP.contains("##vso[task.uploadsummary]")); assert!(BENCH_HISTORY_SUMMARY_STEP.contains("condition: succeededOrFailed()")); } diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index 8b62dab31..1cd27539f 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -273,7 +273,8 @@ mod tests { #[test] fn scheduled_benchmarks_job_round_trips_the_history_artifact() { - // Analysis walks the commit graph, so the leg needs full history. + // Analysis walks the commit graph, so the leg needs full history; + // benchmark inputs can be LFS-tracked. assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); // Per-leg artifact names: the history is partitioned per machine, // and upload-artifact rejects a name reused within one run. @@ -283,12 +284,30 @@ mod tests { "the restore and save steps must agree on the per-leg artifact name" ); assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions/upload-artifact@")); - // Saving on failure too: the samples collected while the pipeline - // is red from a regression are the ones that matter. assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh run download")); assert!(SCHEDULED_IMPL_WORKFLOW.contains("GITHUB_STEP_SUMMARY")); - // Notifying a human is the generic scheduled-failure publisher's - // job; this group only has to fail and leave its findings behind. + // The workflow is identified by its runtime name, not a literal + // filename: the root workflow is owned and renameable, and a rename + // must not silently reset the series. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("WORKFLOW: ${{ github.workflow }}")); + assert!( + !SCHEDULED_IMPL_WORKFLOW.contains("--workflow anvil-scheduled.yml"), + "a hardcoded workflow filename breaks on rename" + ); + // Absence and operational failure must stay distinguishable, and the + // upload is guarded on the restore having reached a known state -- + // otherwise one transient failure publishes an empty store over the + // accumulated chain and reports clean. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_RESTORE=restored")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_RESTORE=cold-start")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("if: always() && env.ANVIL_BENCH_RESTORE != ''")); + // The machine-key escape hatch has to be reachable in CI, which + // workflow-level env is not across a called reusable workflow. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_machine_key:")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }}")); + // Notifying a human is the repo's existing scheduled-failure + // reporting; this group only has to fail and leave findings behind. assert!(!SCHEDULED_IMPL_WORKFLOW.contains("gh issue create")); // Restoring the history reads the runs/artifacts API, and a // reusable workflow cannot grant itself more than its caller. diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index 56feb13e0..50087d5f6 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -118,34 +118,46 @@ stages: # publishes the updated store at the end of the job -- including # when the analysis failed the job, so the samples collected while # the pipeline is red are not lost. + # + # The explicit `checkout` leads the step list rather than going + # through a wrapper parameter: `steps/job.yml` is the file adopters + # fork for 1ESPT and friends, so binding a parameter their wrapper + # does not declare would fail template expansion for the whole + # pipeline. An explicit checkout step needs nothing from the wrapper + # and replaces the implicit default checkout. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. - fetchDepth: '0' - artifacts: - - name: bench-history-linux - path: target/anvil/bench-history steps: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. + - checkout: self + fetchDepth: 0 + lfs: true - template: steps/bench-history-restore.yml parameters: artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml + - template: steps/bench-history-publish.yml + parameters: + artifact: bench-history-linux - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} - fetchDepth: '0' - artifacts: - - name: bench-history-windows - path: target/anvil/bench-history steps: + - checkout: self + fetchDepth: 0 + lfs: true - template: steps/bench-history-restore.yml parameters: artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml + - template: steps/bench-history-publish.yml + parameters: + artifact: bench-history-windows diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml new file mode 100644 index 000000000..3cfe71e04 --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Publishes the updated benchmark history as this leg's artifact. +# +# This does not go through the §4.1 job wrapper's `artifacts` contract +# because it needs a condition the contract does not express: the store is +# only a valid continuation of the chain when the restore reached a known +# state. Publishing after an operational restore failure would overwrite a +# good history with a truncated snapshot. +# +# The condition still includes failed runs: a flagged regression fails the +# job, and those samples belong in the history. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history +steps: + - task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.artifact }} + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) + inputs: + targetPath: ${{ parameters.path }} + artifact: ${{ parameters.artifact }} diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml index 39d7dd839..971e279a3 100644 --- a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -4,7 +4,17 @@ # Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # -# Restores the benchmark history the previous scheduled run published. +# Restores the benchmark history published by the most recent build that +# carries this leg's artifact. +# +# `DownloadPipelineArtifact@2` with `latestFromBranch` resolves a single +# build and yields nothing if that build has no such artifact -- it does +# not walk back. A cancelled or never-publishing latest build would +# therefore cold-start a store that actually has history. This step +# resolves the build itself, so absence of the artifact throughout the +# window (a genuine cold start) stays distinct from an operational +# failure, which fails the job rather than silently publishing a +# truncated snapshot over a good chain. # # See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md parameters: @@ -13,27 +23,71 @@ parameters: - name: path type: string default: target/anvil/bench-history + - name: window + type: number + default: 30 steps: - # The store has to exist even when there is nothing to restore, so the - # first ever run analyzes an empty history instead of erroring. - - pwsh: New-Item -ItemType Directory -Force -Path '${{ parameters.path }}' | Out-Null - displayName: Prepare benchmark history store - - task: DownloadPipelineArtifact@2 + - pwsh: | + $ErrorActionPreference = 'Stop' + $artifact = '${{ parameters.artifact }}' + $path = '${{ parameters.path }}' + $window = ${{ parameters.window }} + + New-Item -ItemType Directory -Force -Path $path | Out-Null + + if (-not $env:SYSTEM_ACCESSTOKEN) { + Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." + exit 1 + } + + $collection = $env:SYSTEM_COLLECTIONURI + $project = $env:SYSTEM_TEAMPROJECTID + $definition = $env:SYSTEM_DEFINITIONID + $branch = $env:BUILD_SOURCEBRANCH + $headers = @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + + # Newest first, whatever the outcome: a flagged regression fails the + # build, so restricting to successful builds would discard exactly + # the stretch of history that matters most. + $runsUri = "$collection$project/_apis/build/builds?definitions=$definition&branchName=$branch&`$top=$window&queryOrder=finishTimeDescending&api-version=7.0" + $runs = (Invoke-RestMethod -Uri $runsUri -Headers $headers).value + + foreach ($run in $runs) { + if ($run.id -eq $env:BUILD_BUILDID) { continue } + + $artifactUri = "$collection$project/_apis/build/builds/$($run.id)/artifacts?artifactName=$artifact&api-version=7.0" + try { + $found = Invoke-RestMethod -Uri $artifactUri -Headers $headers + } catch { + # 404 is "this build has no such artifact" -- the expected, + # common case while walking back. Anything else is an + # operational failure and must not be read as absence. + $status = $_.Exception.Response.StatusCode.value__ + if ($status -eq 404) { continue } + Write-Error "anvil: querying artifacts of build $($run.id) failed with HTTP $status; refusing to continue with an empty history." + exit 1 + } + if (-not $found.resource.downloadUrl) { continue } + + Write-Host "anvil: restoring $artifact from build $($run.id)" + $tmp = [System.IO.Path]::GetTempPath() + $zip = Join-Path $tmp "$artifact.zip" + $staging = Join-Path $tmp "$artifact-extract" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip + Expand-Archive -LiteralPath $zip -DestinationPath $staging -Force + # The archive nests its contents under a directory named for the + # artifact; lift them up into the store path. + $inner = Join-Path $staging $artifact + $source = if (Test-Path $inner) { $inner } else { $staging } + Copy-Item -Path (Join-Path $source '*') -Destination $path -Recurse -Force + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]restored" + exit 0 + } + + # Nothing in the window carried the artifact: a genuine cold start. + Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]cold-start" displayName: Restore ${{ parameters.artifact }} - # The first run has no artifact to restore; a missing artifact is a - # cold start, not a failure. - continueOnError: true - inputs: - buildType: specific - project: $(System.TeamProjectId) - definition: $(System.DefinitionId) - buildVersionToDownload: latestFromBranch - branchName: $(Build.SourceBranch) - # Take the newest run that carries the artifact whatever its - # outcome. Restoring only from green runs would drop every sample - # collected while the pipeline was red from a regression -- - # precisely the window that matters. - allowPartiallySucceededBuilds: true - allowFailedBuilds: true - artifactName: ${{ parameters.artifact }} - targetPath: ${{ parameters.path }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/crates/cargo-anvil/templates/ado/steps/job.yml b/crates/cargo-anvil/templates/ado/steps/job.yml index e7275899d..028b1ccd2 100644 --- a/crates/cargo-anvil/templates/ado/steps/job.yml +++ b/crates/cargo-anvil/templates/ado/steps/job.yml @@ -17,10 +17,6 @@ # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. -# - fetchDepth (string) Optional. When set, the job checks out explicitly at -# this depth ('0' for full history) instead of taking -# the implicit default checkout. 1ESPT wrappers map it -# onto templateContext.inputs instead. # - artifacts (object) Optional list of pipeline artifacts to publish. # Each item: { name: string, path: string }. # The default wrapper appends one @@ -37,9 +33,6 @@ parameters: type: object - name: steps type: stepList - - name: fetchDepth - type: string - default: '' - name: artifacts type: object default: [] @@ -48,9 +41,6 @@ jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: - - ${{ if ne(parameters.fetchDepth, '') }}: - - checkout: self - fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index 1b5b099d1..d643e0380 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -24,6 +24,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -141,38 +149,83 @@ jobs: with: # The analysis orders each series by first-parent commit # topology and locates the merge-base, so it needs the whole - # commit graph. + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. fetch-depth: 0 + lfs: true - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} ARTIFACT: bench-history-${{ matrix.os }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename + # must not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" run: | set -euo pipefail mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that - # carries the artifact. Restoring from the latest *successful* - # run would drop every sample collected while the pipeline was - # red from a regression — precisely the window that matters. - # The in-progress run of this very workflow has not uploaded - # yet, so it simply fails the download and the loop moves on. - for run_id in $(gh run list --workflow anvil-scheduled.yml \ - --branch "$DEFAULT_BRANCH" --limit 10 \ + # carries this leg's artifact. Restoring from the latest + # *successful* run would drop every sample collected while the + # pipeline was red from a regression — precisely the window + # that matters. + # + # Absence and failure are kept distinct. A run is only a + # candidate once the artifacts API confirms the artifact exists + # and has not expired; a download that then fails is an + # operational error (token, API, corrupt payload) and fails the + # job rather than being silently downgraded to a cold start. + # That distinction is what stops one transient failure from + # publishing an empty store over a good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do - if gh run download "$run_id" --name "$ARTIFACT" \ - --dir target/anvil/bench-history 2>/dev/null; then - echo "restored benchmark history from run $run_id" - exit 0 + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 done - echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + + # No run in the window carried the artifact. That is a genuine + # cold start (first run, or the chain lapsed), so it is surfaced + # on the summary rather than only in this log — "history quietly + # restarted" must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks + env: + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - name: Save benchmark history - # always(): the run's own samples belong in the history even - # when the analysis flagged a regression and failed the job. - if: always() + # always(): the run's own samples belong in the history even when + # the analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the + # restore failed operationally the store is not a continuation of + # the chain, and publishing it would overwrite good history with a + # truncated snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bench-history-${{ matrix.os }} diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just index 7b5631370..286b2760f 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -13,13 +13,30 @@ # ANVIL_INCLUDE_* contract and always measures the whole workspace. # # Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. # # The store is the cross-run state the cloud wiring restores before and # publishes after this recipe; locally it is whatever has accumulated # under target/, which on a fresh checkout is empty and analyzes to a # clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. # Run the benchmarks and analyze the accumulated history for regressions. [script("pwsh", "-NoProfile")] @@ -83,6 +100,18 @@ anvil-bench-history: anvil-bench-history-validate-prereqs } Write-Host '' Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." exit 1 @@ -113,13 +142,19 @@ _anvil-bench-history-bless store blessings: } # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs. Depending on a TOML parser here would mean a - # second tool pin for three string fields. + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. $entries = New-Object System.Collections.Generic.List[object] $current = $null foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = ($rawLine -split '#', 2)[0].Trim() - if (-not $line) { continue } + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } if ($line -eq '[[blessing]]') { $current = @{} $entries.Add($current) | Out-Null @@ -129,7 +164,7 @@ _anvil-bench-history-bless store blessings: Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" exit 1 } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { if ($null -eq $current) { Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" exit 1 @@ -137,7 +172,7 @@ _anvil-bench-history-bless store blessings: $current[$Matches[1]] = $Matches[2] continue } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) exit 1 } @@ -180,12 +215,14 @@ _anvil-bench-history-bless store blessings: exit 1 } $resolved = $resolved.Trim() - # Stored commits are abbreviated, and a stored blessing names - # either the concrete benchmark it resolved to (once a run exists - # at that commit) or the prefix filter it was issued with. + # Compare the persisted identity exactly. A `StartsWith` in either + # direction is wrong: a stored concrete `foo/bar` would satisfy a + # prefix test for the broader `foo`, so a committed blessing of + # `foo` would be skipped while still leaving the build red -- with + # a log line claiming it was already in effect. $already = $applied | Where-Object { $resolved.StartsWith($_.commit) -and - (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + (($_.prefixes -contains $benchmark) -or ($_.benchmark -eq $benchmark)) } if ($already) { Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index a30e4a0bb..cc0ed8ac5 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -22,6 +22,25 @@ const LLVM_COV: &str = include_str!("../templates/justfiles/anvil/checks/llvm-co const SEMVER: &str = include_str!("../templates/justfiles/anvil/checks/semver-check.just"); const EXTERNAL_TYPES: &str = include_str!("../templates/justfiles/anvil/checks/external-types.just"); const VERSIONS: &str = include_str!("../templates/justfiles/anvil/versions.just"); +const BENCH_HISTORY: &str = include_str!("../templates/justfiles/anvil/checks/bench-history.just"); + +/// A findings report with one active regression — the state that must gate. +const ACTIVE_REGRESSION: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"regression", + "active":true,"relative_delta":0.4966,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// The same finding after it recovered: reported, but nothing to act on. +const INACTIVE_REGRESSION: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"regression", + "active":false,"relative_delta":0.4966,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// An improvement — never a reason to fail. +const IMPROVEMENT: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"improvement", + "active":true,"relative_delta":-0.31,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// What an empty workspace analyzes to: no runs, no findings. +const NO_FINDINGS: &str = r#"{"notable":false,"findings":[]}"#; fn write(path: &Path, contents: &str) { if let Some(parent) = path.parent() { @@ -51,12 +70,13 @@ fn fixture(imports: &[(&str, &str)], dependency_recipes: &[&str]) -> TempDir { &tmp.path().join("Cargo.toml"), "[package]\nname = \"fixture\"\nversion = \"0.1.0\"\n", ); + write_fake_bin(tmp.path()); + tmp +} - let bin = tmp.path().join("fake-bin"); - std::fs::create_dir_all(&bin).unwrap(); - write( - &bin.join("cargo.ps1"), - r#" +/// The stand-in `cargo` the recipes invoke. A scenario drives every +/// branch through `FAKE_*` environment variables. +const FAKE_CARGO: &str = r#" $joined = $args -join ' ' if ($env:FAKE_CARGO_LOG) { Add-Content -LiteralPath $env:FAKE_CARGO_LOG -Value $joined @@ -123,11 +143,57 @@ if ($args -contains 'nextest') { } exit [int]$env:FAKE_NEXTEST_EXIT } +if ($args -contains 'bench-history') { + # $args inside a function refers to that function's own arguments, so + # the script's are captured here and passed in explicitly. + $cbhArgs = $args + # Writes whatever report the scenario asked for to the path the recipe + # passed, so the recipe's own parsing and gating are what get exercised. + function Write-Report([string[]]$all, [string]$flag, [string]$content) { + $index = [array]::IndexOf($all, $flag) + if ($index -ge 0 -and $content) { + $target = $all[$index + 1] + $parent = Split-Path -Parent $target + if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } + Set-Content -LiteralPath $target -Value $content -Encoding UTF8 + } + } + if ($cbhArgs -contains 'collect') { exit [int]$env:FAKE_CBH_COLLECT_EXIT } + if ($cbhArgs -contains 'analyze') { + Write-Report $cbhArgs '--json' $env:FAKE_CBH_FINDINGS + Write-Report $cbhArgs '--markdown' 'findings' + Write-Report $cbhArgs '--markdown-summary' 'summary' + exit [int]$env:FAKE_CBH_ANALYZE_EXIT + } + if ($cbhArgs -contains 'list') { + Write-Report $cbhArgs '--json' $env:FAKE_CBH_BLESSINGS + exit 0 + } + if ($cbhArgs -contains 'bless') { exit [int]$env:FAKE_CBH_BLESS_EXIT } + exit 0 +} exit 0 -"#, - ); - write(&bin.join("git.ps1"), "exit 0\n"); - tmp +"#; + +/// The stand-in `git`, covering the commit resolution the blessing +/// reconciliation performs. +const FAKE_GIT: &str = r" +if ($args -contains 'rev-parse') { + # The recipe resolves a possibly-abbreviated commit before blessing; + # FAKE_GIT_UNKNOWN_COMMIT makes that resolution fail. + if ($env:FAKE_GIT_UNKNOWN_COMMIT) { exit 128 } + Write-Output '8392995a3b94218612437d0b868df2a48029b6ea' + exit 0 +} +exit 0 +"; + +/// Writes the stand-in `cargo` and `git` into the fixture. +fn write_fake_bin(root: &Path) { + let bin = root.join("fake-bin"); + std::fs::create_dir_all(&bin).unwrap(); + write(&bin.join("cargo.ps1"), FAKE_CARGO); + write(&bin.join("git.ps1"), FAKE_GIT); } fn path_with_fake_bin(root: &Path) -> OsString { @@ -604,3 +670,205 @@ fn windows_arm64_fallback_accepts_empty_nextest_sets_in_both_configurations() { assert_eq!(calls.matches("--no-tests=pass").count(), 2, "calls:\n{calls}"); assert!(!calls.contains("llvm-cov"), "coverage commands must not run:\n{calls}"); } + +/// Runs `anvil-bench-history` in a fixture, with the fake cbh returning +/// `findings` and the given extra environment. Gating is CI-only, so the +/// scenarios that assert on the exit code set `ANVIL_BENCH_GATE`. +fn run_bench_history(findings: &str, extra: &[(&str, &OsStr)]) -> (TempDir, Output) { + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + let log = tmp.path().join("cargo.log"); + let mut environment: Vec<(&str, &OsStr)> = vec![ + ("FAKE_CBH_FINDINGS", OsStr::new(findings)), + ("FAKE_CARGO_LOG", log.as_os_str()), + ("ANVIL_BENCH_GATE", OsStr::new("1")), + ]; + environment.extend_from_slice(extra); + let output = run_just(tmp.path(), &["anvil-bench-history"], &environment); + (tmp, output) +} + +fn cargo_calls(root: &Path) -> String { + std::fs::read_to_string(root.join("cargo.log")).unwrap_or_default() +} + +#[test] +fn bench_history_gates_on_active_regressions_only() { + if !tools_available() { + return; + } + + // An active regression is the one state that gates. + let (tmp, output) = run_bench_history(ACTIVE_REGRESSION, &[]); + assert_failed(&output, "an active regression"); + let text = String::from_utf8_lossy(&output.stdout); + assert!(text.contains("emit_alloc/churn"), "names the benchmark:\n{text}"); + assert!(text.contains("8392995a"), "names the attributed commit:\n{text}"); + let calls = cargo_calls(tmp.path()); + assert!(calls.contains("bench-history collect"), "calls:\n{calls}"); + assert!(calls.contains("bench-history analyze"), "calls:\n{calls}"); + + // A recovered regression and an improvement both need no action. + for (findings, label) in [(INACTIVE_REGRESSION, "inactive"), (IMPROVEMENT, "improvement")] { + let (_tmp, output) = run_bench_history(findings, &[]); + assert!( + output.status.success(), + "{label} finding must not gate:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + } + + // A workspace with no benchmarks analyzes to nothing and stays green: + // adopting the capability must not turn such a repo permanently red. + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[]); + assert!( + output.status.success(), + "an empty history must be a clean no-op:\n{}", + String::from_utf8_lossy(&output.stdout) + ); +} + +#[test] +fn bench_history_reports_without_gating_outside_ci() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + // No ANVIL_BENCH_GATE, and the CI markers explicitly cleared: a laptop's + // measurement noise must not fail a pre-release `anvil-full` and invite + // silencing it with a committed blessing. + let output = run_just( + tmp.path(), + &["anvil-bench-history"], + &[ + ("FAKE_CBH_FINDINGS", OsStr::new(ACTIVE_REGRESSION)), + ("CI", OsStr::new("")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "a local run reports but does not gate:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + let text = String::from_utf8_lossy(&output.stdout); + assert!(text.contains("emit_alloc/churn"), "still reports the finding:\n{text}"); + assert!(text.contains("ANVIL_BENCH_GATE"), "points at the opt-in:\n{text}"); +} + +#[test] +fn bench_history_propagates_tool_failure() { + if !tools_available() { + return; + } + // A tool that fails to *run* is not "no regressions". + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[("FAKE_CBH_COLLECT_EXIT", OsStr::new("3"))]); + assert_failed(&output, "a failing collect"); +} + +/// Runs the private blessing reconciliation directly, so the prefix-matching +/// boundary is pinned without going through a whole analysis. +fn run_bless(blessings_file: &str, applied: &str) -> (TempDir, Output) { + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + write(&tmp.path().join(".config/bench-blessings.toml"), blessings_file); + let log = tmp.path().join("cargo.log"); + let output = run_just( + tmp.path(), + &["_anvil-bench-history-bless", "store", ".config/bench-blessings.toml"], + &[("FAKE_CBH_BLESSINGS", OsStr::new(applied)), ("FAKE_CARGO_LOG", log.as_os_str())], + ); + (tmp, output) +} + +#[test] +fn bench_history_bless_reconciles_on_exact_prefix_identity() { + if !tools_available() { + return; + } + let requested = "[[blessing]]\n\ + benchmark = \"emit_alloc\"\n\ + commit = \"8392995a\"\n\ + reason = \"arena allocator tradeoff\"\n"; + + // A stored blessing of the *narrower* `emit_alloc/churn` does not cover + // the requested broader `emit_alloc`. Treating it as already-applied + // would leave the build red while claiming nothing needed doing. + let narrower = r#"{"blessings":[{"commit":"8392995a3b94","benchmark":"emit_alloc/churn"}]}"#; + let (tmp, output) = run_bless(requested, narrower); + assert!( + output.status.success(), + "reconciliation failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + cargo_calls(tmp.path()).contains("bench-history bless"), + "a narrower stored blessing must not satisfy a broader request:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + + // The exact same prefix already recorded is a no-op, so a scheduled run + // never re-appends a sidecar that is already in effect. + let exact = r#"{"blessings":[{"commit":"8392995a3b94","prefixes":["emit_alloc"]}]}"#; + let (tmp, output) = run_bless(requested, exact); + assert!(output.status.success()); + assert!( + !cargo_calls(tmp.path()).contains("bench-history bless"), + "an already-applied blessing must not be re-appended:\n{}", + String::from_utf8_lossy(&output.stdout) + ); +} + +#[test] +fn bench_history_bless_rejects_malformed_entries() { + if !tools_available() { + return; + } + let applied = r#"{"blessings":[]}"#; + + // A `#` inside a value is content, not a comment: silently truncating a + // reason citing an issue number would lose exactly what makes the audit + // trail worth keeping. + let hashed = "[[blessing]]\n\ + benchmark = \"emit_alloc\"\n\ + commit = \"8392995a\"\n\ + reason = \"accepted in #1234\"\n"; + let (tmp, output) = run_bless(hashed, applied); + assert!( + output.status.success(), + "a # inside a quoted value is content:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("#1234"), + "the reason must survive intact:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + assert!(cargo_calls(tmp.path()).contains("bench-history bless")); + + // Anything the subset cannot represent is rejected, not reinterpreted. + for (body, label) in [ + ("[[blessing]]\nbenchmark = emit_alloc\n", "unquoted value"), + ("[[blessing]]\nbenchmark = \"a\"\ncommit = \"b\"\n", "missing reason"), + ("[[other]]\nbenchmark = \"a\"\n", "unexpected table"), + ] { + let (_tmp, output) = run_bless(body, applied); + assert_failed(&output, label); + } +} diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index 855c19422..a72dfd08d 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -1667,37 +1667,49 @@ stages: # publishes the updated store at the end of the job -- including # when the analysis failed the job, so the samples collected while # the pipeline is red are not lost. + # + # The explicit `checkout` leads the step list rather than going + # through a wrapper parameter: `steps/job.yml` is the file adopters + # fork for 1ESPT and friends, so binding a parameter their wrapper + # does not declare would fail template expansion for the whole + # pipeline. An explicit checkout step needs nothing from the wrapper + # and replaces the implicit default checkout. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. - fetchDepth: '0' - artifacts: - - name: bench-history-linux - path: target/anvil/bench-history steps: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. + - checkout: self + fetchDepth: 0 + lfs: true - template: steps/bench-history-restore.yml parameters: artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml + - template: steps/bench-history-publish.yml + parameters: + artifact: bench-history-linux - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} - fetchDepth: '0' - artifacts: - - name: bench-history-windows - path: target/anvil/bench-history steps: + - checkout: self + fetchDepth: 0 + lfs: true - template: steps/bench-history-restore.yml parameters: artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml + - template: steps/bench-history-publish.yml + parameters: + artifact: bench-history-windows === .pipelines/anvil/steps/advisory-comments.yml === # Copyright (c) Microsoft Corporation. @@ -1797,14 +1809,23 @@ steps: } } -=== .pipelines/anvil/steps/bench-history-restore.yml === +=== .pipelines/anvil/steps/bench-history-publish.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. # GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. # Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # -# Restores the benchmark history the previous scheduled run published. +# Publishes the updated benchmark history as this leg's artifact. +# +# This does not go through the §4.1 job wrapper's `artifacts` contract +# because it needs a condition the contract does not express: the store is +# only a valid continuation of the chain when the restore reached a known +# state. Publishing after an operational restore failure would overwrite a +# good history with a truncated snapshot. +# +# The condition still includes failed runs: a flagged regression fails the +# job, and those samples belong in the history. # # See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md parameters: @@ -1814,29 +1835,107 @@ parameters: type: string default: target/anvil/bench-history steps: - # The store has to exist even when there is nothing to restore, so the - # first ever run analyzes an empty history instead of erroring. - - pwsh: New-Item -ItemType Directory -Force -Path '${{ parameters.path }}' | Out-Null - displayName: Prepare benchmark history store - - task: DownloadPipelineArtifact@2 - displayName: Restore ${{ parameters.artifact }} - # The first run has no artifact to restore; a missing artifact is a - # cold start, not a failure. - continueOnError: true + - task: PublishPipelineArtifact@1 + displayName: Publish ${{ parameters.artifact }} + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) inputs: - buildType: specific - project: $(System.TeamProjectId) - definition: $(System.DefinitionId) - buildVersionToDownload: latestFromBranch - branchName: $(Build.SourceBranch) - # Take the newest run that carries the artifact whatever its - # outcome. Restoring only from green runs would drop every sample - # collected while the pipeline was red from a regression -- - # precisely the window that matters. - allowPartiallySucceededBuilds: true - allowFailedBuilds: true - artifactName: ${{ parameters.artifact }} targetPath: ${{ parameters.path }} + artifact: ${{ parameters.artifact }} + +=== .pipelines/anvil/steps/bench-history-restore.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Restores the benchmark history published by the most recent build that +# carries this leg's artifact. +# +# `DownloadPipelineArtifact@2` with `latestFromBranch` resolves a single +# build and yields nothing if that build has no such artifact -- it does +# not walk back. A cancelled or never-publishing latest build would +# therefore cold-start a store that actually has history. This step +# resolves the build itself, so absence of the artifact throughout the +# window (a genuine cold start) stays distinct from an operational +# failure, which fails the job rather than silently publishing a +# truncated snapshot over a good chain. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history + - name: window + type: number + default: 30 +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $artifact = '${{ parameters.artifact }}' + $path = '${{ parameters.path }}' + $window = ${{ parameters.window }} + + New-Item -ItemType Directory -Force -Path $path | Out-Null + + if (-not $env:SYSTEM_ACCESSTOKEN) { + Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." + exit 1 + } + + $collection = $env:SYSTEM_COLLECTIONURI + $project = $env:SYSTEM_TEAMPROJECTID + $definition = $env:SYSTEM_DEFINITIONID + $branch = $env:BUILD_SOURCEBRANCH + $headers = @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + + # Newest first, whatever the outcome: a flagged regression fails the + # build, so restricting to successful builds would discard exactly + # the stretch of history that matters most. + $runsUri = "$collection$project/_apis/build/builds?definitions=$definition&branchName=$branch&`$top=$window&queryOrder=finishTimeDescending&api-version=7.0" + $runs = (Invoke-RestMethod -Uri $runsUri -Headers $headers).value + + foreach ($run in $runs) { + if ($run.id -eq $env:BUILD_BUILDID) { continue } + + $artifactUri = "$collection$project/_apis/build/builds/$($run.id)/artifacts?artifactName=$artifact&api-version=7.0" + try { + $found = Invoke-RestMethod -Uri $artifactUri -Headers $headers + } catch { + # 404 is "this build has no such artifact" -- the expected, + # common case while walking back. Anything else is an + # operational failure and must not be read as absence. + $status = $_.Exception.Response.StatusCode.value__ + if ($status -eq 404) { continue } + Write-Error "anvil: querying artifacts of build $($run.id) failed with HTTP $status; refusing to continue with an empty history." + exit 1 + } + if (-not $found.resource.downloadUrl) { continue } + + Write-Host "anvil: restoring $artifact from build $($run.id)" + $tmp = [System.IO.Path]::GetTempPath() + $zip = Join-Path $tmp "$artifact.zip" + $staging = Join-Path $tmp "$artifact-extract" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip + Expand-Archive -LiteralPath $zip -DestinationPath $staging -Force + # The archive nests its contents under a directory named for the + # artifact; lift them up into the store path. + $inner = Join-Path $staging $artifact + $source = if (Test-Path $inner) { $inner } else { $staging } + Copy-Item -Path (Join-Path $source '*') -Destination $path -Recurse -Force + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]restored" + exit 0 + } + + # Nothing in the window carried the artifact: a genuine cold start. + Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]cold-start" + displayName: Restore ${{ parameters.artifact }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) === .pipelines/anvil/steps/bench-history-summary.yml === # Copyright (c) Microsoft Corporation. @@ -1987,10 +2086,6 @@ steps: # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. -# - fetchDepth (string) Optional. When set, the job checks out explicitly at -# this depth ('0' for full history) instead of taking -# the implicit default checkout. 1ESPT wrappers map it -# onto templateContext.inputs instead. # - artifacts (object) Optional list of pipeline artifacts to publish. # Each item: { name: string, path: string }. # The default wrapper appends one @@ -2007,9 +2102,6 @@ parameters: type: object - name: steps type: stepList - - name: fetchDepth - type: string - default: '' - name: artifacts type: object default: [] @@ -2018,9 +2110,6 @@ jobs: - job: ${{ parameters.name }} pool: ${{ parameters.pool }} steps: - - ${{ if ne(parameters.fetchDepth, '') }}: - - checkout: self - fetchDepth: ${{ parameters.fetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ each artifact in parameters.artifacts }}: @@ -2960,13 +3049,30 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs # ANVIL_INCLUDE_* contract and always measures the whole workspace. # # Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. # # The store is the cross-run state the cloud wiring restores before and # publishes after this recipe; locally it is whatever has accumulated # under target/, which on a fresh checkout is empty and analyzes to a # clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. # Run the benchmarks and analyze the accumulated history for regressions. [script("pwsh", "-NoProfile")] @@ -3030,6 +3136,18 @@ anvil-bench-history: anvil-bench-history-validate-prereqs } Write-Host '' Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." exit 1 @@ -3060,13 +3178,19 @@ _anvil-bench-history-bless store blessings: } # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs. Depending on a TOML parser here would mean a - # second tool pin for three string fields. + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. $entries = New-Object System.Collections.Generic.List[object] $current = $null foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = ($rawLine -split '#', 2)[0].Trim() - if (-not $line) { continue } + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } if ($line -eq '[[blessing]]') { $current = @{} $entries.Add($current) | Out-Null @@ -3076,7 +3200,7 @@ _anvil-bench-history-bless store blessings: Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" exit 1 } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { if ($null -eq $current) { Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" exit 1 @@ -3084,7 +3208,7 @@ _anvil-bench-history-bless store blessings: $current[$Matches[1]] = $Matches[2] continue } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) exit 1 } @@ -3127,12 +3251,14 @@ _anvil-bench-history-bless store blessings: exit 1 } $resolved = $resolved.Trim() - # Stored commits are abbreviated, and a stored blessing names - # either the concrete benchmark it resolved to (once a run exists - # at that commit) or the prefix filter it was issued with. + # Compare the persisted identity exactly. A `StartsWith` in either + # direction is wrong: a stored concrete `foo/bar` would satisfy a + # prefix test for the broader `foo`, so a committed blessing of + # `foo` would be skipped while still leaving the build red -- with + # a log line claiming it was already in effect. $already = $applied | Where-Object { $resolved.StartsWith($_.commit) -and - (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + (($_.prefixes -contains $benchmark) -or ($_.benchmark -eq $benchmark)) } if ($already) { Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 614111d11..cfeb245a5 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -2361,6 +2361,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -2478,38 +2486,83 @@ jobs: with: # The analysis orders each series by first-parent commit # topology and locates the merge-base, so it needs the whole - # commit graph. + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. fetch-depth: 0 + lfs: true - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} ARTIFACT: bench-history-${{ matrix.os }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename + # must not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" run: | set -euo pipefail mkdir -p target/anvil/bench-history + # Walk back from the newest run and take the first one that - # carries the artifact. Restoring from the latest *successful* - # run would drop every sample collected while the pipeline was - # red from a regression — precisely the window that matters. - # The in-progress run of this very workflow has not uploaded - # yet, so it simply fails the download and the loop moves on. - for run_id in $(gh run list --workflow anvil-scheduled.yml \ - --branch "$DEFAULT_BRANCH" --limit 10 \ + # carries this leg's artifact. Restoring from the latest + # *successful* run would drop every sample collected while the + # pipeline was red from a regression — precisely the window + # that matters. + # + # Absence and failure are kept distinct. A run is only a + # candidate once the artifacts API confirms the artifact exists + # and has not expired; a download that then fails is an + # operational error (token, API, corrupt payload) and fails the + # job rather than being silently downgraded to a cold start. + # That distinction is what stops one transient failure from + # publishing an empty store over a good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do - if gh run download "$run_id" --name "$ARTIFACT" \ - --dir target/anvil/bench-history 2>/dev/null; then - echo "restored benchmark history from run $run_id" - exit 0 + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 done - echo "no $ARTIFACT artifact in the recent scheduled runs; starting with an empty history" + + # No run in the window carried the artifact. That is a genuine + # cold start (first run, or the chain lapsed), so it is surfaced + # on the summary rather than only in this log — "history quietly + # restarted" must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks + env: + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - name: Save benchmark history - # always(): the run's own samples belong in the history even - # when the analysis flagged a regression and failed the job. - if: always() + # always(): the run's own samples belong in the history even when + # the analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the + # restore failed operationally the store is not a continuation of + # the chain, and publishing it would overwrite good history with a + # truncated snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: bench-history-${{ matrix.os }} @@ -2845,13 +2898,30 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs # ANVIL_INCLUDE_* contract and always measures the whole workspace. # # Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. # # The store is the cross-run state the cloud wiring restores before and # publishes after this recipe; locally it is whatever has accumulated # under target/, which on a fresh checkout is empty and analyzes to a # clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. # Run the benchmarks and analyze the accumulated history for regressions. [script("pwsh", "-NoProfile")] @@ -2915,6 +2985,18 @@ anvil-bench-history: anvil-bench-history-validate-prereqs } Write-Host '' Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." exit 1 @@ -2945,13 +3027,19 @@ _anvil-bench-history-bless store blessings: } # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs. Depending on a TOML parser here would mean a - # second tool pin for three string fields. + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. $entries = New-Object System.Collections.Generic.List[object] $current = $null foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = ($rawLine -split '#', 2)[0].Trim() - if (-not $line) { continue } + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } if ($line -eq '[[blessing]]') { $current = @{} $entries.Add($current) | Out-Null @@ -2961,7 +3049,7 @@ _anvil-bench-history-bless store blessings: Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" exit 1 } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { if ($null -eq $current) { Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" exit 1 @@ -2969,7 +3057,7 @@ _anvil-bench-history-bless store blessings: $current[$Matches[1]] = $Matches[2] continue } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) exit 1 } @@ -3012,12 +3100,14 @@ _anvil-bench-history-bless store blessings: exit 1 } $resolved = $resolved.Trim() - # Stored commits are abbreviated, and a stored blessing names - # either the concrete benchmark it resolved to (once a run exists - # at that commit) or the prefix filter it was issued with. + # Compare the persisted identity exactly. A `StartsWith` in either + # direction is wrong: a stored concrete `foo/bar` would satisfy a + # prefix test for the broader `foo`, so a committed blessing of + # `foo` would be skipped while still leaving the build red -- with + # a log line claiming it was already in effect. $already = $applied | Where-Object { $resolved.StartsWith($_.commit) -and - (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + (($_.prefixes -contains $benchmark) -or ($_.benchmark -eq $benchmark)) } if ($already) { Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index a97df441d..eef3cebdb 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -1519,13 +1519,30 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs # ANVIL_INCLUDE_* contract and always measures the whole workspace. # # Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. # # The store is the cross-run state the cloud wiring restores before and # publishes after this recipe; locally it is whatever has accumulated # under target/, which on a fresh checkout is empty and analyzes to a # clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. # Run the benchmarks and analyze the accumulated history for regressions. [script("pwsh", "-NoProfile")] @@ -1589,6 +1606,18 @@ anvil-bench-history: anvil-bench-history-validate-prereqs } Write-Host '' Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." exit 1 @@ -1619,13 +1648,19 @@ _anvil-bench-history-bless store blessings: } # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs. Depending on a TOML parser here would mean a - # second tool pin for three string fields. + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. $entries = New-Object System.Collections.Generic.List[object] $current = $null foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = ($rawLine -split '#', 2)[0].Trim() - if (-not $line) { continue } + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } if ($line -eq '[[blessing]]') { $current = @{} $entries.Add($current) | Out-Null @@ -1635,7 +1670,7 @@ _anvil-bench-history-bless store blessings: Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" exit 1 } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { if ($null -eq $current) { Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" exit 1 @@ -1643,7 +1678,7 @@ _anvil-bench-history-bless store blessings: $current[$Matches[1]] = $Matches[2] continue } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) exit 1 } @@ -1686,12 +1721,14 @@ _anvil-bench-history-bless store blessings: exit 1 } $resolved = $resolved.Trim() - # Stored commits are abbreviated, and a stored blessing names - # either the concrete benchmark it resolved to (once a run exists - # at that commit) or the prefix filter it was issued with. + # Compare the persisted identity exactly. A `StartsWith` in either + # direction is wrong: a stored concrete `foo/bar` would satisfy a + # prefix test for the broader `foo`, so a committed blessing of + # `foo` would be skipped while still leaving the build red -- with + # a log line claiming it was already in effect. $already = $applied | Where-Object { $resolved.StartsWith($_.commit) -and - (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + (($_.prefixes -contains $benchmark) -or ($_.benchmark -eq $benchmark)) } if ($already) { Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just index 7b5631370..286b2760f 100644 --- a/justfiles/anvil/checks/bench-history.just +++ b/justfiles/anvil/checks/bench-history.just @@ -13,13 +13,30 @@ # ANVIL_INCLUDE_* contract and always measures the whole workspace. # # Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history) -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. # # The store is the cross-run state the cloud wiring restores before and # publishes after this recipe; locally it is whatever has accumulated # under target/, which on a fresh checkout is empty and analyzes to a # clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. # Run the benchmarks and analyze the accumulated history for regressions. [script("pwsh", "-NoProfile")] @@ -83,6 +100,18 @@ anvil-bench-history: anvil-bench-history-validate-prereqs } Write-Host '' Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." exit 1 @@ -113,13 +142,19 @@ _anvil-bench-history-bless store blessings: } # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs. Depending on a TOML parser here would mean a - # second tool pin for three string fields. + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. $entries = New-Object System.Collections.Generic.List[object] $current = $null foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = ($rawLine -split '#', 2)[0].Trim() - if (-not $line) { continue } + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } if ($line -eq '[[blessing]]') { $current = @{} $entries.Add($current) | Out-Null @@ -129,7 +164,7 @@ _anvil-bench-history-bless store blessings: Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" exit 1 } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"(.*)"$') { + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { if ($null -eq $current) { Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" exit 1 @@ -137,7 +172,7 @@ _anvil-bench-history-bless store blessings: $current[$Matches[1]] = $Matches[2] continue } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1} (expected a [[blessing]] header or a key = "value" pair)' -f $line, $blessingsFile) + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) exit 1 } @@ -180,12 +215,14 @@ _anvil-bench-history-bless store blessings: exit 1 } $resolved = $resolved.Trim() - # Stored commits are abbreviated, and a stored blessing names - # either the concrete benchmark it resolved to (once a run exists - # at that commit) or the prefix filter it was issued with. + # Compare the persisted identity exactly. A `StartsWith` in either + # direction is wrong: a stored concrete `foo/bar` would satisfy a + # prefix test for the broader `foo`, so a committed blessing of + # `foo` would be skipped while still leaving the build red -- with + # a log line claiming it was already in effect. $already = $applied | Where-Object { $resolved.StartsWith($_.commit) -and - (($_.benchmark -and $_.benchmark.StartsWith($benchmark)) -or ($_.prefixes -contains $benchmark)) + (($_.prefixes -contains $benchmark) -or ($_.benchmark -eq $benchmark)) } if ($already) { Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" From e43fd787bc118a494fbda953d9c3e4b37daafb71 Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Fri, 21 Aug 2026 14:24:32 +0200 Subject: [PATCH 13/24] fix(cargo-anvil): keep the ADO publish inside the wrapper seam; cover the restores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extensibility: - Publish through `steps/job.yml`'s `artifacts` contract again, with an optional per-artifact `condition`. Emitting a bare PublishPipelineArtifact@1 inside the generic step list bypassed the translation a forked (1ESPT) wrapper performs -- routing around the seam rather than expressing the guard inside it. The guard itself is unchanged: an operational restore failure still must not overwrite a good chain. - Note in the wrapper's own contract block that a step list may lead with its own `checkout: self`, so a fork does not add a second one. - Expose the machine key on ADO as a `benchMachineKey` stages parameter, so the documented answer to the top caveat is genuinely available on both backends rather than only on GitHub. Correctness: - Give the blessing listing a process-scoped temp filename. Two jobs sharing a machine -- matrix legs on a self-hosted agent, or concurrent local runs -- read each other's listing through the fixed one. Found by test interference. Verification: - Execute both restore blocks against mocked transports, covering the branches a string assertion cannot see: walking back past a run without the artifact, a positively identified cold start, and every operational failure exiting non-zero *without* marking a publishable restore state. - Add a criterion benchmark over catalog assembly and checksumming, so the dogfooded scheduled job produces a real series instead of an empty one. criterion was already a workspace dependency. Docs: ado.md §12 describes the walk and the guard that shipped rather than the DownloadPipelineArtifact@2 round-trip it replaced, and github.md states what test an input must pass to earn a place on the workflow surface. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 4 +- Cargo.lock | 158 ++++++++++- crates/cargo-anvil/Cargo.toml | 5 + crates/cargo-anvil/benches/catalog.rs | 37 +++ crates/cargo-anvil/docs/design/ado.md | 51 +++- crates/cargo-anvil/docs/design/github.md | 29 +- crates/cargo-anvil/src/anvil/artifacts/ado.rs | 33 +-- .../templates/ado/scheduled-stages.yml | 37 ++- .../ado/steps/bench-history-publish.yml | 31 --- .../cargo-anvil/templates/ado/steps/job.yml | 21 +- .../justfiles/anvil/checks/bench-history.just | 5 +- crates/cargo-anvil/tests/recipe_contracts.rs | 252 ++++++++++++++++++ .../snapshots/snapshots__ado_backend.snap | 96 ++++--- .../snapshots/snapshots__github_backend.snap | 5 +- .../snapshots/snapshots__local_only.snap | 5 +- justfiles/anvil/checks/bench-history.just | 5 +- 16 files changed, 634 insertions(+), 140 deletions(-) create mode 100644 crates/cargo-anvil/benches/catalog.rs delete mode 100644 crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml diff --git a/.anvil.lock b/.anvil.lock index 9099d8ddc..949167e24 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.4.0" -catalog_checksum = "sha256:9cffc53cd58efdf370c607161c95d9e65815667b71be95b9bcb838297386e15b" +catalog_checksum = "sha256:8b6a280ef7508787594e0f020232530d6562ae3431f3a9fe8060623675ce7828" [[file]] path = ".anvil/container/Containerfile" @@ -105,7 +105,7 @@ checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536 [[file]] path = "justfiles/anvil/checks/bench-history.just" -checksum = "sha256:222f3e22c442e8a4dcb155b4cd29052089afa45b823c30b5a88c7d4535ba7c9a" +checksum = "sha256:c0c4cd8c84b1ae778d1f9b8fe9e0f90ca607dd0df626f88da8f52aecc02a23ff" [[file]] path = "justfiles/anvil/checks/bench.just" diff --git a/Cargo.lock b/Cargo.lock index 722242494..7596690a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -32,6 +41,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -304,6 +319,7 @@ version = "0.4.0" dependencies = [ "assert_cmd", "clap", + "criterion", "insta", "mutants", "ohno", @@ -491,6 +507,12 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -571,6 +593,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.1" @@ -746,6 +795,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -780,6 +862,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -2041,6 +2129,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hash32" version = "0.3.1" @@ -2429,6 +2528,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -2910,6 +3018,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2938,6 +3052,16 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3207,7 +3331,7 @@ checksum = "e0ebcaa9da49ee86809a3c5ff57c0af35530951ded4e7aad371839f173da7ef7" dependencies = [ "crossbeam-channel", "crossbeam-utils", - "itertools", + "itertools 0.14.0", "jod-thread", "libc", "miow", @@ -3222,7 +3346,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a584372630d85436a206d362b26e5156367f348043151b132011f360c63fa18" dependencies = [ "either", - "itertools", + "itertools 0.14.0", "ra_ap_parser", "ra_ap_stdx", "rowan", @@ -4229,6 +4353,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.12.0" @@ -5307,6 +5441,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zerofrom" version = "0.1.8" diff --git a/crates/cargo-anvil/Cargo.toml b/crates/cargo-anvil/Cargo.toml index 372a4d751..62385fa9f 100644 --- a/crates/cargo-anvil/Cargo.toml +++ b/crates/cargo-anvil/Cargo.toml @@ -39,12 +39,17 @@ tracing-subscriber = { workspace = true, features = ["fmt"] } [dev-dependencies] assert_cmd = { workspace = true } +criterion = { workspace = true } insta = { workspace = true, features = ["filters"] } predicates = { workspace = true } serial_test = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } +[[bench]] +name = "catalog" +harness = false + # >>> anvil-managed: anvil-lints [lints] workspace = true diff --git a/crates/cargo-anvil/benches/catalog.rs b/crates/cargo-anvil/benches/catalog.rs new file mode 100644 index 000000000..6abe5cfa5 --- /dev/null +++ b/crates/cargo-anvil/benches/catalog.rs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Benchmarks for assembling the built-in catalog. +//! +//! `Catalog::anvil()` builds the whole artifact set — every embedded +//! template, every per-group fan-out — and `checksum()` renders and hashes +//! all of it. Both run on every `cargo anvil` invocation before anything is +//! written, so their cost is paid by every adopter on every update, and both +//! grow with the catalog: this crate's own history is a steady accretion of +//! checks, groups and backend files. +//! +//! They are also the shape a trend watch handles well — pure, deterministic, +//! no I/O, no network — so a move here is a change in the code rather than in +//! the environment. + +use cargo_anvil::Catalog; +use criterion::{Criterion, criterion_group, criterion_main}; + +fn catalog(c: &mut Criterion) { + let mut group = c.benchmark_group("catalog"); + + // Assembly alone: the embedded templates and the per-group expansions. + group.bench_function("anvil", |b| b.iter(Catalog::anvil)); + + // Assembly plus rendering and hashing every artifact body, which is what + // the update path pays to decide whether anything changed. + group.bench_function("checksum", |b| { + let catalog = Catalog::anvil(); + b.iter(|| catalog.checksum()); + }); + + group.finish(); +} + +criterion_group!(benches, catalog); +criterion_main!(benches); diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index 7531da874..f4735dcd7 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -379,7 +379,7 @@ The contract is intentionally small and stable: | `name` | `string` | yes | Job name; ADO derives the display name from it. | | `pool` | `object` | yes | Pool block, passed verbatim to ADO's `pool:` key. `linuxPool` and `windowsPool` at the stage level are object parameters, so users can override their shape (e.g. `{ name, os, image }` for 1ESPT). | | `steps` | `stepList` | yes | Body of the job. Templated step lists are fine — the wrapper splices them in via `${{ each step in parameters.steps }}: - ${{ step }}`. | -| `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. The stages templates don't need to know which backend they're targeting. | +| `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string, condition: string (optional) }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. `condition` defaults to `succeededOrFailed()`; a caller that must not publish in some states sets it, and a fork carries it through to whatever output shape it emits. The stages templates don't need to know which backend they're targeting. | A job that needs a non-default checkout (depth, LFS) puts an explicit `checkout` step at the head of its own `steps` list rather than growing this contract. The @@ -404,7 +404,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} @@ -422,7 +425,10 @@ jobs: - output: pipelineArtifact targetPath: ${{ artifact.path }} artifactName: ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() steps: - ${{ each step in parameters.steps }}: - ${{ step }} @@ -839,19 +845,33 @@ Each scheduled benchmark job: 1. checks out with full history and LFS via an explicit `checkout` step at the head of its own step list (analysis reads the commit graph; benchmark inputs may be LFS-tracked); -2. **restores** the history with `DownloadPipelineArtifact@2` - (`buildVersionToDownload: latestFromBranch`, the default branch); the first run - finds none and starts empty; +2. **restores** the history with `steps/bench-history-restore.yml`, which walks the + pipeline's own builds on the branch newest-first + (`_apis/build/builds?…&queryOrder=finishTimeDescending`) and, per build, queries + the artifacts endpoint for this leg's artifact. A `404` means that build simply + has no such artifact and the walk continues; any other status is an operational + failure and fails the job. Finding none across the whole window is a genuine cold + start; 3. applies any pending blessings, runs collect + analyze, writing findings to a findings file which a following step attaches to the build summary; 4. **publishes** the updated store through the wrapper's `artifacts` parameter - (`{ name: bench-history-, path: }`), which the default wrapper emits - as `PublishPipelineArtifact@1` and 1ESPT wrappers as a `pipelineArtifact` output. - -The restore admits failed and partially succeeded builds, which is what keeps the -chain intact across a regression: a flagged regression fails the stage, so a -success-only restore would discard every sample taken while the pipeline stayed red. -The publish likewise runs whatever the job's outcome. + (`{ name: bench-history-, path: , condition: … }`), which the default + wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a + `pipelineArtifact` output. + +`DownloadPipelineArtifact@2` is not used for the restore: `latestFromBranch` resolves +a single build and does not walk, so a cancelled or never-publishing latest build +would cold-start a store that still has usable history. + +The walk is outcome-agnostic, which is what keeps the chain intact across a +regression: a flagged regression fails the stage, so a success-only restore would +discard every sample taken while the pipeline stayed red. Publishing is likewise not +limited to green runs, but it *is* conditioned on the restore having reached a known +state — the `condition` on the artifact entry. An operational restore failure +therefore neither continues silently nor overwrites a good chain with a truncated +snapshot. That guard is the reason the `artifacts` contract carries an optional +`condition` (§4.1) rather than the benchmark group emitting its own publish task, +which would bypass the translation a forked wrapper performs. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits @@ -864,3 +884,8 @@ the findings remain in the build summary. Blessings are applied from a committed `.config/bench-blessings.toml` before analyze (step 3) — a reviewed pull request, not an out-of-band action. + +The machine key the history is partitioned by is a `benchMachineKey` parameter on the +stages template, surfaced as a stage-level `ANVIL_BENCH_MACHINE_KEY` variable. Empty +uses the hardware fingerprint; a stable pool label trades partition fidelity for +density on a heterogeneous pool. diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index bfeb398af..bd690a56e 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -448,11 +448,30 @@ The reusable workflow declares a small input set so the root workflow can pass o | `windows_runner` | string | `windows-latest` | Runner label for x86_64 Windows jobs. | | `linux_arm_runner` | string | `ubuntu-24.04-arm` | Runner label for aarch64 Linux jobs. | | `windows_arm_runner` | string | `windows-11-arm` | Runner label for aarch64 Windows jobs. | - -The input surface is intentionally narrow: only per-leg *runner labels* are exposed, -because swapping in self-hosted runners is the one common need that doesn't require -otherwise touching the workflow. The OS matrix shape (which legs run) is fixed in the -workflow source — see the discussion under the PR snippet above. +| `bench_machine_key` | string | *(empty)* | Machine key the benchmark history is partitioned by (scheduled workflow only). | + +The input surface is **per-leg runner labels plus a per-capability knob where the +capability's behaviour depends on the runner fleet rather than on the source tree**. +Runner labels are exposed because swapping in self-hosted runners is the one common +need that doesn't require otherwise touching the workflow. `bench_machine_key` earns +an input on the same test: benchmark history is partitioned by a hardware +fingerprint, so a heterogeneous pool can fragment a series into partitions too sparse +to analyze, and only the adopter knows whether their fleet is uniform enough to +substitute a stable pool label. That is a property of *their* runners, invisible to +the catalog, so no recipe default or env var can supply it. + +A knob that fails that test — anything the catalog could decide, or that varies per +developer rather than per fleet — stays an env var read by the recipe +(`ANVIL_BENCH_HISTORY_STORE` is the local-only counterexample) rather than growing +this surface. + +Setting an input means editing the generated root workflow, which takes ownership of +it through the dirty-file flow (§3): subsequent updates Propose into an +`.anvil-proposed` sibling instead of overwriting. That cost is real and is why the +surface stays small. + +The OS matrix shape (which legs run) is fixed in the workflow source — see the +discussion under the PR snippet above. The reusable workflows also declare an optional `workflow_call` secret `CODECOV_TOKEN`. See §10 (Coverage upload) for how it's used. diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index bff0e4069..3cebf2232 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -25,9 +25,6 @@ const JOB_WRAPPER: &str = include_str!("../../../templates/ado/steps/job.yml"); /// Embedded body of the benchmark-history restore step template. const BENCH_HISTORY_RESTORE_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-restore.yml"); -/// Embedded body of the benchmark-history publish step template. -const BENCH_HISTORY_PUBLISH_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-publish.yml"); - /// Embedded body of the benchmark-findings build-summary step template. const BENCH_HISTORY_SUMMARY_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-summary.yml"); @@ -132,17 +129,6 @@ pub fn bench_history_summary() -> Artifact { ) } -/// `.pipelines/anvil/steps/bench-history-publish.yml` — publishes the -/// updated benchmark history as this leg's artifact. -#[must_use] -pub fn bench_history_publish() -> Artifact { - Artifact::backend_file( - Backend::Ado, - ".pipelines/anvil/steps/bench-history-publish.yml", - BENCH_HISTORY_PUBLISH_STEP, - ) -} - /// `.pipelines/anvil/pr.yml` — the PR-tier stages template. #[must_use] pub fn pr_stages() -> Artifact { @@ -220,7 +206,6 @@ pub(crate) fn all() -> Vec { job_wrapper(), bench_history_restore(), bench_history_summary(), - bench_history_publish(), ]; for (group, path) in GROUP_STEPS { out.push(Artifact::backend_file(Backend::Ado, path, render_group_step(group))); @@ -435,7 +420,17 @@ mod tests { } assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-restore.yml")); assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-summary.yml")); - assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-publish.yml")); + // The publish goes through the wrapper's `artifacts` contract rather + // than emitting its own task, so a forked wrapper still performs the + // translation it exists for. The guard rides along as a `condition`. + assert_eq!( + SCHEDULED_STAGES + .matches("condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], ''))") + .count(), + 2, + "each leg's artifact entry carries the restore guard" + ); + assert!(JOB_WRAPPER.contains("${{ if artifact.condition }}")); // Take the newest build carrying the artifact whatever its outcome: // restoring only from green builds would drop every sample collected // while the pipeline was red from a regression. @@ -449,11 +444,11 @@ mod tests { !BENCH_HISTORY_RESTORE_STEP.contains("continueOnError"), "a blanket continueOnError would read every failure as a cold start" ); - // The publish is guarded on the restore having reached a known state. - assert!(BENCH_HISTORY_PUBLISH_STEP.contains("ne(variables['ANVIL_BENCH_RESTORE'], '')")); - assert!(BENCH_HISTORY_PUBLISH_STEP.contains("succeededOrFailed()")); assert!(BENCH_HISTORY_SUMMARY_STEP.contains("##vso[task.uploadsummary]")); assert!(BENCH_HISTORY_SUMMARY_STEP.contains("condition: succeededOrFailed()")); + // The machine-key escape hatch is an input on this backend too. + assert!(SCHEDULED_STAGES.contains("name: benchMachineKey")); + assert!(SCHEDULED_STAGES.contains("ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }}")); } #[test] diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index 50087d5f6..ac2fab1a8 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -13,6 +13,13 @@ parameters: - name: windowsPool type: object default: { vmImage: windows-latest } + - name: benchMachineKey + type: string + # Machine key the benchmark history is partitioned by. Empty uses + # cargo-bench-history's hardware fingerprint; set a stable pool label + # when the agent pool is heterogeneous enough to fragment a series into + # partitions too sparse to analyze. + default: '' stages: - stage: scheduled_test @@ -110,14 +117,22 @@ stages: - stage: scheduled_benchmarks displayName: anvil scheduled-benchmarks dependsOn: [] + variables: + # Read by the bench-history recipe; empty means "use the hardware + # fingerprint". ADO exports pipeline variables as environment + # variables, so this reaches the recipe without further plumbing. + ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} jobs: # Benchmark regression detection. OS scope matches # scheduled-exhaustive. The history is partitioned per machine, so # each leg carries its own artifact rather than sharing one name. - # The restore step runs first; the wrapper's `artifacts` contract - # publishes the updated store at the end of the job -- including - # when the analysis failed the job, so the samples collected while - # the pipeline is red are not lost. + # + # The publish goes through the wrapper's `artifacts` contract like + # every other job, carrying a `condition`: the store is only a valid + # continuation of the chain once the restore reached a known state, + # so an operational restore failure must not overwrite good history + # with a truncated snapshot. Failed runs still publish -- a flagged + # regression fails the job, and those samples belong in the history. # # The explicit `checkout` leads the step list rather than going # through a wrapper parameter: `steps/job.yml` is the file adopters @@ -129,6 +144,10 @@ stages: parameters: name: linux pool: ${{ parameters.linuxPool }} + artifacts: + - name: bench-history-linux + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: # The analysis orders each series by first-parent commit # topology and locates the merge-base, so it needs the whole @@ -142,13 +161,14 @@ stages: artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml - - template: steps/bench-history-publish.yml - parameters: - artifact: bench-history-linux - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} + artifacts: + - name: bench-history-windows + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - checkout: self fetchDepth: 0 @@ -158,6 +178,3 @@ stages: artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml - - template: steps/bench-history-publish.yml - parameters: - artifact: bench-history-windows diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml deleted file mode 100644 index 3cfe71e04..000000000 --- a/crates/cargo-anvil/templates/ado/steps/bench-history-publish.yml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md -# -# Publishes the updated benchmark history as this leg's artifact. -# -# This does not go through the §4.1 job wrapper's `artifacts` contract -# because it needs a condition the contract does not express: the store is -# only a valid continuation of the chain when the restore reached a known -# state. Publishing after an operational restore failure would overwrite a -# good history with a truncated snapshot. -# -# The condition still includes failed runs: a flagged regression fails the -# job, and those samples belong in the history. -# -# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md -parameters: - - name: artifact - type: string - - name: path - type: string - default: target/anvil/bench-history -steps: - - task: PublishPipelineArtifact@1 - displayName: Publish ${{ parameters.artifact }} - condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) - inputs: - targetPath: ${{ parameters.path }} - artifact: ${{ parameters.artifact }} diff --git a/crates/cargo-anvil/templates/ado/steps/job.yml b/crates/cargo-anvil/templates/ado/steps/job.yml index 028b1ccd2..3b9cd0655 100644 --- a/crates/cargo-anvil/templates/ado/steps/job.yml +++ b/crates/cargo-anvil/templates/ado/steps/job.yml @@ -17,15 +17,23 @@ # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# A step list may lead with its own `checkout: self` +# (the benchmark group does, for depth and LFS), so a +# wrapper must NOT add a checkout of its own -- doing +# so would give those jobs two. # - artifacts (object) Optional list of pipeline artifacts to publish. -# Each item: { name: string, path: string }. -# The default wrapper appends one -# PublishPipelineArtifact@1 task per entry; 1ESPT -# wrappers translate these into +# Each item: { name: string, path: string, +# condition: string (optional) }. The default wrapper +# appends one PublishPipelineArtifact@1 task per +# entry; 1ESPT wrappers translate these into # templateContext.outputs.pipelineArtifact blocks # at job level. The contract is the same either way, # so the stages templates don't need to know which # backend they're targeting. +# `condition` defaults to succeededOrFailed(); a +# caller that must not publish in some states sets +# it, and a fork is expected to carry it through to +# whatever output shape it emits. parameters: - name: name type: string @@ -46,7 +54,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just index 286b2760f..64c426d30 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -196,7 +196,10 @@ _anvil-bench-history-bless store blessings: $tmpDir = $env:RUNNER_TEMP if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" & cargo bench-history list blessings --all --local="$store" ` --since 1970-01-01 --no-text --json $listJson @key diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index cc0ed8ac5..e47277ca0 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -872,3 +872,255 @@ fn bench_history_bless_rejects_malformed_entries() { assert_failed(&output, label); } } + +// --------------------------------------------------------------------------- +// Benchmark-history restore blocks. +// +// These carry the most control flow in the benchmark wiring, and the invariant +// they exist for -- an operational failure must never be mistaken for "no +// history yet" -- is invisible to a `contains` assertion on the emitted YAML. +// Both are therefore extracted from their template and executed against mocked +// transports. +// --------------------------------------------------------------------------- + +const SCHEDULED_IMPL: &str = include_str!("../templates/github/scheduled-impl-workflow.yml"); +const ADO_RESTORE: &str = include_str!("../templates/ado/steps/bench-history-restore.yml"); + +/// Extracts a block scalar (`run: |` / `pwsh: |`) from `yaml`, starting the +/// search at `after` and dedenting the body. +fn block_scalar(yaml: &str, after: &str, key: &str) -> String { + let start = yaml.find(after).unwrap_or_else(|| panic!("marker '{after}' not found")); + let rest = &yaml[start..]; + let key_offset = rest.find(key).unwrap_or_else(|| panic!("key '{key}' not found after '{after}'")); + let key_line_start = rest[..key_offset].rfind('\n').map_or(0, |index| index + 1); + let indent = key_offset - key_line_start; + + let body = &rest[key_offset + key.len()..]; + let mut lines = Vec::new(); + for line in body.lines().skip(1) { + let line_indent = line.len() - line.trim_start().len(); + if !line.trim().is_empty() && line_indent <= indent { + break; + } + lines.push(if line.len() > indent + 2 { &line[indent + 2..] } else { "" }); + } + lines.join("\n") +} + +fn git_bash() -> Option<&'static str> { + let candidate = r"C:\Program Files\Git\bin\bash.exe"; + Path::new(candidate).is_file().then_some(candidate) +} + +/// Runs the GitHub restore block with a stubbed `gh`. +/// +/// `runs` are the run ids the listing yields, newest first; `artifact_runs` +/// are those the artifacts API reports as carrying the artifact. +fn run_github_restore(runs: &str, artifact_runs: &str, download_exit: &str) -> (TempDir, Output, String, String) { + let bash = git_bash().expect("git bash checked by caller"); + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + + write( + &root.join("bin/gh"), + r#"#!/usr/bin/env bash +if [ "$1" = "run" ] && [ "$2" = "list" ]; then + for id in $FAKE_GH_RUNS; do echo "$id"; done + exit 0 +fi +if [ "$1" = "api" ]; then + for arg in "$@"; do + case "$arg" in + */actions/runs/*/artifacts) + rid="${arg##*/runs/}" + rid="${rid%%/artifacts}" + for id in $FAKE_GH_ARTIFACT_RUNS; do + if [ "$id" = "$rid" ]; then echo "artifact-$id"; exit 0; fi + done + ;; + esac + done + exit 0 +fi +if [ "$1" = "run" ] && [ "$2" = "download" ]; then + exit "${FAKE_GH_DOWNLOAD_EXIT:-0}" +fi +exit 0 +"#, + ); + + let script = block_scalar(SCHEDULED_IMPL, "- name: Restore benchmark history", "run: |"); + write(&root.join("restore.sh"), &script); + + let github_env = root.join("env.txt"); + let summary = root.join("summary.md"); + write(&github_env, ""); + write(&summary, ""); + + let output = Command::new(bash) + .arg("restore.sh") + .current_dir(root) + .env("PATH", format!("{}:/usr/bin:/bin", root.join("bin").display()).replace('\\', "/")) + .env("FAKE_GH_RUNS", runs) + .env("FAKE_GH_ARTIFACT_RUNS", artifact_runs) + .env("FAKE_GH_DOWNLOAD_EXIT", download_exit) + .env("ARTIFACT", "bench-history-linux") + .env("DEFAULT_BRANCH", "main") + .env("WORKFLOW", "anvil-scheduled") + .env("REPO", "owner/repo") + .env("WINDOW", "30") + .env("GITHUB_ENV", &github_env) + .env("GITHUB_STEP_SUMMARY", &summary) + .output() + .expect("bash runs the extracted restore block"); + + let env_text = std::fs::read_to_string(&github_env).unwrap_or_default(); + let summary_text = std::fs::read_to_string(&summary).unwrap_or_default(); + (tmp, output, env_text, summary_text) +} + +#[test] +fn github_restore_separates_absence_from_failure() { + if git_bash().is_none() { + eprintln!("skipping: git bash not installed"); + return; + } + + // (1) The newest run has no artifact; an older one does. The walk must + // reach it rather than cold-starting on the first miss. + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0"); + assert!( + output.status.success(), + "walking back should succeed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(env.contains("ANVIL_BENCH_RESTORE=restored"), "env:\n{env}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains("run 10"), + "should name the run it restored from:\n{}", + String::from_utf8_lossy(&output.stdout) + ); + + // (2) No run in the window carries it: a genuine cold start, and it must + // be visible on the summary rather than only in the log. + let (_tmp, output, env, summary) = run_github_restore("30 20 10", "", "0"); + assert!(output.status.success()); + assert!(env.contains("ANVIL_BENCH_RESTORE=cold-start"), "env:\n{env}"); + assert!(summary.contains("cold start"), "summary:\n{summary}"); + + // (3) The artifact exists but the download fails. This is the branch whose + // silent reintroduction re-creates the history-loss bug: it must fail and + // leave no publishable restore state. + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "30", "1"); + assert_failed(&output, "a failing download"); + assert!( + !env.contains("ANVIL_BENCH_RESTORE"), + "a failed restore must not mark a publishable state:\n{env}" + ); +} + +/// Runs the ADO restore block with mocked REST/download/extract cmdlets. +/// +/// `artifact_builds` are the build ids whose artifact query succeeds; every +/// other build answers 404 (absence). `failure` injects an operational fault: +/// "query" (non-404 status), "download", or "extract". +fn run_ado_restore(builds: &str, artifact_builds: &str, failure: &str) -> (TempDir, Output, String) { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + + let body = block_scalar(ADO_RESTORE, "steps:", "pwsh: |") + .replace("${{ parameters.artifact }}", "bench-history-linux") + .replace("${{ parameters.path }}", "store") + .replace("${{ parameters.window }}", "30"); + + // Function definitions shadow cmdlets of the same name, so the block runs + // unchanged against these stand-ins. + let prelude = r#" +# An exception whose Response.StatusCode.value__ the block can read, which is +# how it tells absence (404) from an operational failure (anything else). +class FakeHttpException : System.Exception { + [object]$Response + FakeHttpException([int]$status) : base("http $status") { + $this.Response = [pscustomobject]@{ StatusCode = [pscustomobject]@{ value__ = $status } } + } +} +function Invoke-RestMethod { + param([string]$Uri, $Headers) + if ($Uri -notlike '*/artifacts*') { + $ids = $env:FAKE_ADO_BUILDS -split ' ' | Where-Object { $_ } + return [pscustomobject]@{ value = @($ids | ForEach-Object { [pscustomobject]@{ id = $_ } }) } + } + $buildId = ($Uri -replace '.*/builds/', '') -replace '/artifacts.*', '' + $carries = ($env:FAKE_ADO_ARTIFACT_BUILDS -split ' ') -contains $buildId + if (-not $carries) { throw [FakeHttpException]::new(404) } + if ($env:FAKE_ADO_FAILURE -eq 'query') { throw [FakeHttpException]::new(500) } + return [pscustomobject]@{ resource = [pscustomobject]@{ downloadUrl = "https://example/$buildId" } } +} +function Invoke-WebRequest { + param([string]$Uri, $Headers, [string]$OutFile) + if ($env:FAKE_ADO_FAILURE -eq 'download') { throw 'download failed' } + Set-Content -LiteralPath $OutFile -Value 'zip' +} +function Expand-Archive { + param([string]$LiteralPath, [string]$DestinationPath, [switch]$Force) + if ($env:FAKE_ADO_FAILURE -eq 'extract') { throw 'corrupt archive' } + New-Item -ItemType Directory -Force -Path $DestinationPath | Out-Null + Set-Content -LiteralPath (Join-Path $DestinationPath 'run.json') -Value '{}' +} +"#; + + write(&root.join("restore.ps1"), &format!("{prelude}\n{body}")); + + let output = Command::new("pwsh") + .args(["-NoProfile", "-File", "restore.ps1"]) + .current_dir(root) + .env("FAKE_ADO_BUILDS", builds) + .env("FAKE_ADO_ARTIFACT_BUILDS", artifact_builds) + .env("FAKE_ADO_FAILURE", failure) + .env("SYSTEM_ACCESSTOKEN", "token") + .env("SYSTEM_COLLECTIONURI", "https://example/") + .env("SYSTEM_TEAMPROJECTID", "project") + .env("SYSTEM_DEFINITIONID", "7") + .env("BUILD_SOURCEBRANCH", "refs/heads/main") + .env("BUILD_BUILDID", "999") + .output() + .expect("pwsh runs the extracted restore block"); + + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + (tmp, output, stdout) +} + +#[test] +fn ado_restore_separates_absence_from_failure() { + if Command::new("pwsh").arg("--version").output().is_err() { + eprintln!("skipping: pwsh not installed"); + return; + } + + // A 404 on the newest build walks back to an older one that has it. + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "10", ""); + assert!( + output.status.success(), + "walking back should succeed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(stdout.contains("restoring"), "stdout:\n{stdout}"); + assert!(stdout.contains("ANVIL_BENCH_RESTORE]restored"), "stdout:\n{stdout}"); + + // Nothing in the window carries it: a genuine cold start. + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "", ""); + assert!(output.status.success()); + assert!(stdout.contains("ANVIL_BENCH_RESTORE]cold-start"), "stdout:\n{stdout}"); + + // Every operational fault must fail without marking a publishable state, + // which is what the guarded publish depends on to avoid overwriting a + // good chain with a truncated store. + for failure in ["query", "download", "extract"] { + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "30", failure); + assert_failed(&output, failure); + assert!( + !stdout.contains("ANVIL_BENCH_RESTORE]"), + "{failure} must not set a restore state:\n{stdout}" + ); + } +} diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index a72dfd08d..1fc03a6a4 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -1562,6 +1562,13 @@ parameters: - name: windowsPool type: object default: { vmImage: windows-latest } + - name: benchMachineKey + type: string + # Machine key the benchmark history is partitioned by. Empty uses + # cargo-bench-history's hardware fingerprint; set a stable pool label + # when the agent pool is heterogeneous enough to fragment a series into + # partitions too sparse to analyze. + default: '' stages: - stage: scheduled_test @@ -1659,14 +1666,22 @@ stages: - stage: scheduled_benchmarks displayName: anvil scheduled-benchmarks dependsOn: [] + variables: + # Read by the bench-history recipe; empty means "use the hardware + # fingerprint". ADO exports pipeline variables as environment + # variables, so this reaches the recipe without further plumbing. + ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} jobs: # Benchmark regression detection. OS scope matches # scheduled-exhaustive. The history is partitioned per machine, so # each leg carries its own artifact rather than sharing one name. - # The restore step runs first; the wrapper's `artifacts` contract - # publishes the updated store at the end of the job -- including - # when the analysis failed the job, so the samples collected while - # the pipeline is red are not lost. + # + # The publish goes through the wrapper's `artifacts` contract like + # every other job, carrying a `condition`: the store is only a valid + # continuation of the chain once the restore reached a known state, + # so an operational restore failure must not overwrite good history + # with a truncated snapshot. Failed runs still publish -- a flagged + # regression fails the job, and those samples belong in the history. # # The explicit `checkout` leads the step list rather than going # through a wrapper parameter: `steps/job.yml` is the file adopters @@ -1678,6 +1693,10 @@ stages: parameters: name: linux pool: ${{ parameters.linuxPool }} + artifacts: + - name: bench-history-linux + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: # The analysis orders each series by first-parent commit # topology and locates the merge-base, so it needs the whole @@ -1691,13 +1710,14 @@ stages: artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml - - template: steps/bench-history-publish.yml - parameters: - artifact: bench-history-linux - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} + artifacts: + - name: bench-history-windows + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - checkout: self fetchDepth: 0 @@ -1707,9 +1727,6 @@ stages: artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - template: steps/bench-history-summary.yml - - template: steps/bench-history-publish.yml - parameters: - artifact: bench-history-windows === .pipelines/anvil/steps/advisory-comments.yml === # Copyright (c) Microsoft Corporation. @@ -1809,39 +1826,6 @@ steps: } } -=== .pipelines/anvil/steps/bench-history-publish.yml === -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md -# -# Publishes the updated benchmark history as this leg's artifact. -# -# This does not go through the §4.1 job wrapper's `artifacts` contract -# because it needs a condition the contract does not express: the store is -# only a valid continuation of the chain when the restore reached a known -# state. Publishing after an operational restore failure would overwrite a -# good history with a truncated snapshot. -# -# The condition still includes failed runs: a flagged regression fails the -# job, and those samples belong in the history. -# -# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md -parameters: - - name: artifact - type: string - - name: path - type: string - default: target/anvil/bench-history -steps: - - task: PublishPipelineArtifact@1 - displayName: Publish ${{ parameters.artifact }} - condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) - inputs: - targetPath: ${{ parameters.path }} - artifact: ${{ parameters.artifact }} - === .pipelines/anvil/steps/bench-history-restore.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -2086,15 +2070,23 @@ steps: # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# A step list may lead with its own `checkout: self` +# (the benchmark group does, for depth and LFS), so a +# wrapper must NOT add a checkout of its own -- doing +# so would give those jobs two. # - artifacts (object) Optional list of pipeline artifacts to publish. -# Each item: { name: string, path: string }. -# The default wrapper appends one -# PublishPipelineArtifact@1 task per entry; 1ESPT -# wrappers translate these into +# Each item: { name: string, path: string, +# condition: string (optional) }. The default wrapper +# appends one PublishPipelineArtifact@1 task per +# entry; 1ESPT wrappers translate these into # templateContext.outputs.pipelineArtifact blocks # at job level. The contract is the same either way, # so the stages templates don't need to know which # backend they're targeting. +# `condition` defaults to succeededOrFailed(); a +# caller that must not publish in some states sets +# it, and a fork is expected to carry it through to +# whatever output shape it emits. parameters: - name: name type: string @@ -2115,7 +2107,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} @@ -3232,7 +3227,10 @@ _anvil-bench-history-bless store blessings: $tmpDir = $env:RUNNER_TEMP if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" & cargo bench-history list blessings --all --local="$store" ` --since 1970-01-01 --no-text --json $listJson @key diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index cfeb245a5..7935174b8 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -3081,7 +3081,10 @@ _anvil-bench-history-bless store blessings: $tmpDir = $env:RUNNER_TEMP if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" & cargo bench-history list blessings --all --local="$store" ` --since 1970-01-01 --no-text --json $listJson @key diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index eef3cebdb..a67758c54 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -1702,7 +1702,10 @@ _anvil-bench-history-bless store blessings: $tmpDir = $env:RUNNER_TEMP if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" & cargo bench-history list blessings --all --local="$store" ` --since 1970-01-01 --no-text --json $listJson @key diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just index 286b2760f..64c426d30 100644 --- a/justfiles/anvil/checks/bench-history.just +++ b/justfiles/anvil/checks/bench-history.just @@ -196,7 +196,10 @@ _anvil-bench-history-bless store blessings: $tmpDir = $env:RUNNER_TEMP if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - $listJson = Join-Path $tmpDir 'anvil-bench-blessings.json' + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" & cargo bench-history list blessings --all --local="$store" ` --since 1970-01-01 --no-text --json $listJson @key From 751d1f7dda3a9c197846d8f868f1173cb1f5055d Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Fri, 21 Aug 2026 16:01:15 +0200 Subject: [PATCH 14/24] fix(cargo-anvil): sync the dogfooded scheduled workflows with the catalog Resolving the merge with --theirs on the emitted workflows took main's copies, which lack the `actions: read` grant the benchmark history restore needs. anvil noticed and proposed the correct content; this accepts those proposals and drops the .anvil-proposed siblings, which are transient state and not meant to be committed. Without this the repo's own scheduled run would fail the restore on a permissions error while the catalog claimed otherwise. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .github/workflows/anvil-scheduled-impl.yml | 120 +++++++ .../anvil-scheduled-impl.yml.anvil-proposed | 318 ------------------ .github/workflows/anvil-scheduled.yml | 3 + .../anvil-scheduled.yml.anvil-proposed | 28 -- 4 files changed, 123 insertions(+), 346 deletions(-) delete mode 100644 .github/workflows/anvil-scheduled-impl.yml.anvil-proposed delete mode 100644 .github/workflows/anvil-scheduled.yml.anvil-proposed diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 0e23c5719..19b686860 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -24,6 +24,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -129,6 +137,117 @@ jobs: lfs: true - uses: ./.github/actions/anvil-scheduled-exhaustive + scheduled-benchmarks: + # Benchmark regression detection. x86_64-only, matching + # scheduled-exhaustive. The history is partitioned per machine, so + # each leg carries its own artifact rather than sharing one name. + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history reads the Actions runs/artifacts API. + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit + # topology and locates the merge-base, so it needs the whole + # commit graph. LFS matters because benchmark inputs can be + # LFS-tracked and would otherwise arrive as pointer files. + fetch-depth: 0 + lfs: true + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: bench-history-${{ matrix.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename + # must not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + + # Walk back from the newest run and take the first one that + # carries this leg's artifact. Restoring from the latest + # *successful* run would drop every sample collected while the + # pipeline was red from a regression — precisely the window + # that matters. + # + # Absence and failure are kept distinct. A run is only a + # candidate once the artifacts API confirms the artifact exists + # and has not expired; a download that then fails is an + # operational error (token, API, corrupt payload) and fails the + # job rather than being silently downgraded to a cold start. + # That distinction is what stops one transient failure from + # publishing an empty store over a good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId'); do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 + done + + # No run in the window carried the artifact. That is a genuine + # cold start (first run, or the chain lapsed), so it is surfaced + # on the summary rather than only in this log — "history quietly + # restarted" must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - uses: ./.github/actions/anvil-scheduled-benchmarks + env: + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} + - name: Save benchmark history + # always(): the run's own samples belong in the history even when + # the analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the + # restore failed operationally the store is not a continuation of + # the chain, and publishing it would overwrite good history with a + # truncated snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ matrix.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused + # or infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi publish-failure: name: Publish scheduled failure needs: @@ -136,6 +255,7 @@ jobs: - scheduled-advisories - scheduled-runtime-analysis - scheduled-exhaustive + - scheduled-benchmarks if: ${{ always() && vars.ANVIL_PUBLISH_FAILURE_ISSUE != 'false' && contains(needs.*.result, 'failure') }} runs-on: ${{ inputs.linux_runner }} diff --git a/.github/workflows/anvil-scheduled-impl.yml.anvil-proposed b/.github/workflows/anvil-scheduled-impl.yml.anvil-proposed deleted file mode 100644 index 19b686860..000000000 --- a/.github/workflows/anvil-scheduled-impl.yml.anvil-proposed +++ /dev/null @@ -1,318 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md -name: anvil-scheduled-impl - -on: - workflow_call: - inputs: - linux_runner: - description: Runner label for x86_64 Linux jobs. - type: string - default: ubuntu-latest - windows_runner: - description: Runner label for x86_64 Windows jobs. - type: string - default: windows-latest - linux_arm_runner: - description: Runner label for aarch64 Linux jobs. - type: string - default: ubuntu-24.04-arm - windows_arm_runner: - description: Runner label for aarch64 Windows jobs. - type: string - default: windows-11-arm - bench_machine_key: - description: | - Machine key the benchmark history is partitioned by. Leave empty to - use cargo-bench-history's hardware fingerprint. Set a stable pool - label when the runner pool is heterogeneous enough to fragment a - series into partitions too sparse to analyze. - type: string - default: "" - secrets: - CODECOV_TOKEN: - description: | - Codecov upload token. Optional for public repos that have OIDC - configured at Codecov; required for private repos. - required: false - -# The caller grants the maximum token scopes available to this reusable -# workflow. Reset jobs to read-only here, then restore only the publisher's -# issues scope below. See docs/design/github.md §9. -permissions: - contents: read - -# Note on matrices: see pr-impl-workflow.yml for the rationale. OS -# matrices are hardcoded; per-leg runner labels are inputs. - -jobs: - scheduled-test: - strategy: - fail-fast: false - matrix: - os: [linux, windows, linux-arm, windows-arm] - runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner - || matrix.os == 'windows' && inputs.windows_runner - || matrix.os == 'linux-arm' && inputs.linux_arm_runner - || inputs.windows_arm_runner }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - lfs: true - - uses: ./.github/actions/anvil-scheduled-test - with: - free-disk-space: true - - name: Upload coverage to Codecov - # Upload from every leg except windows-11-arm (see the matching - # comment in pr-impl-workflow.yml for the rationale). - # Multi-flag tag combines the OS with a "scheduled" marker so - # the Codecov UI can distinguish PR-tier uploads from scheduled - # uploads while still tracking each platform separately. - if: matrix.os != 'windows-arm' - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 - with: - files: target/coverage/lcov-all-features.info,target/coverage/lcov-no-default.info - flags: scheduled,${{ matrix.os }} - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - - scheduled-advisories: - # Cross-OS / cross-arch because clippy and udeps in this group - # compile per host, so cfg-gated code must be linted/scanned on - # every leg. - strategy: - fail-fast: false - matrix: - os: [linux, windows, linux-arm, windows-arm] - runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner - || matrix.os == 'windows' && inputs.windows_runner - || matrix.os == 'linux-arm' && inputs.linux_arm_runner - || inputs.windows_arm_runner }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - lfs: true - - uses: ./.github/actions/anvil-scheduled-advisories - - scheduled-runtime-analysis: - # Full-workspace counterpart of pr-runtime-analysis. OS matrix - # mirrors pr-runtime-analysis so any OS already considered "worth - # running miri on" gets the stricter profiles (tree-borrows, - # strict-provenance, race-coverage) too. One job per OS leg runs - # `just anvil-scheduled-runtime-analysis`, which executes all four - # miri profiles (base miri + the three stricter ones) sequentially - # within the leg. Parallelism is across OS legs, not across profiles: - # running the profiles sequentially lets them share toolchain setup - # and the target/ cache, which is the right trade given each profile - # already costs hours. - strategy: - fail-fast: false - matrix: - os: [linux, windows, linux-arm, windows-arm] - runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner - || matrix.os == 'windows' && inputs.windows_runner - || matrix.os == 'linux-arm' && inputs.linux_arm_runner - || inputs.windows_arm_runner }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - lfs: true - - uses: ./.github/actions/anvil-scheduled-runtime-analysis - - scheduled-exhaustive: - # x86_64-only by design: this group includes mutants-full (which - # doesn't build on aarch64-pc-windows-msvc — winapi crate - # incompatibility) plus cargo-hack feature powerset and bench. - strategy: - fail-fast: false - matrix: - os: [linux, windows] - runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - lfs: true - - uses: ./.github/actions/anvil-scheduled-exhaustive - - scheduled-benchmarks: - # Benchmark regression detection. x86_64-only, matching - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. - strategy: - fail-fast: false - matrix: - os: [linux, windows] - runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} - permissions: - contents: read - # Restoring the history reads the Actions runs/artifacts API. - actions: read - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. - fetch-depth: 0 - lfs: true - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: bench-history-${{ matrix.os }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - mkdir -p target/anvil/bench-history - - # Walk back from the newest run and take the first one that - # carries this leg's artifact. Restoring from the latest - # *successful* run would drop every sample collected while the - # pipeline was red from a regression — precisely the window - # that matters. - # - # Absence and failure are kept distinct. A run is only a - # candidate once the artifacts API confirms the artifact exists - # and has not expired; a download that then fails is an - # operational error (token, API, corrupt payload) and fails the - # job rather than being silently downgraded to a cold start. - # That distinction is what stops one transient failure from - # publishing an empty store over a good history. - for run_id in $(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId'); do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" - exit 0 - done - - # No run in the window carried the artifact. That is a genuine - # cold start (first run, or the chain lapsed), so it is surfaced - # on the summary rather than only in this log — "history quietly - # restarted" must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - - uses: ./.github/actions/anvil-scheduled-benchmarks - env: - ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - - name: Save benchmark history - # always(): the run's own samples belong in the history even when - # the analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the - # restore failed operationally the store is not a continuation of - # the chain, and publishing it would overwrite good history with a - # truncated snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused - # or infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi - publish-failure: - name: Publish scheduled failure - needs: - - scheduled-test - - scheduled-advisories - - scheduled-runtime-analysis - - scheduled-exhaustive - - scheduled-benchmarks - if: ${{ always() && vars.ANVIL_PUBLISH_FAILURE_ISSUE != 'false' - && contains(needs.*.result, 'failure') }} - runs-on: ${{ inputs.linux_runner }} - permissions: - issues: write - steps: - - name: Create or update failure issue - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 - env: - ANVIL_JOB_RESULTS: ${{ toJSON(needs) }} - with: - script: | - const title = "[Anvil] Scheduled checks failed"; - const marker = ""; - const runUrl = - `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` + - `/actions/runs/${context.runId}`; - const results = JSON.parse(process.env.ANVIL_JOB_RESULTS); - const failedJobs = Object.entries(results) - .filter(([, job]) => job.result === "failure") - .map(([job]) => `- \`${job}\``) - .join("\n"); - const body = [ - marker, - "", - "The Anvil scheduled workflow failed.", - "", - "Failed jobs:", - failedJobs, - "", - `[View workflow run](${runUrl})`, - ].join("\n"); - - const query = - `repo:${context.repo.owner}/${context.repo.repo} ` + - `is:issue is:open in:body "anvil scheduled failure"`; - const { data: search } = - await github.rest.search.issuesAndPullRequests({ - q: query, - per_page: 100, - }); - const existing = search.items.find( - issue => issue.body?.includes(marker), - ); - - if (existing) { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: existing.number, - body, - }); - } else { - await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title, - body, - }); - } diff --git a/.github/workflows/anvil-scheduled.yml b/.github/workflows/anvil-scheduled.yml index 4ecb2a601..ebc8843f3 100644 --- a/.github/workflows/anvil-scheduled.yml +++ b/.github/workflows/anvil-scheduled.yml @@ -21,5 +21,8 @@ jobs: # publish-failure. See docs/design/github.md §9. permissions: contents: read + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. Narrowed to that job inside. + actions: read issues: write secrets: inherit diff --git a/.github/workflows/anvil-scheduled.yml.anvil-proposed b/.github/workflows/anvil-scheduled.yml.anvil-proposed deleted file mode 100644 index ebc8843f3..000000000 --- a/.github/workflows/anvil-scheduled.yml.anvil-proposed +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md -name: anvil-scheduled - -on: - schedule: - - cron: "0 7 * * *" - workflow_dispatch: {} - -permissions: - contents: read - -jobs: - anvil-scheduled: - uses: ./.github/workflows/anvil-scheduled-impl.yml - # A called workflow cannot elevate beyond its caller. The implementation - # resets this upper bound to read-only and restores issues:write only on - # publish-failure. See docs/design/github.md §9. - permissions: - contents: read - # The scheduled-benchmarks job restores its history artifact, which - # reads the Actions runs/artifacts API. Narrowed to that job inside. - actions: read - issues: write - secrets: inherit From edd5fae0a09f6a42367979dbff9f979c53e33f5f Mon Sep 17 00:00:00 2001 From: Martin Kolinek Date: Fri, 21 Aug 2026 16:19:24 +0200 Subject: [PATCH 15/24] refactor(cargo-anvil): keep per-group steps out of the tier templates The scheduled stages template and workflow had accumulated per-group content: the benchmark group's checkout, history restore and summary, and scheduled-test's coverage publication. That made the files that should read as "the list of groups in this tier" carry the details of individual groups. Both generators now splice a group's extra steps into that group's own emitted artifact -- its ADO step template, its GitHub composite action -- at the same point they substitute the group name. A group with no extras is byte-identical to before. Adding steps to a group, or moving a check between groups, no longer touches the tier templates. Every scheduled stage is now the same three lines, and the benchmark job on GitHub is a checkout plus its group action like every other job. The one exception is deliberate: `artifacts` stays at the call site because publishing is a job-level output, and a forked (1ESPT) wrapper translates that list into its own output shape -- a publish task inside the step list would bypass the translation. Both legs now share one declaration, keyed on the agent OS rather than spelled per leg. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 6 +- .../anvil-scheduled-benchmarks/action.yml | 87 +++++++++ .github/workflows/anvil-scheduled-impl.yml | 97 +-------- crates/cargo-anvil/docs/design/ado.md | 34 ++-- crates/cargo-anvil/docs/design/github.md | 15 +- crates/cargo-anvil/src/anvil/artifacts/ado.rs | 104 +++++++--- .../cargo-anvil/src/anvil/artifacts/github.rs | 59 ++++-- .../templates/ado/scheduled-stages.yml | 62 ++---- .../cargo-anvil/templates/ado/steps/group.yml | 5 + .../github/bench-history-restore.yml | 62 ++++++ .../templates/github/bench-history-save.yml | 25 +++ .../templates/github/group-action.yml | 2 + .../github/scheduled-impl-workflow.yml | 97 +-------- crates/cargo-anvil/tests/recipe_contracts.rs | 4 +- .../snapshots/snapshots__ado_backend.snap | 102 +++++----- .../snapshots/snapshots__github_backend.snap | 184 +++++++++--------- 16 files changed, 509 insertions(+), 436 deletions(-) create mode 100644 crates/cargo-anvil/templates/github/bench-history-restore.yml create mode 100644 crates/cargo-anvil/templates/github/bench-history-save.yml diff --git a/.anvil.lock b/.anvil.lock index 720e3e333..3a7136665 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.4.0" -catalog_checksum = "sha256:6fc85e2296671ad38ac46de65a512904ba06f0e1f33c3687cd511e05a01ce0af" +catalog_checksum = "sha256:5cafd37e7ad6aebffbed60e42052fc6ee16a7528a7c66664d08f10c7b62309cc" [[file]] path = ".anvil/container/Containerfile" @@ -61,7 +61,7 @@ checksum = "sha256:aae1d9e983c289d124e217ac6412c0421e4f53064441cef60e20e19d7c737 [[file]] path = ".github/actions/anvil-scheduled-benchmarks/action.yml" -checksum = "sha256:6e9b213ba8be25178638db734dd7d5df7b7774cb9d2a20b79bb0dd2aa8d7f57b" +checksum = "sha256:7b5914afe3f042601c429bd97016766d9c1a664676adcf0eb715ed22716d1d30" [[file]] path = ".github/actions/anvil-scheduled-exhaustive/action.yml" @@ -89,7 +89,7 @@ checksum = "sha256:18350505aedb0d3e4bc0941016205acbd17b5619b83caa24689fd9d2ddcc1 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:a3ceba28e8a03bff59441ee7eb6fd1c5ad8d4e1e2d4a17a0bf07177f5f9c20a0" +checksum = "sha256:0827666ae955790776302429199b9ec6910f88dd73e493c54d3c38f80b17559e" [[file]] path = ".github/workflows/anvil-scheduled.yml" diff --git a/.github/actions/anvil-scheduled-benchmarks/action.yml b/.github/actions/anvil-scheduled-benchmarks/action.yml index 01d76fdfc..e50acc61a 100644 --- a/.github/actions/anvil-scheduled-benchmarks/action.yml +++ b/.github/actions/anvil-scheduled-benchmarks/action.yml @@ -37,6 +37,68 @@ inputs: runs: using: composite steps: + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Per-leg: the history is partitioned by machine, and one run cannot + # upload the same artifact name twice. + ARTIFACT: bench-history-${{ runner.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression — precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. That distinction is what + # stops one transient failure from publishing an empty store over a + # good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId'); do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 + done + + # No run in the window carried the artifact. That is a genuine cold + # start (first run, or the chain lapsed), so it is surfaced on the + # summary rather than only in this log — "history quietly restarted" + # must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-setup with: group: scheduled-benchmarks @@ -53,3 +115,28 @@ runs: # checks never read it. GITHUB_TOKEN: ${{ github.token }} run: just anvil-scheduled-benchmarks + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ runner.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 19b686860..deea2f30a 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -139,8 +139,8 @@ jobs: scheduled-benchmarks: # Benchmark regression detection. x86_64-only, matching - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. + # scheduled-exhaustive. The history round-trip lives in the group's + # composite action; only the job-level concerns are here. strategy: fail-fast: false matrix: @@ -153,101 +153,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. fetch-depth: 0 lfs: true - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: bench-history-${{ matrix.os }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - mkdir -p target/anvil/bench-history - - # Walk back from the newest run and take the first one that - # carries this leg's artifact. Restoring from the latest - # *successful* run would drop every sample collected while the - # pipeline was red from a regression — precisely the window - # that matters. - # - # Absence and failure are kept distinct. A run is only a - # candidate once the artifacts API confirms the artifact exists - # and has not expired; a download that then fails is an - # operational error (token, API, corrupt payload) and fails the - # job rather than being silently downgraded to a cold start. - # That distinction is what stops one transient failure from - # publishing an empty store over a good history. - for run_id in $(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId'); do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" - exit 0 - done - - # No run in the window carried the artifact. That is a genuine - # cold start (first run, or the chain lapsed), so it is surfaced - # on the summary rather than only in this log — "history quietly - # restarted" must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - - name: Save benchmark history - # always(): the run's own samples belong in the history even when - # the analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the - # restore failed operationally the store is not a continuation of - # the chain, and publishing it would overwrite good history with a - # truncated snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused - # or infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + publish-failure: name: Publish scheduled failure needs: diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index f4735dcd7..f55dd59bc 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -460,6 +460,19 @@ keyword, which for parameters defined at the call site is the stages template itself — so the path is written relative to `pr.yml` / `scheduled.yml`, *not* relative to `steps/job.yml`. +**Per-group steps.** Some groups need steps around the uniform runner — the +benchmark group checks out at full depth and round-trips its history store, +`scheduled-test` publishes coverage. Those are spliced into the group's own +emitted step template at generation time, not written at the call site, so the +stages templates stay a plain list of groups. Moving a check between groups, or +giving a group extra steps, never edits `pr.yml` / `scheduled.yml`. + +The one thing that cannot move is a **job-level output**: `artifacts` is declared +where the job is constructed, because a forked wrapper translates that list into +its own output shape and a publish task inside the step list would bypass the +translation. That declaration is therefore the only per-group content the stages +templates carry. + ### 4.2 Stages template shape Approximate shape (anvil writes this verbatim; users normally don't edit it): @@ -842,22 +855,21 @@ own artifact (`bench-history-`). Each scheduled benchmark job: -1. checks out with full history and LFS via an explicit `checkout` step at the head - of its own step list (analysis reads the commit graph; benchmark inputs may be - LFS-tracked); -2. **restores** the history with `steps/bench-history-restore.yml`, which walks the - pipeline's own builds on the branch newest-first +1. checks out with full history and LFS, and restores, applies blessings, runs + the analysis and attaches findings — all inside `steps/scheduled-benchmarks.yml`, + the group's own emitted step template (§4.1), so the stages template stays a + plain list of groups; +2. the restore walks the pipeline's own builds on the branch newest-first (`_apis/build/builds?…&queryOrder=finishTimeDescending`) and, per build, queries the artifacts endpoint for this leg's artifact. A `404` means that build simply has no such artifact and the walk continues; any other status is an operational failure and fails the job. Finding none across the whole window is a genuine cold start; -3. applies any pending blessings, runs collect + analyze, writing findings to a - findings file which a following step attaches to the build summary; -4. **publishes** the updated store through the wrapper's `artifacts` parameter - (`{ name: bench-history-, path: , condition: … }`), which the default - wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a - `pipelineArtifact` output. +3. **publishes** the updated store through the wrapper's `artifacts` parameter + (`{ name: bench-history-$(Agent.OS), path: , condition: … }`), which the + default wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a + `pipelineArtifact` output. The name is derived from the agent OS so both legs + share one declaration. `DownloadPipelineArtifact@2` is not used for the restore: `latestFromBranch` resolves a single build and does not walk, so a cancelled or never-publishing latest build diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index c049f0b83..71a6d3734 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -16,7 +16,9 @@ need to change: These change when anvil's groups or impact wiring evolve; most users won't ever edit them. 3. **Per-group composite actions** (`.github/actions/anvil-*/`). Each is a multi-step - composite that runs setup + the matching `just anvil--` recipe. + composite that runs setup + the matching `just anvil--` recipe, plus + any steps that group needs around it (the benchmark group's history round-trip). + Keeping those in the group's own action leaves the workflows a plain list of jobs. See also: @@ -836,12 +838,17 @@ clearable when a check is removed from the catalog. The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs `cargo-bench-history`, whose history persists across scheduled runs as GitHub **Actions artifacts**. The history is partitioned per machine, so each leg of the -group's matrix carries its own artifact (`bench-history-`) — which also keeps -the names distinct within a run, as artifact upload requires. +group's matrix carries its own artifact (`bench-history-`). + +The round-trip lives in the group's **composite action**, not in the workflow: the +scheduled workflow's benchmark job is a checkout plus the group action, like every +other job. Only job-level concerns stay in the workflow — the matrix, the +`actions: read` grant, the full-depth checkout, and the machine-key input. Each scheduled benchmark job: -1. checks out with `fetch-depth: 0` (analysis reads the commit graph); +1. checks out with `fetch-depth: 0` and `lfs: true` (analysis reads the commit + graph; benchmark inputs may be LFS-tracked); 2. **restores** the history by walking back from the newest `anvil-scheduled` run on the default branch and taking the first that carries the leg's artifact; the first run finds none and starts empty; diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index 3cebf2232..7a17aac70 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -70,10 +70,63 @@ const GROUP_STEP_TEMPLATE: &str = include_str!("../../../templates/ado/steps/gro /// Placeholder token the per-group template uses for the group name. const GROUP_PLACEHOLDER: &str = "__GROUP__"; +/// Placeholder lines for steps a group needs around the uniform runner. +/// Substituted away entirely for groups that need none. +const PRE_STEPS_PLACEHOLDER: &str = "__PRE_STEPS__\n"; +const POST_STEPS_PLACEHOLDER: &str = "__POST_STEPS__\n"; + +/// Steps that run before the uniform group runner, per group. +/// +/// These live in the group's own emitted step template rather than at the +/// call site, so `pr.yml` / `scheduled.yml` stay a plain list of groups. A +/// group absent from the table gets nothing. +const GROUP_PRE_STEPS: &[(&str, &str)] = &[( + "scheduled-benchmarks", + // The analysis orders each series by first-parent commit topology and + // locates the merge-base, so it needs the whole commit graph. LFS + // matters because benchmark inputs can be LFS-tracked and would + // otherwise arrive as pointer files. + // + // The checkout is explicit rather than a wrapper parameter: job.yml is + // the file adopters fork, so binding a parameter their copy lacks would + // fail expansion for the whole pipeline. + " - checkout: self\n\ + \x20 fetchDepth: 0\n\ + \x20 lfs: true\n\ + \x20 - template: bench-history-restore.yml\n\ + \x20 parameters:\n\ + \x20 artifact: bench-history-$(Agent.OS)\n", +)]; + +/// Steps that run after the uniform group runner, per group. +const GROUP_POST_STEPS: &[(&str, &str)] = &[ + ( + "scheduled-test", + " - task: PublishCodeCoverageResults@2\n\ + \x20 condition: succeededOrFailed()\n\ + \x20 displayName: Publish coverage\n\ + \x20 inputs:\n\ + \x20 summaryFileLocation: target/coverage/cobertura-*.xml\n\ + \x20 failIfCoverageEmpty: false\n", + ), + ("scheduled-benchmarks", " - template: bench-history-summary.yml\n"), +]; + +/// The extra steps registered for `group`, or the empty string. +fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> &'static str { + table + .iter() + .find_map(|&(name, steps)| (name == group).then_some(steps)) + .unwrap_or("") +} + /// Render the step template for one group. #[must_use] fn render_group_step(group: &str) -> String { - GROUP_STEP_TEMPLATE.replace(GROUP_PLACEHOLDER, group) + GROUP_STEP_TEMPLATE + .replace(GROUP_PLACEHOLDER, group) + .replace(PRE_STEPS_PLACEHOLDER, extra_steps(GROUP_PRE_STEPS, group)) + .replace(POST_STEPS_PLACEHOLDER, extra_steps(GROUP_POST_STEPS, group)) } /// Repo-root-relative path for one group's step template. @@ -384,7 +437,9 @@ mod tests { ] { assert!(SCHEDULED_STAGES.contains(needle), "scheduled stages missing '{needle}'"); } - assert!(SCHEDULED_STAGES.contains("PublishCodeCoverageResults@2")); + // Coverage publication lives in the group's own step template now; + // the stages file is a plain list of groups. + assert!(render_group_step("scheduled-test").contains("PublishCodeCoverageResults@2")); assert!(SCHEDULED_STAGES.contains("- template: steps/job.yml")); assert!( !SCHEDULED_STAGES.contains("\n - job: "), @@ -394,35 +449,30 @@ mod tests { #[test] fn scheduled_benchmarks_stage_round_trips_the_history_artifact() { - // The checkout leads the group's own step list rather than going - // through a wrapper parameter: job.yml is the file adopters fork, - // so binding a parameter their copy lacks would fail expansion for - // the whole pipeline. + // The stage is a plain list of groups; the round-trip lives in the + // group's own step template. + let group_step = render_group_step("scheduled-benchmarks"); assert!( !JOB_WRAPPER.contains("fetchDepth"), - "the job wrapper contract must stay frozen; put checkout in the group's step list" + "the job wrapper contract must stay frozen; put checkout in the group's step template" ); - assert_eq!( - SCHEDULED_STAGES.matches("- checkout: self").count(), - 2, - "both benchmark legs check out explicitly" - ); - assert_eq!(SCHEDULED_STAGES.matches("fetchDepth: 0").count(), 2); + assert!(group_step.contains("- checkout: self")); + assert!(group_step.contains("fetchDepth: 0")); // Benchmark inputs can be LFS-tracked. - assert_eq!(SCHEDULED_STAGES.matches("lfs: true").count(), 2); - // Per-leg artifact names: the history is partitioned per machine. - for needle in ["bench-history-linux", "bench-history-windows"] { - assert_eq!( - SCHEDULED_STAGES.matches(needle).count(), - 2, - "the restore and publish sides must agree on the artifact name '{needle}'" - ); - } - assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-restore.yml")); - assert!(SCHEDULED_STAGES.contains("template: steps/bench-history-summary.yml")); - // The publish goes through the wrapper's `artifacts` contract rather - // than emitting its own task, so a forked wrapper still performs the - // translation it exists for. The guard rides along as a `condition`. + assert!(group_step.contains("lfs: true")); + assert!(group_step.contains("template: bench-history-restore.yml")); + assert!(group_step.contains("template: bench-history-summary.yml")); + // A group with no registered extras gets none of this. + assert!(!render_group_step("scheduled-exhaustive").contains("bench-history")); + assert!(!render_group_step("scheduled-exhaustive").contains("checkout: self")); + // Coverage publication likewise moved off the call site. + assert!(render_group_step("scheduled-test").contains("PublishCodeCoverageResults@2")); + assert!( + !SCHEDULED_STAGES.contains("PublishCodeCoverageResults@2"), + "the stages template must not carry per-group steps" + ); + // The publish stays a job-level output so a forked (1ESPT) wrapper + // still translates it; the guard rides along as a `condition`. assert_eq!( SCHEDULED_STAGES .matches("condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], ''))") diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index 59d2089cb..883f26d6f 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -55,10 +55,40 @@ const GROUP_ACTION_TEMPLATE: &str = include_str!("../../../templates/github/grou /// Placeholder token the per-group template uses for the group name. const GROUP_PLACEHOLDER: &str = "__GROUP__"; +/// Placeholder lines for steps a group needs around the uniform runner. +const PRE_STEPS_PLACEHOLDER: &str = "__PRE_STEPS__\n"; +const POST_STEPS_PLACEHOLDER: &str = "__POST_STEPS__\n"; + +/// Steps that run before the uniform group runner, per group. +/// +/// These live in the group's own composite action rather than in the +/// scheduled workflow, whose jobs stay a plain list of groups. +const GROUP_PRE_STEPS: &[(&str, &str)] = &[( + "scheduled-benchmarks", + include_str!("../../../templates/github/bench-history-restore.yml"), +)]; + +/// Steps that run after the uniform group runner, per group. +const GROUP_POST_STEPS: &[(&str, &str)] = &[( + "scheduled-benchmarks", + include_str!("../../../templates/github/bench-history-save.yml"), +)]; + +/// The extra steps registered for `group`, or the empty string. +fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> &'static str { + table + .iter() + .find_map(|&(name, steps)| (name == group).then_some(steps)) + .unwrap_or("") +} + /// Render the `action.yml` for one check group's composite action. #[must_use] fn render_group_action(group: &str) -> String { - GROUP_ACTION_TEMPLATE.replace(GROUP_PLACEHOLDER, group) + GROUP_ACTION_TEMPLATE + .replace(GROUP_PLACEHOLDER, group) + .replace(PRE_STEPS_PLACEHOLDER, extra_steps(GROUP_PRE_STEPS, group)) + .replace(POST_STEPS_PLACEHOLDER, extra_steps(GROUP_POST_STEPS, group)) } /// Repo-root-relative path for a per-group composite action. @@ -301,35 +331,38 @@ mod tests { #[test] fn scheduled_benchmarks_job_round_trips_the_history_artifact() { - // Analysis walks the commit graph, so the leg needs full history; - // benchmark inputs can be LFS-tracked. + // The job is a plain checkout + group action like every other one; + // the round-trip lives in the group's composite action. + let group_action = render_group_action("scheduled-benchmarks"); assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); // Per-leg artifact names: the history is partitioned per machine, // and upload-artifact rejects a name reused within one run. assert_eq!( - SCHEDULED_IMPL_WORKFLOW.matches("bench-history-${{ matrix.os }}").count(), + group_action.matches("bench-history-${{ runner.os }}").count(), 2, "the restore and save steps must agree on the per-leg artifact name" ); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions/upload-artifact@")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh run download")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("GITHUB_STEP_SUMMARY")); + assert!(group_action.contains("actions/upload-artifact@")); + assert!(group_action.contains("gh run download")); + assert!(group_action.contains("GITHUB_STEP_SUMMARY")); + // A group with no registered extras gets none of this. + assert!(!render_group_action("scheduled-exhaustive").contains("bench-history")); // The workflow is identified by its runtime name, not a literal // filename: the root workflow is owned and renameable, and a rename // must not silently reset the series. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("WORKFLOW: ${{ github.workflow }}")); + assert!(group_action.contains("WORKFLOW: ${{ github.workflow }}")); assert!( - !SCHEDULED_IMPL_WORKFLOW.contains("--workflow anvil-scheduled.yml"), + !group_action.contains("--workflow anvil-scheduled.yml"), "a hardcoded workflow filename breaks on rename" ); // Absence and operational failure must stay distinguishable, and the // upload is guarded on the restore having reached a known state -- // otherwise one transient failure publishes an empty store over the // accumulated chain and reports clean. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_RESTORE=restored")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_RESTORE=cold-start")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("if: always() && env.ANVIL_BENCH_RESTORE != ''")); + assert!(group_action.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); + assert!(group_action.contains("ANVIL_BENCH_RESTORE=restored")); + assert!(group_action.contains("ANVIL_BENCH_RESTORE=cold-start")); + assert!(group_action.contains("if: always() && env.ANVIL_BENCH_RESTORE != ''")); // The machine-key escape hatch has to be reachable in CI, which // workflow-level env is not across a called reusable workflow. assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_machine_key:")); diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index ac2fab1a8..a258d500f 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -25,32 +25,20 @@ stages: - stage: scheduled_test displayName: anvil scheduled-test jobs: - # Publish coverage from both legs so OS-gated code is fully - # represented (see the pr-stages.yml comment for the rationale). + # Coverage is published from both legs so OS-gated code is fully + # represented; that step lives in steps/scheduled-test.yml. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (linux) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (windows) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - stage: scheduled_advisories displayName: anvil scheduled-advisories @@ -123,58 +111,32 @@ stages: # variables, so this reaches the recipe without further plumbing. ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} jobs: - # Benchmark regression detection. OS scope matches - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. + # The history round-trip lives in steps/scheduled-benchmarks.yml. Only + # the artifact declaration is here, because publishing is a job-level + # output: a forked (1ESPT) wrapper translates this list into its own + # output shape, which a task inside the step list would bypass. # - # The publish goes through the wrapper's `artifacts` contract like - # every other job, carrying a `condition`: the store is only a valid - # continuation of the chain once the restore reached a known state, - # so an operational restore failure must not overwrite good history - # with a truncated snapshot. Failed runs still publish -- a flagged - # regression fails the job, and those samples belong in the history. - # - # The explicit `checkout` leads the step list rather than going - # through a wrapper parameter: `steps/job.yml` is the file adopters - # fork for 1ESPT and friends, so binding a parameter their wrapper - # does not declare would fail template expansion for the whole - # pipeline. An explicit checkout step needs nothing from the wrapper - # and replaces the implicit default checkout. + # The condition is the guard that keeps an operational restore failure + # from overwriting a good chain with a truncated store; failed runs + # still publish, since a flagged regression fails the job and those + # samples belong in the history. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} artifacts: - - name: bench-history-linux + - name: bench-history-$(Agent.OS) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. - - checkout: self - fetchDepth: 0 - lfs: true - - template: steps/bench-history-restore.yml - parameters: - artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - - template: steps/bench-history-summary.yml - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} artifacts: - - name: bench-history-windows + - name: bench-history-$(Agent.OS) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - - checkout: self - fetchDepth: 0 - lfs: true - - template: steps/bench-history-restore.yml - parameters: - artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - - template: steps/bench-history-summary.yml diff --git a/crates/cargo-anvil/templates/ado/steps/group.yml b/crates/cargo-anvil/templates/ado/steps/group.yml index 0d2802e96..8d1411a1e 100644 --- a/crates/cargo-anvil/templates/ado/steps/group.yml +++ b/crates/cargo-anvil/templates/ado/steps/group.yml @@ -5,6 +5,9 @@ # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token __GROUP__ is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -16,6 +19,7 @@ parameters: type: string default: '' steps: +__PRE_STEPS__ - template: setup.yml parameters: group: __GROUP__ @@ -54,3 +58,4 @@ steps: ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} +__POST_STEPS__ diff --git a/crates/cargo-anvil/templates/github/bench-history-restore.yml b/crates/cargo-anvil/templates/github/bench-history-restore.yml new file mode 100644 index 000000000..e019c72a5 --- /dev/null +++ b/crates/cargo-anvil/templates/github/bench-history-restore.yml @@ -0,0 +1,62 @@ + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Per-leg: the history is partitioned by machine, and one run cannot + # upload the same artifact name twice. + ARTIFACT: bench-history-${{ runner.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression — precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. That distinction is what + # stops one transient failure from publishing an empty store over a + # good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId'); do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 + done + + # No run in the window carried the artifact. That is a genuine cold + # start (first run, or the chain lapsed), so it is surfaced on the + # summary rather than only in this log — "history quietly restarted" + # must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/crates/cargo-anvil/templates/github/bench-history-save.yml b/crates/cargo-anvil/templates/github/bench-history-save.yml new file mode 100644 index 000000000..fc25ea224 --- /dev/null +++ b/crates/cargo-anvil/templates/github/bench-history-save.yml @@ -0,0 +1,25 @@ + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ runner.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/crates/cargo-anvil/templates/github/group-action.yml b/crates/cargo-anvil/templates/github/group-action.yml index 6158a0149..ceeb1d797 100644 --- a/crates/cargo-anvil/templates/github/group-action.yml +++ b/crates/cargo-anvil/templates/github/group-action.yml @@ -37,6 +37,7 @@ inputs: runs: using: composite steps: +__PRE_STEPS__ - uses: ./.github/actions/anvil-setup with: group: __GROUP__ @@ -53,3 +54,4 @@ runs: # checks never read it. GITHUB_TOKEN: ${{ github.token }} run: just anvil-__GROUP__ +__POST_STEPS__ diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index 19b686860..deea2f30a 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -139,8 +139,8 @@ jobs: scheduled-benchmarks: # Benchmark regression detection. x86_64-only, matching - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. + # scheduled-exhaustive. The history round-trip lives in the group's + # composite action; only the job-level concerns are here. strategy: fail-fast: false matrix: @@ -153,101 +153,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. fetch-depth: 0 lfs: true - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: bench-history-${{ matrix.os }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - mkdir -p target/anvil/bench-history - - # Walk back from the newest run and take the first one that - # carries this leg's artifact. Restoring from the latest - # *successful* run would drop every sample collected while the - # pipeline was red from a regression — precisely the window - # that matters. - # - # Absence and failure are kept distinct. A run is only a - # candidate once the artifacts API confirms the artifact exists - # and has not expired; a download that then fails is an - # operational error (token, API, corrupt payload) and fails the - # job rather than being silently downgraded to a cold start. - # That distinction is what stops one transient failure from - # publishing an empty store over a good history. - for run_id in $(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId'); do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" - exit 0 - done - - # No run in the window carried the artifact. That is a genuine - # cold start (first run, or the chain lapsed), so it is surfaced - # on the summary rather than only in this log — "history quietly - # restarted" must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - - name: Save benchmark history - # always(): the run's own samples belong in the history even when - # the analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the - # restore failed operationally the store is not a continuation of - # the chain, and publishing it would overwrite good history with a - # truncated snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused - # or infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + publish-failure: name: Publish scheduled failure needs: diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 6ca45d27f..be3ca115c 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -985,7 +985,7 @@ fn bench_history_bless_rejects_malformed_entries() { // transports. // --------------------------------------------------------------------------- -const SCHEDULED_IMPL: &str = include_str!("../templates/github/scheduled-impl-workflow.yml"); +const GH_BENCH_RESTORE: &str = include_str!("../templates/github/bench-history-restore.yml"); const ADO_RESTORE: &str = include_str!("../templates/ado/steps/bench-history-restore.yml"); /// Extracts a block scalar (`run: |` / `pwsh: |`) from `yaml`, starting the @@ -1051,7 +1051,7 @@ exit 0 "#, ); - let script = block_scalar(SCHEDULED_IMPL, "- name: Restore benchmark history", "run: |"); + let script = block_scalar(GH_BENCH_RESTORE, "- name: Restore benchmark history", "run: |"); write(&root.join("restore.sh"), &script); let github_env = root.join("env.txt"); diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index b60c51d0c..ed2e8a8db 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -1574,32 +1574,20 @@ stages: - stage: scheduled_test displayName: anvil scheduled-test jobs: - # Publish coverage from both legs so OS-gated code is fully - # represented (see the pr-stages.yml comment for the rationale). + # Coverage is published from both legs so OS-gated code is fully + # represented; that step lives in steps/scheduled-test.yml. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (linux) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (windows) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - stage: scheduled_advisories displayName: anvil scheduled-advisories @@ -1672,61 +1660,35 @@ stages: # variables, so this reaches the recipe without further plumbing. ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} jobs: - # Benchmark regression detection. OS scope matches - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. + # The history round-trip lives in steps/scheduled-benchmarks.yml. Only + # the artifact declaration is here, because publishing is a job-level + # output: a forked (1ESPT) wrapper translates this list into its own + # output shape, which a task inside the step list would bypass. # - # The publish goes through the wrapper's `artifacts` contract like - # every other job, carrying a `condition`: the store is only a valid - # continuation of the chain once the restore reached a known state, - # so an operational restore failure must not overwrite good history - # with a truncated snapshot. Failed runs still publish -- a flagged - # regression fails the job, and those samples belong in the history. - # - # The explicit `checkout` leads the step list rather than going - # through a wrapper parameter: `steps/job.yml` is the file adopters - # fork for 1ESPT and friends, so binding a parameter their wrapper - # does not declare would fail template expansion for the whole - # pipeline. An explicit checkout step needs nothing from the wrapper - # and replaces the implicit default checkout. + # The condition is the guard that keeps an operational restore failure + # from overwriting a good chain with a truncated store; failed runs + # still publish, since a flagged regression fails the job and those + # samples belong in the history. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} artifacts: - - name: bench-history-linux + - name: bench-history-$(Agent.OS) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. - - checkout: self - fetchDepth: 0 - lfs: true - - template: steps/bench-history-restore.yml - parameters: - artifact: bench-history-linux - template: steps/scheduled-benchmarks.yml - - template: steps/bench-history-summary.yml - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} artifacts: - - name: bench-history-windows + - name: bench-history-$(Agent.OS) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: - - checkout: self - fetchDepth: 0 - lfs: true - - template: steps/bench-history-restore.yml - parameters: - artifact: bench-history-windows - template: steps/scheduled-benchmarks.yml - - template: steps/bench-history-summary.yml === .pipelines/anvil/steps/advisory-comments.yml === # Copyright (c) Microsoft Corporation. @@ -2123,6 +2085,9 @@ jobs: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token pr-fast is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2181,6 +2146,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token pr-mutants is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2239,6 +2207,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token pr-runtime-analysis is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2297,6 +2268,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token pr-test is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2355,6 +2329,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token scheduled-advisories is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2413,6 +2390,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token scheduled-benchmarks is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2424,6 +2404,12 @@ parameters: type: string default: '' steps: + - checkout: self + fetchDepth: 0 + lfs: true + - template: bench-history-restore.yml + parameters: + artifact: bench-history-$(Agent.OS) - template: setup.yml parameters: group: scheduled-benchmarks @@ -2462,6 +2448,7 @@ steps: ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} + - template: bench-history-summary.yml === .pipelines/anvil/steps/scheduled-exhaustive.yml === # Copyright (c) Microsoft Corporation. @@ -2471,6 +2458,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token scheduled-exhaustive is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2529,6 +2519,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token scheduled-runtime-analysis is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2587,6 +2580,9 @@ steps: # Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md # The token scheduled-test is substituted by cargo-anvil at emit time with # the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. parameters: - name: include_modified type: string @@ -2636,6 +2632,12 @@ steps: ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} + - task: PublishCodeCoverageResults@2 + condition: succeededOrFailed() + displayName: Publish coverage + inputs: + summaryFileLocation: target/coverage/cobertura-*.xml + failIfCoverageEmpty: false === .pipelines/anvil/steps/setup.yml === # Copyright (c) Microsoft Corporation. diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index b1a344e9f..1213f3812 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1679,6 +1679,68 @@ inputs: runs: using: composite steps: + - name: Restore benchmark history + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # Per-leg: the history is partitioned by machine, and one run cannot + # upload the same artifact name twice. + ARTIFACT: bench-history-${{ runner.os }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + mkdir -p target/anvil/bench-history + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression — precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. That distinction is what + # stops one transient failure from publishing an empty store over a + # good history. + for run_id in $(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId'); do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + exit 0 + done + + # No run in the window carried the artifact. That is a genuine cold + # start (first run, or the chain lapsed), so it is surfaced on the + # summary rather than only in this log — "history quietly restarted" + # must not look like "no regressions". + echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-setup with: group: scheduled-benchmarks @@ -1695,6 +1757,31 @@ runs: # checks never read it. GITHUB_TOKEN: ${{ github.token }} run: just anvil-scheduled-benchmarks + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the job. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bench-history-${{ runner.os }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + - name: Publish benchmark findings + if: always() + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi === .github/actions/anvil-scheduled-exhaustive/action.yml === # Copyright (c) Microsoft Corporation. @@ -2488,8 +2575,8 @@ jobs: scheduled-benchmarks: # Benchmark regression detection. x86_64-only, matching - # scheduled-exhaustive. The history is partitioned per machine, so - # each leg carries its own artifact rather than sharing one name. + # scheduled-exhaustive. The history round-trip lives in the group's + # composite action; only the job-level concerns are here. strategy: fail-fast: false matrix: @@ -2502,101 +2589,14 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - # The analysis orders each series by first-parent commit - # topology and locates the merge-base, so it needs the whole - # commit graph. LFS matters because benchmark inputs can be - # LFS-tracked and would otherwise arrive as pointer files. + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. fetch-depth: 0 lfs: true - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: bench-history-${{ matrix.os }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - mkdir -p target/anvil/bench-history - - # Walk back from the newest run and take the first one that - # carries this leg's artifact. Restoring from the latest - # *successful* run would drop every sample collected while the - # pipeline was red from a regression — precisely the window - # that matters. - # - # Absence and failure are kept distinct. A run is only a - # candidate once the artifacts API confirms the artifact exists - # and has not expired; a download that then fails is an - # operational error (token, API, corrupt payload) and fails the - # job rather than being silently downgraded to a cold start. - # That distinction is what stops one transient failure from - # publishing an empty store over a good history. - for run_id in $(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId'); do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" - exit 0 - done - - # No run in the window carried the artifact. That is a genuine - # cold start (first run, or the chain lapsed), so it is surfaced - # on the summary rather than only in this log — "history quietly - # restarted" must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} - - name: Save benchmark history - # always(): the run's own samples belong in the history even when - # the analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the - # restore failed operationally the store is not a continuation of - # the chain, and publishing it would overwrite good history with a - # truncated snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused - # or infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + publish-failure: name: Publish scheduled failure needs: From 5225d8f798b3eaeb78c674ce0bcf2eb191e8619b Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 24 Aug 2026 14:43:57 +0200 Subject: [PATCH 16/24] Address review round 3: fail-closed restore, template fragments, per-leg artifact identity Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 6 +- .../anvil-scheduled-benchmarks/action.yml | 47 +++++--- .github/workflows/anvil-scheduled-impl.yml | 3 + crates/cargo-anvil/benches/catalog.rs | 10 +- crates/cargo-anvil/docs/design/ado.md | 27 +++-- crates/cargo-anvil/docs/design/github.md | 6 +- crates/cargo-anvil/docs/design/updates.md | 17 +++ crates/cargo-anvil/src/anvil/artifacts/ado.rs | 87 +++++++++----- .../cargo-anvil/src/anvil/artifacts/github.rs | 56 +++++++-- .../templates/ado/scheduled-stages.yml | 4 +- .../ado/steps/bench-history-restore.yml | 54 ++++++--- .../ado/steps/scheduled-benchmarks-post.yml | 7 ++ .../ado/steps/scheduled-benchmarks-pre.yml | 26 ++++ .../ado/steps/scheduled-test-post.yml | 15 +++ .../github/bench-history-restore.yml | 46 ++++--- .../templates/github/bench-history-save.yml | 7 +- .../github/scheduled-impl-workflow.yml | 3 + crates/cargo-anvil/tests/recipe_contracts.rs | 113 +++++++++++++++--- .../snapshots/snapshots__ado_backend.snap | 86 ++++++++++--- .../snapshots/snapshots__github_backend.snap | 50 +++++--- 20 files changed, 510 insertions(+), 160 deletions(-) create mode 100644 crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml create mode 100644 crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml create mode 100644 crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml diff --git a/.anvil.lock b/.anvil.lock index 3a7136665..946514ad8 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.4.0" -catalog_checksum = "sha256:5cafd37e7ad6aebffbed60e42052fc6ee16a7528a7c66664d08f10c7b62309cc" +catalog_checksum = "sha256:51c0b93f5abdad28ce7122bb273c4c5db14ebe3e09d0ce0b13d417b76fcdf103" [[file]] path = ".anvil/container/Containerfile" @@ -61,7 +61,7 @@ checksum = "sha256:aae1d9e983c289d124e217ac6412c0421e4f53064441cef60e20e19d7c737 [[file]] path = ".github/actions/anvil-scheduled-benchmarks/action.yml" -checksum = "sha256:7b5914afe3f042601c429bd97016766d9c1a664676adcf0eb715ed22716d1d30" +checksum = "sha256:3df218f81287c5f76cbca91ce6d7a69ba968de552e6b9353cdf1e294335ba280" [[file]] path = ".github/actions/anvil-scheduled-exhaustive/action.yml" @@ -89,7 +89,7 @@ checksum = "sha256:18350505aedb0d3e4bc0941016205acbd17b5619b83caa24689fd9d2ddcc1 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:0827666ae955790776302429199b9ec6910f88dd73e493c54d3c38f80b17559e" +checksum = "sha256:a2aed61fd040e3a86ef96991b8ba6a0853c144a954b85211ca8b5e7b1f30a88c" [[file]] path = ".github/workflows/anvil-scheduled.yml" diff --git a/.github/actions/anvil-scheduled-benchmarks/action.yml b/.github/actions/anvil-scheduled-benchmarks/action.yml index e50acc61a..260728146 100644 --- a/.github/actions/anvil-scheduled-benchmarks/action.yml +++ b/.github/actions/anvil-scheduled-benchmarks/action.yml @@ -37,13 +37,17 @@ inputs: runs: using: composite steps: +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} - # Per-leg: the history is partitioned by machine, and one run cannot - # upload the same artifact name twice. - ARTIFACT: bench-history-${{ runner.os }} + # Per-leg identity comes from the workflow: only it knows the matrix + # value. Deriving the name from the runner OS instead would collide + # if two legs ever ran on like-OS runners, merging two machines' + # samples into one series. + ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The caller's workflow *name*, not a hardcoded filename: the root # scheduled workflow is an owned, renameable file, and a rename must @@ -53,7 +57,20 @@ runs: WINDOW: "30" run: | set -euo pipefail - mkdir -p target/anvil/bench-history + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + } # Walk back from the newest run and take the first that carries this # leg's artifact. Restoring from the latest *successful* run would @@ -64,9 +81,7 @@ runs: # once the artifacts API confirms the artifact exists and has not # expired; a download that then fails is an operational error # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. That distinction is what - # stops one transient failure from publishing an empty store over a - # good history. + # silently downgraded to a cold start. for run_id in $(gh run list --workflow "$WORKFLOW" \ --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do @@ -76,22 +91,22 @@ runs: | head -n1) [ -n "$artifact_id" ] || continue - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ "failing rather than continuing with an empty history, which would" \ "publish a truncated store over the existing chain." exit 1 fi echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + complete_restore restored exit 0 done - # No run in the window carried the artifact. That is a genuine cold - # start (first run, or the chain lapsed), so it is surfaced on the - # summary rather than only in this log — "history quietly restarted" - # must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log — "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" { printf '### Benchmark history: cold start\n\n' @@ -115,6 +130,8 @@ runs: # checks never read it. GITHUB_TOKEN: ${{ github.token }} run: just anvil-scheduled-benchmarks +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Save benchmark history # always(): the run's own samples belong in the history even when the # analysis flagged a regression and failed the job. @@ -126,7 +143,7 @@ runs: if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-history-${{ runner.os }} + name: ${{ env.ANVIL_BENCH_ARTIFACT }} path: target/anvil/bench-history # Comfortably longer than the scheduled cadence, so a paused or # infrequent schedule does not break the chain. diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index deea2f30a..b5ec21385 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -160,6 +160,9 @@ jobs: - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} + # Per-leg artifact identity: only the workflow knows the matrix + # value, and each leg's history must stay its own series. + ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} publish-failure: name: Publish scheduled failure diff --git a/crates/cargo-anvil/benches/catalog.rs b/crates/cargo-anvil/benches/catalog.rs index 6abe5cfa5..e4aff3efc 100644 --- a/crates/cargo-anvil/benches/catalog.rs +++ b/crates/cargo-anvil/benches/catalog.rs @@ -10,6 +10,10 @@ //! grow with the catalog: this crate's own history is a steady accretion of //! checks, groups and backend files. //! +//! The two are timed separately rather than end-to-end, so a move can be +//! attributed to assembly or to rendering rather than leaving the reader to +//! guess which half shifted. +//! //! They are also the shape a trend watch handles well — pure, deterministic, //! no I/O, no network — so a move here is a change in the code rather than in //! the environment. @@ -23,8 +27,10 @@ fn catalog(c: &mut Criterion) { // Assembly alone: the embedded templates and the per-group expansions. group.bench_function("anvil", |b| b.iter(Catalog::anvil)); - // Assembly plus rendering and hashing every artifact body, which is what - // the update path pays to decide whether anything changed. + // Rendering and hashing every artifact body, over an already-assembled + // catalog. Assembly is deliberately outside the timed closure; the + // benchmark above covers it, and the update path's total cost is the + // two together. group.bench_function("checksum", |b| { let catalog = Catalog::anvil(); b.iter(|| catalog.checksum()); diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index f55dd59bc..7ef8835a4 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -866,10 +866,20 @@ Each scheduled benchmark job: failure and fails the job. Finding none across the whole window is a genuine cold start; 3. **publishes** the updated store through the wrapper's `artifacts` parameter - (`{ name: bench-history-$(Agent.OS), path: , condition: … }`), which the - default wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a - `pipelineArtifact` output. The name is derived from the agent OS so both legs - share one declaration. + (`{ name: bench-history-$(Agent.JobName), path: , condition: … }`), which + the default wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a + `pipelineArtifact` output. The name is keyed on the job, which ADO guarantees is + unique within a stage, so two legs can never declare one artifact and merge two + machines' samples into a single series. + +The restore stages its download and creates the store path **only once it has +reached a known state** — restored, or a positively identified cold start. An +operational failure therefore leaves no store directory at all, so a publisher that +runs unconditionally has nothing to upload and the chain survives. That matters +because `artifacts[].condition` is an optional field: a wrapper forked before it +existed accepts the entry and ignores the condition silently, and this is exactly +the case where ignoring it would overwrite good history. The condition remains as a +second line of defence, not the only one. `DownloadPipelineArtifact@2` is not used for the restore: `latestFromBranch` resolves a single build and does not walk, so a cancelled or never-publishing latest build @@ -878,12 +888,9 @@ would cold-start a store that still has usable history. The walk is outcome-agnostic, which is what keeps the chain intact across a regression: a flagged regression fails the stage, so a success-only restore would discard every sample taken while the pipeline stayed red. Publishing is likewise not -limited to green runs, but it *is* conditioned on the restore having reached a known -state — the `condition` on the artifact entry. An operational restore failure -therefore neither continues silently nor overwrites a good chain with a truncated -snapshot. That guard is the reason the `artifacts` contract carries an optional -`condition` (§4.1) rather than the benchmark group emitting its own publish task, -which would bypass the translation a forked wrapper performs. +limited to green runs. The `condition` on the artifact entry lives in the `artifacts` +contract (§4.1) rather than in a publish task the benchmark group emits itself, which +would bypass the translation a forked wrapper performs. Surfacing is by **build failure**, not a PR comment — the regression is discovered after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 71a6d3734..9f7504b38 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -409,15 +409,17 @@ is the canonical YAML: ```text caller anvil-scheduled.yml - permissions upper bound: contents:read + issues:write + permissions upper bound: contents:read + actions:read + issues:write └─ called anvil-scheduled-impl.yml default reset: contents:read ├─ scheduled-test (Linux/Windows × x64/ARM64) ├─ scheduled-advisories (Linux/Windows × x64/ARM64) ├─ scheduled-runtime-analysis (Linux/Windows × x64/ARM64) ├─ scheduled-exhaustive (Linux/Windows x64) + ├─ scheduled-benchmarks (Linux/Windows x64) + │ job override: actions:read (history-artifact restore) └─ publish-failure - needs: all four scheduled groups + needs: all five scheduled groups condition: at least one failure and publication not disabled job override: issues:write only ``` diff --git a/crates/cargo-anvil/docs/design/updates.md b/crates/cargo-anvil/docs/design/updates.md index e0b36b63e..f2f333400 100644 --- a/crates/cargo-anvil/docs/design/updates.md +++ b/crates/cargo-anvil/docs/design/updates.md @@ -174,6 +174,23 @@ The ADO `steps/job.yml` extension wrapper is the deliberate exception to the "do not edit" wording: its header says it is emitted by cargo-anvil and explicitly invites repository customization because that file is the supported 1ESPT hook. +### 2.1 Growing a contract a fork implements + +`steps/job.yml` is the one owned file adopters are expected to fork, so its +parameter contract is the one place where a change can silently do nothing. A +generated file that binds a *new parameter* the fork does not declare fails +template expansion loudly. A new *optional field inside an existing object* +parameter is the dangerous case: the fork accepts the object, ignores the field, +and diverges in behaviour with no error anywhere — the emitter, `.anvil.lock` and +the update flow cannot see it. + +The rule that follows: **a behaviour that protects data must not depend on a fork +having adopted a field.** Express it so the generated side is safe on its own, and +let the contract field be a second line of defence. `artifacts[].condition` is the +worked example — it guards against publishing a truncated benchmark history, and +the restore step is written so a failed restore leaves nothing to publish at all, +which holds even for a wrapper that predates the field. + ## 3. Managed regions Co-owned files with one or more tool-managed sections delimited by sentinel comments. diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index 7a17aac70..8b75d765c 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -80,44 +80,42 @@ const POST_STEPS_PLACEHOLDER: &str = "__POST_STEPS__\n"; /// These live in the group's own emitted step template rather than at the /// call site, so `pr.yml` / `scheduled.yml` stay a plain list of groups. A /// group absent from the table gets nothing. +/// +/// The bodies are template files like every other emitted YAML, rather than +/// Rust string literals: they are fragments of a `steps:` list, so they are +/// not standalone-valid templates, but keeping them in `templates/ado/` +/// means the indentation is real YAML instead of escapes. const GROUP_PRE_STEPS: &[(&str, &str)] = &[( "scheduled-benchmarks", - // The analysis orders each series by first-parent commit topology and - // locates the merge-base, so it needs the whole commit graph. LFS - // matters because benchmark inputs can be LFS-tracked and would - // otherwise arrive as pointer files. - // - // The checkout is explicit rather than a wrapper parameter: job.yml is - // the file adopters fork, so binding a parameter their copy lacks would - // fail expansion for the whole pipeline. - " - checkout: self\n\ - \x20 fetchDepth: 0\n\ - \x20 lfs: true\n\ - \x20 - template: bench-history-restore.yml\n\ - \x20 parameters:\n\ - \x20 artifact: bench-history-$(Agent.OS)\n", + include_str!("../../../templates/ado/steps/scheduled-benchmarks-pre.yml"), )]; /// Steps that run after the uniform group runner, per group. const GROUP_POST_STEPS: &[(&str, &str)] = &[ ( "scheduled-test", - " - task: PublishCodeCoverageResults@2\n\ - \x20 condition: succeededOrFailed()\n\ - \x20 displayName: Publish coverage\n\ - \x20 inputs:\n\ - \x20 summaryFileLocation: target/coverage/cobertura-*.xml\n\ - \x20 failIfCoverageEmpty: false\n", + include_str!("../../../templates/ado/steps/scheduled-test-post.yml"), + ), + ( + "scheduled-benchmarks", + include_str!("../../../templates/ado/steps/scheduled-benchmarks-post.yml"), ), - ("scheduled-benchmarks", " - template: bench-history-summary.yml\n"), ]; -/// The extra steps registered for `group`, or the empty string. -fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> &'static str { - table - .iter() - .find_map(|&(name, steps)| (name == group).then_some(steps)) - .unwrap_or("") +/// The extra steps registered for `group`, with the fragment's own license +/// header removed: the emitted file already carries one, and a second copy +/// mid-list is noise. The explanatory comments are kept. +fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> String { + let Some(text) = table.iter().find_map(|&(name, steps)| (name == group).then_some(steps)) else { + return String::new(); + }; + let body: Vec<&str> = text + .lines() + .skip_while(|line| line.starts_with("# Copyright") || line.starts_with("# Licensed") || *line == "#") + .collect(); + let mut out = body.join("\n"); + out.push('\n'); + out } /// Render the step template for one group. @@ -125,8 +123,8 @@ fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> & fn render_group_step(group: &str) -> String { GROUP_STEP_TEMPLATE .replace(GROUP_PLACEHOLDER, group) - .replace(PRE_STEPS_PLACEHOLDER, extra_steps(GROUP_PRE_STEPS, group)) - .replace(POST_STEPS_PLACEHOLDER, extra_steps(GROUP_POST_STEPS, group)) + .replace(PRE_STEPS_PLACEHOLDER, &extra_steps(GROUP_PRE_STEPS, group)) + .replace(POST_STEPS_PLACEHOLDER, &extra_steps(GROUP_POST_STEPS, group)) } /// Repo-root-relative path for one group's step template. @@ -426,6 +424,23 @@ mod tests { ); } + #[test] + fn per_group_step_keys_name_real_groups() { + // `extra_steps` matches by string equality and falls back to "", so a + // renamed or dropped group would silently lose its steps -- and for + // the benchmark group that means analyzing an empty store and + // reporting "no regressions", the one outcome it must never produce + // by accident. + for (table, label) in [(GROUP_PRE_STEPS, "pre"), (GROUP_POST_STEPS, "post")] { + let mut seen = Vec::new(); + for &(group, _) in table { + assert!(GROUPS.contains(&group), "{label}-step key '{group}' is not a catalog group"); + assert!(!seen.contains(&group), "{label}-step key '{group}' is registered twice"); + seen.push(group); + } + } + } + #[test] fn scheduled_stages_has_four_groups() { for needle in [ @@ -488,8 +503,18 @@ mod tests { // Absence and operational failure must stay distinguishable, or one // transient error publishes an empty store over a good history. assert!(BENCH_HISTORY_RESTORE_STEP.contains("if ($status -eq 404) { continue }")); - assert!(BENCH_HISTORY_RESTORE_STEP.contains("ANVIL_BENCH_RESTORE]restored")); - assert!(BENCH_HISTORY_RESTORE_STEP.contains("ANVIL_BENCH_RESTORE]cold-start")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("Complete-Restore 'restored'")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("Complete-Restore 'cold-start'")); + // Fail-closed by construction: the store path is created only inside + // Complete-Restore, so an operational failure leaves nothing for a + // wrapper that ignores the artifact condition to upload. + assert_eq!( + BENCH_HISTORY_RESTORE_STEP + .matches("New-Item -ItemType Directory -Force -Path $path") + .count(), + 1, + "the store path must be created only on a completed restore" + ); assert!( !BENCH_HISTORY_RESTORE_STEP.contains("continueOnError"), "a blanket continueOnError would read every failure as a cold start" diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index 883f26d6f..4a759e8fd 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -74,12 +74,20 @@ const GROUP_POST_STEPS: &[(&str, &str)] = &[( include_str!("../../../templates/github/bench-history-save.yml"), )]; -/// The extra steps registered for `group`, or the empty string. -fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> &'static str { - table - .iter() - .find_map(|&(name, steps)| (name == group).then_some(steps)) - .unwrap_or("") +/// The extra steps registered for `group`, with the fragment's own license +/// header removed: the composite action already carries one. Explanatory +/// comments are kept. +fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> String { + let Some(text) = table.iter().find_map(|&(name, steps)| (name == group).then_some(steps)) else { + return String::new(); + }; + let body: Vec<&str> = text + .lines() + .skip_while(|line| line.starts_with("# Copyright") || line.starts_with("# Licensed") || *line == "#") + .collect(); + let mut out = body.join("\n"); + out.push('\n'); + out } /// Render the `action.yml` for one check group's composite action. @@ -87,8 +95,8 @@ fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> & fn render_group_action(group: &str) -> String { GROUP_ACTION_TEMPLATE .replace(GROUP_PLACEHOLDER, group) - .replace(PRE_STEPS_PLACEHOLDER, extra_steps(GROUP_PRE_STEPS, group)) - .replace(POST_STEPS_PLACEHOLDER, extra_steps(GROUP_POST_STEPS, group)) + .replace(PRE_STEPS_PLACEHOLDER, &extra_steps(GROUP_PRE_STEPS, group)) + .replace(POST_STEPS_PLACEHOLDER, &extra_steps(GROUP_POST_STEPS, group)) } /// Repo-root-relative path for a per-group composite action. @@ -329,6 +337,21 @@ mod tests { ); } + #[test] + fn per_group_step_keys_name_real_groups() { + // See the ADO twin: a stale key silently drops the group's steps, + // which for the benchmark group means an empty store reporting + // "no regressions". + for (table, label) in [(GROUP_PRE_STEPS, "pre"), (GROUP_POST_STEPS, "post")] { + let mut seen = Vec::new(); + for &(group, _) in table { + assert!(GROUPS.contains(&group), "{label}-step key '{group}' is not a catalog group"); + assert!(!seen.contains(&group), "{label}-step key '{group}' is registered twice"); + seen.push(group); + } + } + } + #[test] fn scheduled_benchmarks_job_round_trips_the_history_artifact() { // The job is a plain checkout + group action like every other one; @@ -337,11 +360,14 @@ mod tests { assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); // Per-leg artifact names: the history is partitioned per machine, // and upload-artifact rejects a name reused within one run. + // Per-leg artifact identity comes from the workflow, which is the + // only place that knows the matrix value. assert_eq!( - group_action.matches("bench-history-${{ runner.os }}").count(), + group_action.matches("${{ env.ANVIL_BENCH_ARTIFACT }}").count(), 2, "the restore and save steps must agree on the per-leg artifact name" ); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }}")); assert!(group_action.contains("actions/upload-artifact@")); assert!(group_action.contains("gh run download")); assert!(group_action.contains("GITHUB_STEP_SUMMARY")); @@ -360,8 +386,16 @@ mod tests { // otherwise one transient failure publishes an empty store over the // accumulated chain and reports clean. assert!(group_action.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); - assert!(group_action.contains("ANVIL_BENCH_RESTORE=restored")); - assert!(group_action.contains("ANVIL_BENCH_RESTORE=cold-start")); + assert!(group_action.contains("complete_restore restored")); + assert!(group_action.contains("complete_restore cold-start")); + // Fail-closed by construction: the store path is created only inside + // complete_restore, so an operational failure leaves nothing to + // upload over the accumulated chain. + assert_eq!( + group_action.matches("mkdir -p target/anvil/bench-history").count(), + 1, + "the store path must be created only on a completed restore" + ); assert!(group_action.contains("if: always() && env.ANVIL_BENCH_RESTORE != ''")); // The machine-key escape hatch has to be reachable in CI, which // workflow-level env is not across a called reusable workflow. diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index a258d500f..c7aefb07e 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -125,7 +125,7 @@ stages: name: linux pool: ${{ parameters.linuxPool }} artifacts: - - name: bench-history-$(Agent.OS) + - name: bench-history-$(Agent.JobName) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: @@ -135,7 +135,7 @@ stages: name: windows pool: ${{ parameters.windowsPool }} artifacts: - - name: bench-history-$(Agent.OS) + - name: bench-history-$(Agent.JobName) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml index 971e279a3..a7ceb3dc2 100644 --- a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -13,8 +13,16 @@ # therefore cold-start a store that actually has history. This step # resolves the build itself, so absence of the artifact throughout the # window (a genuine cold start) stays distinct from an operational -# failure, which fails the job rather than silently publishing a -# truncated snapshot over a good chain. +# failure. +# +# The store path is created only once the restore has reached a known +# state -- restored, or a positively identified cold start. An operational +# failure therefore leaves *no store directory at all*, so a publisher +# that runs unconditionally has nothing to upload and the accumulated +# chain survives. That makes the guard structural rather than dependent on +# every job wrapper honouring the `artifacts` condition: a fork predating +# that field ignores it silently, and this is the case where ignoring it +# would overwrite good history with a truncated store. # # See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md parameters: @@ -33,13 +41,27 @@ steps: $path = '${{ parameters.path }}' $window = ${{ parameters.window }} - New-Item -ItemType Directory -Force -Path $path | Out-Null - if (-not $env:SYSTEM_ACCESSTOKEN) { Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." exit 1 } + # Everything lands here first; $path is created only on success. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + # Publishes the staged store as the restore's result and records how + # the restore ended, so the publish step can tell a continuation of + # the chain from a job that never established one. + function Complete-Restore([string]$state) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + if (Test-Path (Join-Path $staging '*')) { + Copy-Item -Path (Join-Path $staging '*') -Destination $path -Recurse -Force + } + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + } + $collection = $env:SYSTEM_COLLECTIONURI $project = $env:SYSTEM_TEAMPROJECTID $definition = $env:SYSTEM_DEFINITIONID @@ -70,24 +92,24 @@ steps: if (-not $found.resource.downloadUrl) { continue } Write-Host "anvil: restoring $artifact from build $($run.id)" - $tmp = [System.IO.Path]::GetTempPath() - $zip = Join-Path $tmp "$artifact.zip" - $staging = Join-Path $tmp "$artifact-extract" - Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract" + Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip - Expand-Archive -LiteralPath $zip -DestinationPath $staging -Force + Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force # The archive nests its contents under a directory named for the - # artifact; lift them up into the store path. - $inner = Join-Path $staging $artifact - $source = if (Test-Path $inner) { $inner } else { $staging } - Copy-Item -Path (Join-Path $source '*') -Destination $path -Recurse -Force - Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]restored" + # artifact; lift them up into the staging directory. + $inner = Join-Path $extract $artifact + $source = if (Test-Path $inner) { $inner } else { $extract } + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + Complete-Restore 'restored' exit 0 } - # Nothing in the window carried the artifact: a genuine cold start. + # Nothing in the window carried the artifact: a genuine cold start, + # which is a valid empty store rather than a failure. Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" - Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]cold-start" + Complete-Restore 'cold-start' displayName: Restore ${{ parameters.artifact }} env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml new file mode 100644 index 000000000..a6c54ee5e --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, after the uniform runner. Not a standalone template: this is +# part of a `steps:` list. + - template: bench-history-summary.yml diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml new file mode 100644 index 000000000..c0d9a982a --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, ahead of the uniform runner. Not a standalone template: this is +# part of a `steps:` list, which is why it has no `steps:` key of its own. +# +# The analysis orders each series by first-parent commit topology and +# locates the merge-base, so it needs the whole commit graph. LFS matters +# because benchmark inputs can be LFS-tracked and would otherwise arrive as +# pointer files. +# +# The checkout is explicit rather than a job-wrapper parameter: job.yml is +# the file adopters fork, so binding a parameter their copy lacks would fail +# expansion for the whole pipeline. +# +# The artifact is named for the *job*, which ADO guarantees is unique within +# a stage. Naming it for the agent OS instead would let two legs on like-OS +# pools declare one artifact, colliding on publish and merging two machines' +# samples into a single series. + - checkout: self + fetchDepth: 0 + lfs: true + - template: bench-history-restore.yml + parameters: + artifact: bench-history-$(Agent.JobName) diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml new file mode 100644 index 000000000..519f79120 --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-test group's emitted step template, +# after the uniform runner. Not a standalone template: this is part of a +# `steps:` list. +# +# Coverage is published from every leg so OS-gated code is fully +# represented. + - task: PublishCodeCoverageResults@2 + condition: succeededOrFailed() + displayName: Publish coverage + inputs: + summaryFileLocation: target/coverage/cobertura-*.xml + failIfCoverageEmpty: false diff --git a/crates/cargo-anvil/templates/github/bench-history-restore.yml b/crates/cargo-anvil/templates/github/bench-history-restore.yml index e019c72a5..cebfb8cba 100644 --- a/crates/cargo-anvil/templates/github/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/github/bench-history-restore.yml @@ -1,10 +1,17 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} - # Per-leg: the history is partitioned by machine, and one run cannot - # upload the same artifact name twice. - ARTIFACT: bench-history-${{ runner.os }} + # Per-leg identity comes from the workflow: only it knows the matrix + # value. Deriving the name from the runner OS instead would collide + # if two legs ever ran on like-OS runners, merging two machines' + # samples into one series. + ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The caller's workflow *name*, not a hardcoded filename: the root # scheduled workflow is an owned, renameable file, and a rename must @@ -14,7 +21,20 @@ WINDOW: "30" run: | set -euo pipefail - mkdir -p target/anvil/bench-history + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + } # Walk back from the newest run and take the first that carries this # leg's artifact. Restoring from the latest *successful* run would @@ -25,9 +45,7 @@ # once the artifacts API confirms the artifact exists and has not # expired; a download that then fails is an operational error # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. That distinction is what - # stops one transient failure from publishing an empty store over a - # good history. + # silently downgraded to a cold start. for run_id in $(gh run list --workflow "$WORKFLOW" \ --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do @@ -37,22 +55,22 @@ | head -n1) [ -n "$artifact_id" ] || continue - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ "failing rather than continuing with an empty history, which would" \ "publish a truncated store over the existing chain." exit 1 fi echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + complete_restore restored exit 0 done - # No run in the window carried the artifact. That is a genuine cold - # start (first run, or the chain lapsed), so it is surfaced on the - # summary rather than only in this log — "history quietly restarted" - # must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log — "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" { printf '### Benchmark history: cold start\n\n' diff --git a/crates/cargo-anvil/templates/github/bench-history-save.yml b/crates/cargo-anvil/templates/github/bench-history-save.yml index fc25ea224..e57f78f9f 100644 --- a/crates/cargo-anvil/templates/github/bench-history-save.yml +++ b/crates/cargo-anvil/templates/github/bench-history-save.yml @@ -1,3 +1,8 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Save benchmark history # always(): the run's own samples belong in the history even when the # analysis flagged a regression and failed the job. @@ -9,7 +14,7 @@ if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-history-${{ runner.os }} + name: ${{ env.ANVIL_BENCH_ARTIFACT }} path: target/anvil/bench-history # Comfortably longer than the scheduled cadence, so a paused or # infrequent schedule does not break the chain. diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index deea2f30a..b5ec21385 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -160,6 +160,9 @@ jobs: - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} + # Per-leg artifact identity: only the workflow knows the matrix + # value, and each leg's history must stay its own series. + ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} publish-failure: name: Publish scheduled failure diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index be3ca115c..9ce8e1360 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -234,6 +234,19 @@ fn assert_failed(output: &Output, context: &str) { ); } +/// Both streams of a recipe run, for assertion messages. +/// +/// A recipe that dies before producing output says why on stderr, so a +/// failure message carrying only stdout hides the actual cause. +fn both_streams(output: &Output) -> String { + format!( + "\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- status: {:?} ---", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + output.status + ) +} + #[test] fn impact_format_resolves_directory_aliases_and_fails_closed() { if !tools_available() { @@ -809,8 +822,8 @@ fn bench_history_gates_on_active_regressions_only() { let (tmp, output) = run_bench_history(ACTIVE_REGRESSION, &[]); assert_failed(&output, "an active regression"); let text = String::from_utf8_lossy(&output.stdout); - assert!(text.contains("emit_alloc/churn"), "names the benchmark:\n{text}"); - assert!(text.contains("8392995a"), "names the attributed commit:\n{text}"); + assert!(text.contains("emit_alloc/churn"), "names the benchmark:{}", both_streams(&output)); + assert!(text.contains("8392995a"), "names the attributed commit:{}", both_streams(&output)); let calls = cargo_calls(tmp.path()); assert!(calls.contains("bench-history collect"), "calls:\n{calls}"); assert!(calls.contains("bench-history analyze"), "calls:\n{calls}"); @@ -818,11 +831,7 @@ fn bench_history_gates_on_active_regressions_only() { // A recovered regression and an improvement both need no action. for (findings, label) in [(INACTIVE_REGRESSION, "inactive"), (IMPROVEMENT, "improvement")] { let (_tmp, output) = run_bench_history(findings, &[]); - assert!( - output.status.success(), - "{label} finding must not gate:\n{}", - String::from_utf8_lossy(&output.stdout) - ); + assert!(output.status.success(), "{label} finding must not gate:{}", both_streams(&output)); } // A workspace with no benchmarks analyzes to nothing and stays green: @@ -830,8 +839,8 @@ fn bench_history_gates_on_active_regressions_only() { let (_tmp, output) = run_bench_history(NO_FINDINGS, &[]); assert!( output.status.success(), - "an empty history must be a clean no-op:\n{}", - String::from_utf8_lossy(&output.stdout) + "an empty history must be a clean no-op:{}", + both_streams(&output) ); } @@ -861,8 +870,8 @@ fn bench_history_reports_without_gating_outside_ci() { ); assert!( output.status.success(), - "a local run reports but does not gate:\n{}", - String::from_utf8_lossy(&output.stdout) + "a local run reports but does not gate:{}", + both_streams(&output) ); let text = String::from_utf8_lossy(&output.stdout); assert!(text.contains("emit_alloc/churn"), "still reports the finding:\n{text}"); @@ -921,8 +930,8 @@ fn bench_history_bless_reconciles_on_exact_prefix_identity() { ); assert!( cargo_calls(tmp.path()).contains("bench-history bless"), - "a narrower stored blessing must not satisfy a broader request:\n{}", - String::from_utf8_lossy(&output.stdout) + "a narrower stored blessing must not satisfy a broader request:{}", + both_streams(&output) ); // The exact same prefix already recorded is a no-op, so a scheduled run @@ -932,8 +941,8 @@ fn bench_history_bless_reconciles_on_exact_prefix_identity() { assert!(output.status.success()); assert!( !cargo_calls(tmp.path()).contains("bench-history bless"), - "an already-applied blessing must not be re-appended:\n{}", - String::from_utf8_lossy(&output.stdout) + "an already-applied blessing must not be re-appended:{}", + both_streams(&output) ); } @@ -959,8 +968,8 @@ fn bench_history_bless_rejects_malformed_entries() { ); assert!( String::from_utf8_lossy(&output.stdout).contains("#1234"), - "the reason must survive intact:\n{}", - String::from_utf8_lossy(&output.stdout) + "the reason must survive intact:{}", + both_streams(&output) ); assert!(cargo_calls(tmp.path()).contains("bench-history bless")); @@ -1099,8 +1108,8 @@ fn github_restore_separates_absence_from_failure() { assert!(env.contains("ANVIL_BENCH_RESTORE=restored"), "env:\n{env}"); assert!( String::from_utf8_lossy(&output.stdout).contains("run 10"), - "should name the run it restored from:\n{}", - String::from_utf8_lossy(&output.stdout) + "should name the run it restored from:{}", + both_streams(&output) ); // (2) No run in the window carries it: a genuine cold start, and it must @@ -1226,3 +1235,69 @@ fn ado_restore_separates_absence_from_failure() { ); } } + +#[test] +fn bench_history_bless_listings_are_process_scoped() { + if !tools_available() { + return; + } + // The blessing reconciliation writes `list blessings` output to a temp + // file before reading it back. A fixed name lets two jobs sharing a + // machine -- matrix legs on a self-hosted agent, concurrent local runs -- + // read each other's listing. Reverting the process scoping must trip + // this deterministically rather than by chance under parallel runs. + let shared_temp = TempDir::new().unwrap(); + + // Each invocation asks for a different benchmark and is told a different + // already-applied set, so consuming the other's listing is observable: + // each would then think its blessing was already in effect and skip it. + let cases = [ + ("alpha", r#"{"blessings":[{"commit":"8392995a3b94","prefixes":["beta"]}]}"#), + ("beta", r#"{"blessings":[{"commit":"8392995a3b94","prefixes":["alpha"]}]}"#), + ]; + + for (benchmark, applied) in cases { + let file = format!("[[blessing]]\nbenchmark = \"{benchmark}\"\ncommit = \"8392995a\"\nreason = \"deliberate\"\n"); + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + write(&tmp.path().join(".config/bench-blessings.toml"), &file); + let log = tmp.path().join("cargo.log"); + let output = run_just( + tmp.path(), + &["_anvil-bench-history-bless", "store", ".config/bench-blessings.toml"], + &[ + ("FAKE_CBH_BLESSINGS", OsStr::new(applied)), + ("FAKE_CARGO_LOG", log.as_os_str()), + // Both invocations share one temp directory, which is what a + // fixed listing filename would collide in. + ("RUNNER_TEMP", shared_temp.path().as_os_str()), + ], + ); + assert!( + output.status.success(), + "reconciliation failed for {benchmark}:{}", + both_streams(&output) + ); + // Its own listing says a *different* benchmark is blessed, so this + // one must still be applied. + assert!( + cargo_calls(tmp.path()).contains("bench-history bless"), + "{benchmark} must be blessed from its own listing:{}", + both_streams(&output) + ); + } + + // One listing file per process, so the two runs never shared one. + let listings: Vec<_> = std::fs::read_dir(shared_temp.path()) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with("anvil-bench-blessings")) + .collect(); + assert_eq!(listings.len(), 2, "each invocation must write its own listing, got: {listings:?}"); +} diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index ed2e8a8db..c57472cca 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -1674,7 +1674,7 @@ stages: name: linux pool: ${{ parameters.linuxPool }} artifacts: - - name: bench-history-$(Agent.OS) + - name: bench-history-$(Agent.JobName) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: @@ -1684,7 +1684,7 @@ stages: name: windows pool: ${{ parameters.windowsPool }} artifacts: - - name: bench-history-$(Agent.OS) + - name: bench-history-$(Agent.JobName) path: target/anvil/bench-history condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) steps: @@ -1804,8 +1804,16 @@ steps: # therefore cold-start a store that actually has history. This step # resolves the build itself, so absence of the artifact throughout the # window (a genuine cold start) stays distinct from an operational -# failure, which fails the job rather than silently publishing a -# truncated snapshot over a good chain. +# failure. +# +# The store path is created only once the restore has reached a known +# state -- restored, or a positively identified cold start. An operational +# failure therefore leaves *no store directory at all*, so a publisher +# that runs unconditionally has nothing to upload and the accumulated +# chain survives. That makes the guard structural rather than dependent on +# every job wrapper honouring the `artifacts` condition: a fork predating +# that field ignores it silently, and this is the case where ignoring it +# would overwrite good history with a truncated store. # # See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md parameters: @@ -1824,13 +1832,27 @@ steps: $path = '${{ parameters.path }}' $window = ${{ parameters.window }} - New-Item -ItemType Directory -Force -Path $path | Out-Null - if (-not $env:SYSTEM_ACCESSTOKEN) { Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." exit 1 } + # Everything lands here first; $path is created only on success. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + # Publishes the staged store as the restore's result and records how + # the restore ended, so the publish step can tell a continuation of + # the chain from a job that never established one. + function Complete-Restore([string]$state) { + New-Item -ItemType Directory -Force -Path $path | Out-Null + if (Test-Path (Join-Path $staging '*')) { + Copy-Item -Path (Join-Path $staging '*') -Destination $path -Recurse -Force + } + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + } + $collection = $env:SYSTEM_COLLECTIONURI $project = $env:SYSTEM_TEAMPROJECTID $definition = $env:SYSTEM_DEFINITIONID @@ -1861,24 +1883,24 @@ steps: if (-not $found.resource.downloadUrl) { continue } Write-Host "anvil: restoring $artifact from build $($run.id)" - $tmp = [System.IO.Path]::GetTempPath() - $zip = Join-Path $tmp "$artifact.zip" - $staging = Join-Path $tmp "$artifact-extract" - Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract" + Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip - Expand-Archive -LiteralPath $zip -DestinationPath $staging -Force + Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force # The archive nests its contents under a directory named for the - # artifact; lift them up into the store path. - $inner = Join-Path $staging $artifact - $source = if (Test-Path $inner) { $inner } else { $staging } - Copy-Item -Path (Join-Path $source '*') -Destination $path -Recurse -Force - Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]restored" + # artifact; lift them up into the staging directory. + $inner = Join-Path $extract $artifact + $source = if (Test-Path $inner) { $inner } else { $extract } + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + Complete-Restore 'restored' exit 0 } - # Nothing in the window carried the artifact: a genuine cold start. + # Nothing in the window carried the artifact: a genuine cold start, + # which is a valid empty store rather than a failure. Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" - Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]cold-start" + Complete-Restore 'cold-start' displayName: Restore ${{ parameters.artifact }} env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) @@ -2404,12 +2426,29 @@ parameters: type: string default: '' steps: +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, ahead of the uniform runner. Not a standalone template: this is +# part of a `steps:` list, which is why it has no `steps:` key of its own. +# +# The analysis orders each series by first-parent commit topology and +# locates the merge-base, so it needs the whole commit graph. LFS matters +# because benchmark inputs can be LFS-tracked and would otherwise arrive as +# pointer files. +# +# The checkout is explicit rather than a job-wrapper parameter: job.yml is +# the file adopters fork, so binding a parameter their copy lacks would fail +# expansion for the whole pipeline. +# +# The artifact is named for the *job*, which ADO guarantees is unique within +# a stage. Naming it for the agent OS instead would let two legs on like-OS +# pools declare one artifact, colliding on publish and merging two machines' +# samples into a single series. - checkout: self fetchDepth: 0 lfs: true - template: bench-history-restore.yml parameters: - artifact: bench-history-$(Agent.OS) + artifact: bench-history-$(Agent.JobName) - template: setup.yml parameters: group: scheduled-benchmarks @@ -2448,6 +2487,9 @@ steps: ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, after the uniform runner. Not a standalone template: this is +# part of a `steps:` list. - template: bench-history-summary.yml === .pipelines/anvil/steps/scheduled-exhaustive.yml === @@ -2632,6 +2674,12 @@ steps: ANVIL_INCLUDE_MODIFIED: ${{ parameters.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ parameters.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ parameters.include_required }} +# Fragment spliced into the scheduled-test group's emitted step template, +# after the uniform runner. Not a standalone template: this is part of a +# `steps:` list. +# +# Coverage is published from every leg so OS-gated code is fully +# represented. - task: PublishCodeCoverageResults@2 condition: succeededOrFailed() displayName: Publish coverage diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 1213f3812..7085a9f89 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1679,13 +1679,17 @@ inputs: runs: using: composite steps: +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Restore benchmark history shell: bash env: GH_TOKEN: ${{ github.token }} - # Per-leg: the history is partitioned by machine, and one run cannot - # upload the same artifact name twice. - ARTIFACT: bench-history-${{ runner.os }} + # Per-leg identity comes from the workflow: only it knows the matrix + # value. Deriving the name from the runner OS instead would collide + # if two legs ever ran on like-OS runners, merging two machines' + # samples into one series. + ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} # The caller's workflow *name*, not a hardcoded filename: the root # scheduled workflow is an owned, renameable file, and a rename must @@ -1695,7 +1699,20 @@ runs: WINDOW: "30" run: | set -euo pipefail - mkdir -p target/anvil/bench-history + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + } # Walk back from the newest run and take the first that carries this # leg's artifact. Restoring from the latest *successful* run would @@ -1706,9 +1723,7 @@ runs: # once the artifacts API confirms the artifact exists and has not # expired; a download that then fails is an operational error # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. That distinction is what - # stops one transient failure from publishing an empty store over a - # good history. + # silently downgraded to a cold start. for run_id in $(gh run list --workflow "$WORKFLOW" \ --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ --json databaseId --jq '.[].databaseId'); do @@ -1718,22 +1733,22 @@ runs: | head -n1) [ -n "$artifact_id" ] || continue - if ! gh run download "$run_id" --name "$ARTIFACT" --dir target/anvil/bench-history; then + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ "failing rather than continuing with an empty history, which would" \ "publish a truncated store over the existing chain." exit 1 fi echo "restored benchmark history from run $run_id" - echo "ANVIL_BENCH_RESTORE=restored" >> "$GITHUB_ENV" + complete_restore restored exit 0 done - # No run in the window carried the artifact. That is a genuine cold - # start (first run, or the chain lapsed), so it is surfaced on the - # summary rather than only in this log — "history quietly restarted" - # must not look like "no regressions". - echo "ANVIL_BENCH_RESTORE=cold-start" >> "$GITHUB_ENV" + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log — "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" { printf '### Benchmark history: cold start\n\n' @@ -1757,6 +1772,8 @@ runs: # checks never read it. GITHUB_TOKEN: ${{ github.token }} run: just anvil-scheduled-benchmarks +# Fragment spliced into the scheduled-benchmarks group's composite action. +# Not a standalone action: this is part of a steps list. - name: Save benchmark history # always(): the run's own samples belong in the history even when the # analysis flagged a regression and failed the job. @@ -1768,7 +1785,7 @@ runs: if: always() && env.ANVIL_BENCH_RESTORE != '' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: bench-history-${{ runner.os }} + name: ${{ env.ANVIL_BENCH_ARTIFACT }} path: target/anvil/bench-history # Comfortably longer than the scheduled cadence, so a paused or # infrequent schedule does not break the chain. @@ -2596,6 +2613,9 @@ jobs: - uses: ./.github/actions/anvil-scheduled-benchmarks env: ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} + # Per-leg artifact identity: only the workflow knows the matrix + # value, and each leg's history must stay its own series. + ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} publish-failure: name: Publish scheduled failure From e397a46de2847fa250879c0eaf7dd5f1b8511356 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Tue, 25 Aug 2026 18:42:42 +0200 Subject: [PATCH 17/24] fix(cargo-anvil): scope ADO restore temp paths and fail closed on store desync Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 10 ++--- .../anvil-scheduled-benchmarks/action.yml | 3 ++ .github/workflows/anvil-scheduled-impl.yml | 2 +- .github/workflows/anvil-scheduled.yml | 2 +- .../ado/steps/bench-history-restore.yml | 13 ++++-- .../github/bench-history-restore.yml | 3 ++ .../github/scheduled-impl-workflow.yml | 2 +- .../github/scheduled-root-workflow.yml | 2 +- .../justfiles/anvil/checks/bench-history.just | 24 +++++++++- crates/cargo-anvil/tests/recipe_contracts.rs | 44 +++++++++++++++++++ .../snapshots/snapshots__ado_backend.snap | 37 ++++++++++++++-- .../snapshots/snapshots__github_backend.snap | 31 +++++++++++-- .../snapshots/snapshots__local_only.snap | 24 +++++++++- justfiles/anvil/checks/bench-history.just | 24 +++++++++- 14 files changed, 199 insertions(+), 22 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index d0d514e1b..c4f985f3c 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.4.0" -catalog_checksum = "sha256:712752edd247fcab2f0f011469bfff674aa981958476b0271bbdf0eed4fea446" +catalog_checksum = "sha256:f88c5b1aa3cd7b274b5533a01f1e727f8fa6e3cf63fba2c48699965076e8b52a" [[file]] path = ".anvil/container/Containerfile" @@ -61,7 +61,7 @@ checksum = "sha256:aae1d9e983c289d124e217ac6412c0421e4f53064441cef60e20e19d7c737 [[file]] path = ".github/actions/anvil-scheduled-benchmarks/action.yml" -checksum = "sha256:3df218f81287c5f76cbca91ce6d7a69ba968de552e6b9353cdf1e294335ba280" +checksum = "sha256:1873a7e67180d294713d4d6b1f4d31be570c43616434dd136eb6cb90da02a4e1" [[file]] path = ".github/actions/anvil-scheduled-exhaustive/action.yml" @@ -89,11 +89,11 @@ checksum = "sha256:18350505aedb0d3e4bc0941016205acbd17b5619b83caa24689fd9d2ddcc1 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:a2aed61fd040e3a86ef96991b8ba6a0853c144a954b85211ca8b5e7b1f30a88c" +checksum = "sha256:16cf6c108a4afe83ea7fd2f37d52410e7f4aff3da6d86f8427c28c5a5393c073" [[file]] path = ".github/workflows/anvil-scheduled.yml" -checksum = "sha256:e0ed2fc2b6608d775cf461a4b255f53a4bf96e3e7055726f3d5f4bf6dd16daae" +checksum = "sha256:4e8fdc164ae1963b81bf5215c17fc575c56dfa9a76f6ce13e706524d1099cf18" [[file]] path = "justfiles/anvil/checks/aprz.just" @@ -105,7 +105,7 @@ checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536 [[file]] path = "justfiles/anvil/checks/bench-history.just" -checksum = "sha256:c0c4cd8c84b1ae778d1f9b8fe9e0f90ca607dd0df626f88da8f52aecc02a23ff" +checksum = "sha256:7b29f5b7fe0ba09cc951ff1a140f9f5d0b599a4212574b3f70c5d085d1485036" [[file]] path = "justfiles/anvil/checks/bench.just" diff --git a/.github/actions/anvil-scheduled-benchmarks/action.yml b/.github/actions/anvil-scheduled-benchmarks/action.yml index 260728146..0b0c6da68 100644 --- a/.github/actions/anvil-scheduled-benchmarks/action.yml +++ b/.github/actions/anvil-scheduled-benchmarks/action.yml @@ -70,6 +70,9 @@ runs: cp -R "$staging/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach the recipe's store from the published one. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" } # Walk back from the newest run and take the first that carries this diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index b5ec21385..65fe41e90 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -41,7 +41,7 @@ on: # The caller grants the maximum token scopes available to this reusable # workflow. Reset jobs to read-only here, then restore only the publisher's -# issues scope below. See docs/design/github.md §9. +# issues scope below. See docs/design/github.md. permissions: contents: read diff --git a/.github/workflows/anvil-scheduled.yml b/.github/workflows/anvil-scheduled.yml index ebc8843f3..e52452206 100644 --- a/.github/workflows/anvil-scheduled.yml +++ b/.github/workflows/anvil-scheduled.yml @@ -18,7 +18,7 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml # A called workflow cannot elevate beyond its caller. The implementation # resets this upper bound to read-only and restores issues:write only on - # publish-failure. See docs/design/github.md §9. + # publish-failure. See docs/design/github.md. permissions: contents: read # The scheduled-benchmarks job restores its history artifact, which diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml index a7ceb3dc2..bc6cdda77 100644 --- a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -47,7 +47,10 @@ steps: } # Everything lands here first; $path is created only on success. - $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging" + # $PID-scoped: on a self-hosted agent two concurrent jobs share the + # temp directory, and an artifact name is not unique across + # overlapping pipeline runs. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging-$PID" Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $staging | Out-Null @@ -60,6 +63,10 @@ steps: Copy-Item -Path (Join-Path $staging '*') -Destination $path -Recurse -Force } Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + # The recipe refuses to write anywhere but here, so a wrapper that + # retargets `path` cannot silently detach the recipe's store from + # the one that is actually published. + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_WIRED_STORE]$path" } $collection = $env:SYSTEM_COLLECTIONURI @@ -92,8 +99,8 @@ steps: if (-not $found.resource.downloadUrl) { continue } Write-Host "anvil: restoring $artifact from build $($run.id)" - $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact.zip" - $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract" + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-$PID.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract-$PID" Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force diff --git a/crates/cargo-anvil/templates/github/bench-history-restore.yml b/crates/cargo-anvil/templates/github/bench-history-restore.yml index cebfb8cba..28498ba3e 100644 --- a/crates/cargo-anvil/templates/github/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/github/bench-history-restore.yml @@ -34,6 +34,9 @@ cp -R "$staging/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach the recipe's store from the published one. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" } # Walk back from the newest run and take the first that carries this diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index b5ec21385..65fe41e90 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -41,7 +41,7 @@ on: # The caller grants the maximum token scopes available to this reusable # workflow. Reset jobs to read-only here, then restore only the publisher's -# issues scope below. See docs/design/github.md §9. +# issues scope below. See docs/design/github.md. permissions: contents: read diff --git a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml index ebc8843f3..e52452206 100644 --- a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml @@ -18,7 +18,7 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml # A called workflow cannot elevate beyond its caller. The implementation # resets this upper bound to read-only and restores issues:write only on - # publish-failure. See docs/design/github.md §9. + # publish-failure. See docs/design/github.md. permissions: contents: read # The scheduled-benchmarks job restores its history artifact, which diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just index 64c426d30..8ee6e15bc 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -43,7 +43,29 @@ anvil-bench-history: anvil-bench-history-validate-prereqs $ErrorActionPreference = 'Stop' - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + if ($wantPath -ne $gotPath) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } $reportDir = 'target/anvil/bench' $findingsMd = Join-Path $reportDir 'findings.md' $summaryMd = Join-Path $reportDir 'findings-summary.md' diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 21c8e9dfd..18649e481 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -899,6 +899,50 @@ fn bench_history_propagates_tool_failure() { assert_failed(&output, "a failing collect"); } +#[test] +fn bench_history_refuses_a_store_the_wiring_does_not_publish() { + if !tools_available() { + return; + } + + // The wiring announces the one path it restores into and publishes from. + // A store pointing anywhere else would never persist, so every run would + // cold-start and analyze to a clean no-op -- reporting green exactly when + // the history needed to report red has been lost. + let (_tmp, output) = run_bench_history( + NO_FINDINGS, + &[ + ("ANVIL_BENCH_WIRED_STORE", OsStr::new("target/anvil/bench-history")), + ("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/somewhere-else")), + ], + ); + assert_failed(&output, "a store the wiring does not publish"); + + // Agreement is not a desync, however it is spelled: the comparison is on + // the resolved path, not the literal string. + let (_tmp, output) = run_bench_history( + NO_FINDINGS, + &[ + ("ANVIL_BENCH_WIRED_STORE", OsStr::new("target/anvil/bench-history")), + ("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/anvil/../anvil/bench-history")), + ], + ); + assert!( + output.status.success(), + "the same path spelled differently is not a desync:{}", + both_streams(&output) + ); + + // Without wiring there is nothing to disagree with: a local run may put + // its store wherever it likes. + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/somewhere-else"))]); + assert!( + output.status.success(), + "an unwired run may choose its own store:{}", + both_streams(&output) + ); +} + /// Runs the private blessing reconciliation directly, so the prefix-matching /// boundary is pinned without going through a whole analysis. fn run_bless(blessings_file: &str, applied: &str) -> (TempDir, Output) { diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index 9570abff0..7a2725a62 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -1838,7 +1838,10 @@ steps: } # Everything lands here first; $path is created only on success. - $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging" + # $PID-scoped: on a self-hosted agent two concurrent jobs share the + # temp directory, and an artifact name is not unique across + # overlapping pipeline runs. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging-$PID" Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $staging | Out-Null @@ -1851,6 +1854,10 @@ steps: Copy-Item -Path (Join-Path $staging '*') -Destination $path -Recurse -Force } Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + # The recipe refuses to write anywhere but here, so a wrapper that + # retargets `path` cannot silently detach the recipe's store from + # the one that is actually published. + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_WIRED_STORE]$path" } $collection = $env:SYSTEM_COLLECTIONURI @@ -1883,8 +1890,8 @@ steps: if (-not $found.resource.downloadUrl) { continue } Write-Host "anvil: restoring $artifact from build $($run.id)" - $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact.zip" - $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract" + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-$PID.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract-$PID" Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force @@ -3124,7 +3131,29 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs anvil-bench-history: anvil-bench-history-validate-prereqs $ErrorActionPreference = 'Stop' - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + if ($wantPath -ne $gotPath) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } $reportDir = 'target/anvil/bench' $findingsMd = Join-Path $reportDir 'findings.md' $summaryMd = Join-Path $reportDir 'findings-summary.md' diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 2f064d5cc..fdba2b6c2 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1712,6 +1712,9 @@ runs: cp -R "$staging/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach the recipe's store from the published one. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" } # Walk back from the newest run and take the first that carries this @@ -2494,7 +2497,7 @@ on: # The caller grants the maximum token scopes available to this reusable # workflow. Reset jobs to read-only here, then restore only the publisher's -# issues scope below. See docs/design/github.md §9. +# issues scope below. See docs/design/github.md. permissions: contents: read @@ -2707,7 +2710,7 @@ jobs: uses: ./.github/workflows/anvil-scheduled-impl.yml # A called workflow cannot elevate beyond its caller. The implementation # resets this upper bound to read-only and restores issues:write only on - # publish-failure. See docs/design/github.md §9. + # publish-failure. See docs/design/github.md. permissions: contents: read # The scheduled-benchmarks job restores its history artifact, which @@ -3036,7 +3039,29 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs anvil-bench-history: anvil-bench-history-validate-prereqs $ErrorActionPreference = 'Stop' - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + if ($wantPath -ne $gotPath) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } $reportDir = 'target/anvil/bench' $findingsMd = Join-Path $reportDir 'findings.md' $summaryMd = Join-Path $reportDir 'findings-summary.md' diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index 2855dd17b..88af7ae7f 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -1549,7 +1549,29 @@ anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs anvil-bench-history: anvil-bench-history-validate-prereqs $ErrorActionPreference = 'Stop' - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + if ($wantPath -ne $gotPath) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } $reportDir = 'target/anvil/bench' $findingsMd = Join-Path $reportDir 'findings.md' $summaryMd = Join-Path $reportDir 'findings-summary.md' diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just index 64c426d30..8ee6e15bc 100644 --- a/justfiles/anvil/checks/bench-history.just +++ b/justfiles/anvil/checks/bench-history.just @@ -43,7 +43,29 @@ anvil-bench-history: anvil-bench-history-validate-prereqs $ErrorActionPreference = 'Stop' - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { $env:ANVIL_BENCH_HISTORY_STORE } else { 'target/anvil/bench-history' } + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + if ($wantPath -ne $gotPath) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } $reportDir = 'target/anvil/bench' $findingsMd = Join-Path $reportDir 'findings.md' $summaryMd = Join-Path $reportDir 'findings-summary.md' From a3b4973e470dfe752f4050e026c0da3a6f045022 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Thu, 27 Aug 2026 19:40:52 +0200 Subject: [PATCH 18/24] docs(cargo-anvil): record the ADO artifact-retention asymmetry Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/docs/design/benchmarks.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md index 812144e17..82f178a6e 100644 --- a/crates/cargo-anvil/docs/design/benchmarks.md +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -99,6 +99,16 @@ restore/save building blocks live in [github.md](./github.md) and [ado.md](./ado.md). cbh's own durable backends (for example Azure Blob) are outside anvil's scope: the artifact rolling window is the supported store. +The window's *depth* is not symmetric between backends, and cannot be made so +from here. GitHub's save sets an explicit retention, so the horizon is a +property of the emitted workflow. ADO publishes through the job wrapper's +artifact contract, which exposes no retention control, so the horizon there is +whatever the project or organization build-retention policy grants — commonly +shorter. An ADO chain can therefore lapse sooner than a GitHub one for the same +schedule. This is a visible cold start rather than a silent wrong answer, so it +is documented as a caveat rather than worked around: an adopter who needs a +guaranteed horizon on ADO raises the project's build retention. + ## 5. Surfacing: failing the scheduled build An active regression fails the scheduled build; the findings — each benchmark, From 4837202ca86a48ca5374d333096255021fb7c4cb Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 10:54:53 +0200 Subject: [PATCH 19/24] refactor(cargo-anvil): make the history round-trip an input of the shared group action Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 6 +- .github/actions/anvil-run-group/action.yml | 146 +++++++++ .github/workflows/anvil-scheduled-impl.yml | 132 +-------- crates/cargo-anvil/docs/design/github.md | 21 +- .../cargo-anvil/src/anvil/artifacts/github.rs | 66 +++-- .../templates/github/run-group-action.yml | 146 +++++++++ .../github/scheduled-impl-workflow.yml | 132 +-------- crates/cargo-anvil/tests/recipe_contracts.rs | 2 +- .../snapshots/snapshots__github_backend.snap | 278 ++++++++++-------- 9 files changed, 524 insertions(+), 405 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index 537c86431..16fc419d7 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:145ce9fca2dbb27cda90cde5e133b1d9afeca33b37f7a099aaa79b4f0e56eb45" +catalog_checksum = "sha256:f7a9769186228a682b3fb625c60c09a8118d4f3ddad342571acca8026bc72885" [[file]] path = ".anvil/container/Containerfile" @@ -45,7 +45,7 @@ checksum = "sha256:9940d1947482150ac08fcb9b4150da99f5ae60642f4caeea137577ce0e709 [[file]] path = ".github/actions/anvil-run-group/action.yml" -checksum = "sha256:b6ea795dbdc88145a7cc02199134e95cd8baa052f95a8913308630bbd3928a10" +checksum = "sha256:05c15dc3ab40b3d06cf4bd673bfe4e1e919880b0678339a8236ceabe59b1a0e5" [[file]] path = ".github/actions/anvil-setup/action.yml" @@ -65,7 +65,7 @@ checksum = "sha256:0c2530d9a38e6a74e0a7fd4f999b4a1790f97de30b58b68c6c2344600da19 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:89493c9a2d6c18b04d4e1ed0a0b9daf86c8ab28d82251e5f844682f7021faca9" +checksum = "sha256:c646ea2c791d4c62104bb790439ed2d46a8c7ea2b870bb6a344f7f237d4654b3" [[file]] path = ".github/workflows/anvil-scheduled.yml" diff --git a/.github/actions/anvil-run-group/action.yml b/.github/actions/anvil-run-group/action.yml index c87681dc0..486266786 100644 --- a/.github/actions/anvil-run-group/action.yml +++ b/.github/actions/anvil-run-group/action.yml @@ -42,6 +42,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -51,6 +74,101 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -60,6 +178,7 @@ runs: ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -82,6 +201,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index cb8aace14..79ded336f 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -144,10 +144,6 @@ jobs: group: scheduled-exhaustive scheduled-benchmarks: - # The history round-trip is written out here rather than hidden behind a - # per-group action: only the workflow knows the matrix leg, and the - # artifact name has to carry it so two runners' samples never merge into - # one series. strategy: fail-fast: false matrix: @@ -155,136 +151,26 @@ jobs: runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} permissions: contents: read - # Restoring walks the Actions runs/artifacts API. + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. actions: read - env: - ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} - ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # The analysis orders each series by first-parent commit topology # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. fetch-depth: 0 lfs: true - - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - - # Staged first. The store path is created only once the restore has - # reached a known state, so an operational failure leaves no store - # at all and a publisher that runs unconditionally has nothing to - # upload over the accumulated chain. - staging="$(mktemp -d)" - - complete_restore() { - mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ - fi - echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" - # The recipe refuses to write anywhere but here, so an override - # cannot silently detach it from the store that is published. - echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" - } - - # Assigned rather than consumed directly by `for`: `set -e` ignores - # the exit status of a command substitution used as a word list, so - # a failed listing would yield an empty list and fall through to a - # cold start -- publishing a truncated store over the chain and - # reporting green for want of the history needed to report red. - if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId')"; then - echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ - "refusing to continue, since treating this as a cold start" \ - "would publish a truncated store over the existing chain." - exit 1 - fi - - # Walk back from the newest run and take the first that carries this - # leg's artifact. Restoring from the latest *successful* run would - # drop every sample collected while the pipeline was red from a - # regression -- precisely the window that matters. - # - # Absence and failure are kept distinct. A run is only a candidate - # once the artifacts API confirms the artifact exists and has not - # expired; a download that then fails is an operational error - # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. - for run_id in $run_ids; do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - complete_restore restored - exit 0 - done - - # No run in the window carried the artifact: a genuine cold start - # (first run, or the chain lapsed), which is a valid empty store. - # Surfaced on the summary rather than only in this log -- "history - # quietly restarted" must not look like "no regressions". - complete_restore cold-start - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - - uses: ./.github/actions/anvil-run-group with: group: scheduled-benchmarks - - - name: Save benchmark history - # always(): the run's own samples belong in the history even when the - # analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the restore - # failed operationally the store is not a continuation of the chain, - # and publishing it would overwrite good history with a truncated - # snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused or - # infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} publish-failure: name: Publish scheduled failure diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index 58867511b..1d7b7166d 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -21,8 +21,8 @@ need to change: the concrete failure without duplicating group membership. See [Failure attribution and commit statuses](#failure-attribution-and-commit-statuses). A group needing steps around the runner (the benchmark group's history - round-trip) carries them in its own job, since only the workflow knows the - matrix leg those steps have to name. + round-trip) turns them on with an input, so the workflows stay a plain + list of groups and the other groups skip the steps. See also: @@ -1035,10 +1035,19 @@ The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs **Actions artifacts**. The history is partitioned per machine, so each leg of the group's matrix carries its own artifact (`bench-history-`). -The round-trip lives in the group's **composite action**, not in the workflow: the -scheduled workflow's benchmark job is a checkout plus the group action, like every -other job. Only job-level concerns stay in the workflow — the matrix, the -`actions: read` grant, the full-depth checkout, and the machine-key input. +The round-trip lives in the **shared group action**, behind an input, rather than +in the workflow or in an action of its own. Actions are the only reuse primitive +that shares a runner with the group run, which restore and save must: a reusable +workflow would put them on a different machine from the store they manage. +Groups that leave the input off skip the steps entirely, so the scheduled +workflow stays a plain list of groups. + +The action cannot supply everything, and the remainder is exactly what GitHub +scopes to the job: an action cannot request `permissions`, and the checkout has +already happened before it starts. So the `actions: read` grant and the +full-depth checkout stay in the workflow, along with the matrix. The artifact +name is passed in for the same reason — the matrix value is in scope only at the +call site. Each scheduled benchmark job: diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index ed0a7c1dd..fb19a5d38 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -459,46 +459,64 @@ export -f just #[test] fn scheduled_benchmarks_job_round_trips_the_history_artifact() { - // The round-trip lives in the job rather than behind a shared action: - // only the workflow knows the matrix leg, and a composite action - // cannot read the caller's `env` context through an expression. + // The round-trip is a parameter of the shared group action, not a + // second way to run a group: the eight other jobs skip it, and the + // scheduled workflow stays a plain list of groups. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_history: true")); + // Per-leg identity is supplied by the caller. The matrix value is in + // scope only in the workflow, and merging two runners' samples into + // one series would destroy the comparison. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_artifact: bench-history-${{ matrix.os }}")); + assert_eq!( + SCHEDULED_IMPL_WORKFLOW.matches("bench_history: true").count(), + 1, + "only the benchmark group pays for the history round-trip" + ); + // Neither of these can be expressed inside an action: an action + // cannot request permissions, and the checkout has already run by + // the time it starts. They are the whole per-group remainder. assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }}")); - // The upload's name is a literal expression, not an env lookup: - // upload-artifact's `with:` is evaluated where `matrix` is in scope. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("name: bench-history-${{ matrix.os }}")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions/upload-artifact@")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("gh run download")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("GITHUB_STEP_SUMMARY")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions: read")); + assert!(SCHEDULED_ROOT_WORKFLOW.contains("actions: read")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_machine_key:")); + + // Everything else lives in the action, guarded so the other groups + // skip it. + assert!(RUN_GROUP_ACTION.contains("if: inputs.bench_history == 'true'")); + assert!(RUN_GROUP_ACTION.contains("ARTIFACT: ${{ inputs.bench_artifact }}")); + assert!(RUN_GROUP_ACTION.contains("name: ${{ inputs.bench_artifact }}")); + assert!(RUN_GROUP_ACTION.contains("ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }}")); + assert!(RUN_GROUP_ACTION.contains("actions/upload-artifact@")); + assert!(RUN_GROUP_ACTION.contains("gh run download")); + assert!(RUN_GROUP_ACTION.contains("GITHUB_STEP_SUMMARY")); + // An enabled round-trip with no artifact name would restore nothing + // and publish nothing, reporting a false clean every run. + assert!(RUN_GROUP_ACTION.contains("if [ -z \"$ARTIFACT\" ]; then")); // The workflow is identified by its runtime name, not a literal // filename: the root workflow is owned and renameable, and a rename // must not silently reset the series. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("WORKFLOW: ${{ github.workflow }}")); + assert!(RUN_GROUP_ACTION.contains("WORKFLOW: ${{ github.workflow }}")); assert!( - !SCHEDULED_IMPL_WORKFLOW.contains("--workflow anvil-scheduled.yml"), + !RUN_GROUP_ACTION.contains("--workflow anvil-scheduled.yml"), "a hardcoded workflow filename breaks on rename" ); // Absence and operational failure must stay distinguishable. The run // listing is assigned rather than consumed by `for`, because `set -e` // ignores a command substitution used as a word list -- a failed // listing would otherwise read as a cold start. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("if ! run_ids=")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("complete_restore restored")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("complete_restore cold-start")); + assert!(RUN_GROUP_ACTION.contains("if ! run_ids=")); + assert!(RUN_GROUP_ACTION.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); + assert!(RUN_GROUP_ACTION.contains("complete_restore restored")); + assert!(RUN_GROUP_ACTION.contains("complete_restore cold-start")); // Fail-closed by construction: the store path is created only inside // complete_restore, so an operational failure leaves nothing to // upload over the accumulated chain. assert_eq!( - SCHEDULED_IMPL_WORKFLOW.matches("mkdir -p target/anvil/bench-history").count(), + RUN_GROUP_ACTION.matches("mkdir -p target/anvil/bench-history").count(), 1, "the store path must be created only on a completed restore" ); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("if: always() && env.ANVIL_BENCH_RESTORE != ''")); - // The machine-key escape hatch has to be reachable in CI, which - // workflow-level env is not across a called reusable workflow. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_machine_key:")); - assert!(SCHEDULED_IMPL_WORKFLOW.contains("ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }}")); + assert!(RUN_GROUP_ACTION.contains("if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != ''")); // Notifying a human is the scheduled tier's own publish-failure job, // which this group must be a dependency of -- otherwise a benchmark // regression fails the run without ever reaching the tracking issue. @@ -512,10 +530,6 @@ export -f just publish_needs.contains("- scheduled-benchmarks"), "publish-failure must depend on scheduled-benchmarks:\n{publish_needs}" ); - // Restoring the history reads the runs/artifacts API, and a - // reusable workflow cannot grant itself more than its caller. - assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions: read")); - assert!(SCHEDULED_ROOT_WORKFLOW.contains("actions: read")); } #[test] diff --git a/crates/cargo-anvil/templates/github/run-group-action.yml b/crates/cargo-anvil/templates/github/run-group-action.yml index c87681dc0..486266786 100644 --- a/crates/cargo-anvil/templates/github/run-group-action.yml +++ b/crates/cargo-anvil/templates/github/run-group-action.yml @@ -42,6 +42,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -51,6 +74,101 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -60,6 +178,7 @@ runs: ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -82,6 +201,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index cb8aace14..79ded336f 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -144,10 +144,6 @@ jobs: group: scheduled-exhaustive scheduled-benchmarks: - # The history round-trip is written out here rather than hidden behind a - # per-group action: only the workflow knows the matrix leg, and the - # artifact name has to carry it so two runners' samples never merge into - # one series. strategy: fail-fast: false matrix: @@ -155,136 +151,26 @@ jobs: runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} permissions: contents: read - # Restoring walks the Actions runs/artifacts API. + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. actions: read - env: - ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} - ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # The analysis orders each series by first-parent commit topology # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. fetch-depth: 0 lfs: true - - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - - # Staged first. The store path is created only once the restore has - # reached a known state, so an operational failure leaves no store - # at all and a publisher that runs unconditionally has nothing to - # upload over the accumulated chain. - staging="$(mktemp -d)" - - complete_restore() { - mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ - fi - echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" - # The recipe refuses to write anywhere but here, so an override - # cannot silently detach it from the store that is published. - echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" - } - - # Assigned rather than consumed directly by `for`: `set -e` ignores - # the exit status of a command substitution used as a word list, so - # a failed listing would yield an empty list and fall through to a - # cold start -- publishing a truncated store over the chain and - # reporting green for want of the history needed to report red. - if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId')"; then - echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ - "refusing to continue, since treating this as a cold start" \ - "would publish a truncated store over the existing chain." - exit 1 - fi - - # Walk back from the newest run and take the first that carries this - # leg's artifact. Restoring from the latest *successful* run would - # drop every sample collected while the pipeline was red from a - # regression -- precisely the window that matters. - # - # Absence and failure are kept distinct. A run is only a candidate - # once the artifacts API confirms the artifact exists and has not - # expired; a download that then fails is an operational error - # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. - for run_id in $run_ids; do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - complete_restore restored - exit 0 - done - - # No run in the window carried the artifact: a genuine cold start - # (first run, or the chain lapsed), which is a valid empty store. - # Surfaced on the summary rather than only in this log -- "history - # quietly restarted" must not look like "no regressions". - complete_restore cold-start - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - - uses: ./.github/actions/anvil-run-group with: group: scheduled-benchmarks - - - name: Save benchmark history - # always(): the run's own samples belong in the history even when the - # analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the restore - # failed operationally the store is not a continuation of the chain, - # and publishing it would overwrite good history with a truncated - # snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused or - # infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} publish-failure: name: Publish scheduled failure diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 35cebd5a8..1b083ed03 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -1116,7 +1116,7 @@ fn bench_history_bless_rejects_malformed_entries() { // transports. // --------------------------------------------------------------------------- -const GH_BENCH_RESTORE: &str = include_str!("../templates/github/scheduled-impl-workflow.yml"); +const GH_BENCH_RESTORE: &str = include_str!("../templates/github/run-group-action.yml"); const ADO_RESTORE: &str = include_str!("../templates/ado/steps/bench-history-restore.yml"); /// Extracts a block scalar (`run: |` / `pwsh: |`) from `yaml`, starting the diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 962cb0483..17eb17b51 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1538,6 +1538,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -1547,6 +1570,101 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then + cp -R "$staging/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -1556,6 +1674,7 @@ runs: ANVIL_INCLUDE_MODIFIED: ${{ inputs.include_modified }} ANVIL_INCLUDE_AFFECTED: ${{ inputs.include_affected }} ANVIL_INCLUDE_REQUIRED: ${{ inputs.include_required }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -1578,6 +1697,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true @@ -2302,10 +2448,6 @@ jobs: group: scheduled-exhaustive scheduled-benchmarks: - # The history round-trip is written out here rather than hidden behind a - # per-group action: only the workflow knows the matrix leg, and the - # artifact name has to carry it so two runners' samples never merge into - # one series. strategy: fail-fast: false matrix: @@ -2313,136 +2455,26 @@ jobs: runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} permissions: contents: read - # Restoring walks the Actions runs/artifacts API. + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. actions: read - env: - ANVIL_BENCH_ARTIFACT: bench-history-${{ matrix.os }} - ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # The analysis orders each series by first-parent commit topology # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. fetch-depth: 0 lfs: true - - - name: Restore benchmark history - shell: bash - env: - GH_TOKEN: ${{ github.token }} - ARTIFACT: ${{ env.ANVIL_BENCH_ARTIFACT }} - DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - # The caller's workflow *name*, not a hardcoded filename: the root - # scheduled workflow is an owned, renameable file, and a rename - # must not silently reset the series. - WORKFLOW: ${{ github.workflow }} - REPO: ${{ github.repository }} - WINDOW: "30" - run: | - set -euo pipefail - - # Staged first. The store path is created only once the restore has - # reached a known state, so an operational failure leaves no store - # at all and a publisher that runs unconditionally has nothing to - # upload over the accumulated chain. - staging="$(mktemp -d)" - - complete_restore() { - mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ - fi - echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" - # The recipe refuses to write anywhere but here, so an override - # cannot silently detach it from the store that is published. - echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" - } - - # Assigned rather than consumed directly by `for`: `set -e` ignores - # the exit status of a command substitution used as a word list, so - # a failed listing would yield an empty list and fall through to a - # cold start -- publishing a truncated store over the chain and - # reporting green for want of the history needed to report red. - if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ - --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ - --json databaseId --jq '.[].databaseId')"; then - echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ - "refusing to continue, since treating this as a cold start" \ - "would publish a truncated store over the existing chain." - exit 1 - fi - - # Walk back from the newest run and take the first that carries this - # leg's artifact. Restoring from the latest *successful* run would - # drop every sample collected while the pipeline was red from a - # regression -- precisely the window that matters. - # - # Absence and failure are kept distinct. A run is only a candidate - # once the artifacts API confirms the artifact exists and has not - # expired; a download that then fails is an operational error - # (token, API, corrupt payload) and fails the job rather than being - # silently downgraded to a cold start. - for run_id in $run_ids; do - artifact_id=$(gh api --paginate \ - "repos/$REPO/actions/runs/$run_id/artifacts" \ - --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ - | head -n1) - [ -n "$artifact_id" ] || continue - - if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then - echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ - "failing rather than continuing with an empty history, which would" \ - "publish a truncated store over the existing chain." - exit 1 - fi - echo "restored benchmark history from run $run_id" - complete_restore restored - exit 0 - done - - # No run in the window carried the artifact: a genuine cold start - # (first run, or the chain lapsed), which is a valid empty store. - # Surfaced on the summary rather than only in this log -- "history - # quietly restarted" must not look like "no regressions". - complete_restore cold-start - echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" - { - printf '### Benchmark history: cold start\n\n' - printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ - "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" - printf 'so this run starts a new series. Trend detection needs several runs of history.\n' - } >> "$GITHUB_STEP_SUMMARY" - - uses: ./.github/actions/anvil-run-group with: group: scheduled-benchmarks - - - name: Save benchmark history - # always(): the run's own samples belong in the history even when the - # analysis flagged a regression and failed the job. - # - # Guarded on the restore having reached a known state: if the restore - # failed operationally the store is not a continuation of the chain, - # and publishing it would overwrite good history with a truncated - # snapshot. - if: always() && env.ANVIL_BENCH_RESTORE != '' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bench-history-${{ matrix.os }} - path: target/anvil/bench-history - # Comfortably longer than the scheduled cadence, so a paused or - # infrequent schedule does not break the chain. - retention-days: 90 - if-no-files-found: ignore - - - name: Publish benchmark findings - if: always() - shell: bash - run: | - set -euo pipefail - if [ -f target/anvil/bench/findings.md ]; then - cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" - fi + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} publish-failure: name: Publish scheduled failure From 7870c578ca25b60108300b649e95dc5e1779a61d Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 17:32:33 +0200 Subject: [PATCH 20/24] chore(cargo-anvil): regenerate after merging main --- .anvil.lock | 8 ++++---- crates/cargo-anvil/README.md | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index bf3c42a22..fd7d1c31f 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:8214d406143dc17e686646c39e95841fa8450224fe19b594f53476fe117ddabb" +catalog_checksum = "sha256:ae25040fc8a1473c7ecb281bb9ab2c250a6f98e7fb1367417cf40bfc52b93e97" [[file]] path = ".anvil/container/Containerfile" @@ -89,7 +89,7 @@ checksum = "sha256:50f04b4ea6c99df8ad7434d320f34dccdb6db6a77b83e3090e35de0ca3a15 [[file]] path = "justfiles/anvil/checks/bolero.just" -checksum = "sha256:003032f39c781851b880c1d7e63b93573f5252cca9f785b98b04dfa2e858aff3" +checksum = "sha256:754827bb664723169b8d48a6f1e69f9122b5440ba0b94a91622f4d23313e2503" [[file]] path = "justfiles/anvil/checks/careful.just" @@ -181,7 +181,7 @@ checksum = "sha256:28734aa77526ca5c53d89a32c3e20ae6b42d2032c73ab6a1dbc9831f25cb0 [[file]] path = "justfiles/anvil/checks/readme-check.just" -checksum = "sha256:d346399f288570066e53fd123baf05f7d4a57f17da8ee687a6d881204c5bae12" +checksum = "sha256:6d4b4c3e4a59e825a3f613e1011272c711d8d6cbdef315e5f6aadf653c6ae206" [[file]] path = "justfiles/anvil/checks/semver-check.just" @@ -265,7 +265,7 @@ checksum = "sha256:d7dc849363748f40df2768f13b364a3237767eda84525a1c901d4caddbace [[file]] path = "justfiles/anvil/versions.just" -checksum = "sha256:d7f264273b95169aff37303746c01cfe12379a82bfd772610a4deb7880ea4821" +checksum = "sha256:69d843dd7fdfee808d74d46485ab3b290a839418db390233eeb89ef8893e9f5d" [[region]] host = ".delta.toml" diff --git a/crates/cargo-anvil/README.md b/crates/cargo-anvil/README.md index a656e95e4..e7a7b6ab7 100644 --- a/crates/cargo-anvil/README.md +++ b/crates/cargo-anvil/README.md @@ -281,6 +281,7 @@ schedule against the default branch, not on PRs: scheduled-exhaustivemutants-full cargo-hackfeature powerset benchcompile-only + scheduled-benchmarksbench-historyregression detection over the accumulated benchmark history @@ -445,7 +446,7 @@ And `docs/verification.md` for the continuous-validation strategy. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQblcBzF-_WZVYbCN9Rt1pYQLsblkUTM0oENsMbNe4wSAldeq9hZIGDa2NhcmdvLWFudmlsZTAuNS4wa2NhcmdvX2Fudmls + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQbYF1H1HhNkR8b0O8-IuuMC_kbb7jYeYiyz8sbR_Uu9yf0xCdhZIGDa2NhcmdvLWFudmlsZTAuNS4wa2NhcmdvX2Fudmls [__link0]: https://crates.io/crates/cargo-delta [__link1]: https://crates.io/crates/cargo-spellcheck [__link2]: https://crates.io/crates/cargo-coverage-gate From 75974da5b4c285d0139bcbdb00229008865ea6b2 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 19:48:10 +0200 Subject: [PATCH 21/24] fix(cargo-anvil): guard the empty-artifact restore copy and read CI markers as boolean-like Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 4 +- .../ado/steps/bench-history-restore.yml | 9 +- .../justfiles/anvil/checks/bench-history.just | 12 +- crates/cargo-anvil/tests/recipe_contracts.rs | 34 ++ .../snapshots/snapshots__ado_backend.snap | 21 +- .../snapshots/snapshots__github_backend.snap | 12 +- .../snapshots/snapshots__local_only.snap | 12 +- justfiles/anvil/checks/bench-history.just | 16 +- .../checks/bench-history.just.anvil-proposed | 310 ------------------ .../anvil/groups/scheduled-benchmarks.just | 9 +- .../scheduled-benchmarks.just.anvil-proposed | 34 -- 11 files changed, 115 insertions(+), 358 deletions(-) delete mode 100644 justfiles/anvil/checks/bench-history.just.anvil-proposed delete mode 100644 justfiles/anvil/groups/scheduled-benchmarks.just.anvil-proposed diff --git a/.anvil.lock b/.anvil.lock index a58ecd148..06fd4c36e 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:7e46faa7014f444f7b8f2cf74f3f43230332b69c0dd96caacbed9bb94210bf39" +catalog_checksum = "sha256:ba4d919f3a60199f357b4d5d897c26a8326dcd852c5729707db5bfaa0ae49357" [[file]] path = ".anvil/container/Dockerfile.dockerignore" @@ -53,7 +53,7 @@ checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536 [[file]] path = "justfiles/anvil/checks/bench-history.just" -checksum = "sha256:3ffc3f95f69abed8902e8849afb7da67c979f611411c448cb2f25125f4b5b9a8" +checksum = "sha256:ff030b67dcedd1e1d73264b5fb32d86ba5d83dc2ba539cfa063fe5ad45bcd3b8" [[file]] path = "justfiles/anvil/checks/bench.just" diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml index 46e7c5814..d32bf2bba 100644 --- a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -125,10 +125,15 @@ steps: Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force # The archive nests its contents under a directory named for the - # artifact; lift them up into the staging directory. + # artifact; lift them up into the staging directory. An artifact + # that is present but empty is a valid restore -- a workspace with + # no benchmarks yet publishes exactly that -- and `Copy-Item` over + # an empty wildcard throws, so the copy is guarded. $inner = Join-Path $extract $artifact $source = if (Test-Path $inner) { $inner } else { $extract } - Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + if (Test-Path (Join-Path $source '*')) { + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + } Complete-Restore 'restored' exit 0 } diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just index 029f5cfc2..2e65f9dee 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -131,7 +131,17 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if (-not $value) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) if (-not $gate) { Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 58b8ade0c..abe92a07a 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -1629,6 +1629,40 @@ fn bench_history_reports_without_gating_outside_ci() { assert!(text.contains("ANVIL_BENCH_GATE"), "points at the opt-in:\n{text}"); } +#[test] +fn bench_history_gate_reads_ci_markers_as_boolean_like() { + if !tools_available() { + return; + } + // PowerShell treats every non-empty string as true, so an exported + // CI=false would otherwise gate a local run on a shared trend measured + // on that developer's own hardware. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new("false")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "CI=false must not gate a local run:{}", + both_streams(&output) + ); + + // Anything else non-empty still gates: the CI side stays fail-closed. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new("true")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert_failed(&output, "CI=true"); +} + #[test] fn bench_history_propagates_tool_failure() { if !tools_available() { diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index 4f2ffe118..f246fd216 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -857,10 +857,15 @@ steps: Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force # The archive nests its contents under a directory named for the - # artifact; lift them up into the staging directory. + # artifact; lift them up into the staging directory. An artifact + # that is present but empty is a valid restore -- a workspace with + # no benchmarks yet publishes exactly that -- and `Copy-Item` over + # an empty wildcard throws, so the copy is guarded. $inner = Join-Path $extract $artifact $source = if (Test-Path $inner) { $inner } else { $extract } - Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + if (Test-Path (Join-Path $source '*')) { + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + } Complete-Restore 'restored' exit 0 } @@ -2194,7 +2199,17 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if (-not $value) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) if (-not $gate) { Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 36e50054a..4bf6f3832 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1902,7 +1902,17 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if (-not $value) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) if (-not $gate) { Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index b70077466..41f600f48 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -588,7 +588,17 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if (-not $value) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) if (-not $gate) { Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just index 2ee34bf83..2e65f9dee 100644 --- a/justfiles/anvil/checks/bench-history.just +++ b/justfiles/anvil/checks/bench-history.just @@ -9,8 +9,8 @@ # Unscoped by design. A benchmark's series is only comparable when the # same suite is measured at every commit, so impact-scoping the run would # punch holes in the history that detection cannot distinguish from a -# benchmark being deleted. The recipe therefore ignores the -# ANVIL_INCLUDE_* contract and always measures the whole workspace. +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. # # Environment contract (all optional): # ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). @@ -131,7 +131,17 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if (-not $value) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) if (-not $gate) { Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' diff --git a/justfiles/anvil/checks/bench-history.just.anvil-proposed b/justfiles/anvil/checks/bench-history.just.anvil-proposed deleted file mode 100644 index 029f5cfc2..000000000 --- a/justfiles/anvil/checks/bench-history.just.anvil-proposed +++ /dev/null @@ -1,310 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md - -# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md - -# Unscoped by design. A benchmark's series is only comparable when the -# same suite is measured at every commit, so impact-scoping the run would -# punch holes in the history that detection cannot distinguish from a -# benchmark being deleted. The recipe therefore ignores impact scoping and -# always measures the whole workspace. -# -# Environment contract (all optional): -# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). -# Local-only: the generated cloud wiring restores and -# publishes the default path, so overriding it in CI -# would leave the recipe reading a different directory -# than the one the artifact round-trip maintains. -# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, -# for pools heterogeneous enough to fragment a series. -# Both backends plumb this through their scheduled -# wiring, so it is settable in CI as well as locally. -# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe -# locally. CI sets it automatically; see below. -# -# The store is the cross-run state the cloud wiring restores before and -# publishes after this recipe; locally it is whatever has accumulated -# under target/, which on a fresh checkout is empty and analyzes to a -# clean no-op. -# -# Gating is CI-only by default. The recipe behaves identically either way -# --- it always runs the benches and always writes its findings --- but a -# laptop produces measurement noise that a shared, homogeneous runner pool -# does not, and `anvil-scheduled` / `anvil-full` are run locally before a -# release. Failing those on thermal throttling would invite committing a -# blessing to silence it, which would pollute the reviewed, audited -# blessings file with an artifact of one developer's hardware. - -# Run the benchmarks and analyze the accumulated history for regressions. -[script("pwsh", "-NoProfile")] -anvil-bench-history: anvil-bench-history-validate-prereqs - $ErrorActionPreference = 'Stop' - - # The CI wiring restores into, and publishes from, one fixed path. If - # the recipe wrote anywhere else the store would never persist: every - # run would cold-start and analyze to a clean no-op, reporting green - # precisely when it has lost the history it needs to report red. The - # wiring therefore announces its path and the recipe refuses to - # disagree with it. - $wired = $env:ANVIL_BENCH_WIRED_STORE - $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { - $env:ANVIL_BENCH_HISTORY_STORE - } elseif ($wired) { - $wired - } else { - 'target/anvil/bench-history' - } - - if ($wired) { - $wantPath = [System.IO.Path]::GetFullPath($wired) - $gotPath = [System.IO.Path]::GetFullPath($store) - # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on - # every platform, so on Linux two genuinely different directories would - # compare equal and the detached-store failure this guard exists to - # catch would slip through. - $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } - if (-not $same) { - Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." - exit 1 - } - } - $reportDir = 'target/anvil/bench' - $findingsMd = Join-Path $reportDir 'findings.md' - $summaryMd = Join-Path $reportDir 'findings-summary.md' - $findingsJson = Join-Path $reportDir 'findings.json' - $blessingsFile = '.config/bench-blessings.toml' - - [System.IO.Directory]::CreateDirectory($store) | Out-Null - [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null - - # The machine key partitions every series. cargo-bench-history derives - # it from the host's hardware fingerprint; an adopter whose runner pool - # is heterogeneous enough to fragment the series into unanalyzable - # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label - # instead. It has to be the same on collect, bless, list and analyze, - # so every invocation below splats the same argument list. - $key = @() - if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } - - # --skip-existing makes a re-run at an already-recorded commit a - # success that writes nothing, so a re-queued scheduled build does not - # fail on the duplicate and does not overwrite the original sample. - Write-Host 'anvil-bench-history: collecting benchmark results' - & cargo bench-history collect --local="$store" --skip-existing --all-features @key - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - # Blessings accept an intentional change. They live in a reviewed, - # committed file and are applied into the store here, ahead of the - # analysis, so the store stays single-writer. - & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - Write-Host 'anvil-bench-history: analyzing history' - & cargo bench-history analyze --local="$store" ` - --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - - # Findings never affect cargo-bench-history's own exit code -- the - # machine-readable report is the signal. An *active* regression is the - # one thing that gates: an inactive finding has already recovered, and - # an improvement needs no action. - $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json - $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) - if ($regressions.Count -eq 0) { - Write-Host 'anvil-bench-history: no active regressions' - exit 0 - } - - Write-Host '' - Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red - foreach ($r in $regressions) { - $id = ($r.segments -join '/') - $delta = '{0:P2}' -f $r.relative_delta - Write-Host " $id ($($r.kind)) $delta at $($r.commit)" - } - Write-Host '' - Write-Host "Findings: $findingsMd" - - # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate - # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- - # only the exit code differs. - $gate = $env:ANVIL_BENCH_GATE -eq '1' -or $env:CI -or $env:TF_BUILD - if (-not $gate) { - Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow - Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' - Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' - exit 0 - } - - Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." - exit 1 - -# Apply the committed blessings into the history store, idempotently. -# -# `bless` writes an append-only sidecar into the store, so an entry that -# is already in effect must not be re-applied on every scheduled run. -# The already-applied set comes from `list blessings`, widened past the -# default look-back so an old entry is not mistaken for a missing one. -# -# The file is a table array; unknown keys are ignored so the schema can -# grow without breaking older tool pins: -# -# [[blessing]] -# benchmark = "my_pkg/my_group/my_case" -# commit = "8392995a" -# reason = "switched to the arena allocator; the extra setup is intentional" -[private] -[script("pwsh", "-NoProfile")] -_anvil-bench-history-bless store blessings: - $ErrorActionPreference = 'Stop' - $store = '{{store}}' - $blessingsFile = '{{blessings}}' - - if (-not (Test-Path -LiteralPath $blessingsFile)) { - Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" - exit 0 - } - - # A deliberately small TOML subset: `[[blessing]]` headers plus - # `key = "value"` pairs with no escapes. Anything outside it is - # rejected rather than reinterpreted, so a value this cannot represent - # fails loudly instead of being silently rewritten. The same subset is - # documented in the emitted file's own header. - $entries = New-Object System.Collections.Generic.List[object] - $current = $null - foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { - $line = $rawLine.Trim() - # A `#` only starts a comment outside a value. Stripping to the - # first `#` unconditionally would silently truncate a reason - # citing an issue or PR number -- exactly what a rationale - # contains -- so comments are only recognised at line start. - if (-not $line -or $line.StartsWith('#')) { continue } - if ($line -eq '[[blessing]]') { - $current = @{} - $entries.Add($current) | Out-Null - continue - } - if ($line -match '^\[') { - Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" - exit 1 - } - if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { - if ($null -eq $current) { - Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" - exit 1 - } - $current[$Matches[1]] = $Matches[2] - continue - } - Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) - exit 1 - } - - if ($entries.Count -eq 0) { - Write-Host "anvil-bench-history: $blessingsFile declares no blessings" - exit 0 - } - - foreach ($e in $entries) { - foreach ($required in @('benchmark', 'commit', 'reason')) { - if (-not $e[$required]) { - Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" - exit 1 - } - } - # `benchmark` and `commit` are passed to cbh and git. A leading `-` - # would be read as an option there, so reject it here rather than let - # the file's documented subset be reinterpreted: `benchmark = "--all"` - # would otherwise accept every benchmark at the commit while the log - # claimed a benchmark of that name. - foreach ($guarded in @('benchmark', 'commit')) { - if ($e[$guarded].StartsWith('-')) { - Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" - exit 1 - } - } - } - - $key = @() - if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } - - $tmpDir = $env:RUNNER_TEMP - if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } - if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } - # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted - # agent, or concurrent local runs) would otherwise race on one filename and - # read each other's listing. - $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" - - & cargo bench-history list blessings --all --local="$store" ` - --since 1970-01-01 --no-text --json $listJson @key - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) - - # A blessing can only be applied at a commit the store still holds a clean - # run for: cbh rejects a context commit with no data point, and `collect` - # only ever records the current commit, so a commit the store has forgotten - # can never be re-established by a later run. Without this, any cold start - # or artifact eviction past a blessed commit would turn the group - # permanently red until a human edited the ledger -- a blessing whose - # commit has aged out is inapplicable, not invalid. - $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" - & cargo bench-history list runs --local="$store" ` - --since 1970-01-01 --no-text --json $runsJson @key - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - $stored = [System.Collections.Generic.List[string]]::new() - foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { - foreach ($c in $set.commits) { - if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } - } - } - - foreach ($e in $entries) { - $commit = $e['commit'] - $benchmark = $e['benchmark'] - # Resolve to a full commit id up front: the file may carry an - # abbreviated id, and a bogus one should fail here with git's own - # message rather than silently bless nothing. - $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) - if ($LASTEXITCODE -ne 0 -or -not $resolved) { - Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" - exit 1 - } - $resolved = $resolved.Trim() - # Compare the persisted identity exactly. `list blessings --all` is the - # window view: it reports resolved, concrete benchmark ids and never - # populates `prefixes`. Since cbh's positional is a prefix, a ledger - # entry naming a family expands to several concrete rows and matches - # none of them, so such an entry is re-applied each run. That is the - # safe direction to be wrong in: re-blessing is idempotent in effect, - # whereas a loose prefix test would skip an entry that is only - # partially applied and leave the build red while claiming otherwise. - $already = $applied | Where-Object { - $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) - } - if ($already) { - Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" - continue - } - if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { - Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." - continue - } - Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" - # `--` so a benchmark id is never parsed as a flag: bless's positional - # takes no hyphen values, and it has an `--all` sibling that would - # otherwise accept every benchmark at the commit. - & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - } - -# Install prerequisites for the `anvil-bench-history` recipe. -[group("anvil-setup")] -anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) - -# Validate prerequisites for the `anvil-bench-history` recipe. -[group("anvil-setup")] -anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs diff --git a/justfiles/anvil/groups/scheduled-benchmarks.just b/justfiles/anvil/groups/scheduled-benchmarks.just index add6a09b1..9a4e119aa 100644 --- a/justfiles/anvil/groups/scheduled-benchmarks.just +++ b/justfiles/anvil/groups/scheduled-benchmarks.just @@ -10,10 +10,17 @@ # carried between runs. Keeping it in its own group isolates that history # round-trip and its fail-on-regression semantics from the rest of the # scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. # Run the scheduled benchmark regression detection. [group("anvil")] -anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ anvil-bench-history # Install prerequisites for the `anvil-scheduled-benchmarks` recipe. diff --git a/justfiles/anvil/groups/scheduled-benchmarks.just.anvil-proposed b/justfiles/anvil/groups/scheduled-benchmarks.just.anvil-proposed deleted file mode 100644 index 9a4e119aa..000000000 --- a/justfiles/anvil/groups/scheduled-benchmarks.just.anvil-proposed +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) Microsoft Corporation. -# Licensed under the MIT License. -# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. -# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. -# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md - -# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md - -# scheduled-benchmarks holds the one check whose verdict depends on state -# carried between runs. Keeping it in its own group isolates that history -# round-trip and its fail-on-regression semantics from the rest of the -# scheduled work, so a red build names the regression unambiguously. -# -# Routed through _anvil-unscoped like every scheduled group: the series only -# stays comparable if the same suite is measured at every commit, so scoping -# the run to a change set would silently break detection. - -# Run the scheduled benchmark regression detection. -[group("anvil")] -anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") - -[private] -_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ - anvil-bench-history - -# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. -[group("anvil-setup")] -anvil-scheduled-benchmarks-setup installer="install": \ - (anvil-bench-history-setup installer) - -# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. -[group("anvil-setup")] -anvil-scheduled-benchmarks-validate-prereqs: \ - anvil-bench-history-validate-prereqs From afe9a9a9c3795de69235832e3bd36b5040906601 Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 20:51:41 +0200 Subject: [PATCH 22/24] fix(cargo-anvil): treat a blank CI marker as unset Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 4 ++-- .../justfiles/anvil/checks/bench-history.just | 2 +- crates/cargo-anvil/tests/recipe_contracts.rs | 15 +++++++++++++++ justfiles/anvil/checks/bench-history.just | 2 +- 4 files changed, 19 insertions(+), 4 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index 06fd4c36e..c8e804209 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:ba4d919f3a60199f357b4d5d897c26a8326dcd852c5729707db5bfaa0ae49357" +catalog_checksum = "sha256:efb5685148ddff925d233577f497d4e76dc2570667eee47d44218b1370bf39f7" [[file]] path = ".anvil/container/Dockerfile.dockerignore" @@ -53,7 +53,7 @@ checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536 [[file]] path = "justfiles/anvil/checks/bench-history.just" -checksum = "sha256:ff030b67dcedd1e1d73264b5fb32d86ba5d83dc2ba539cfa063fe5ad45bcd3b8" +checksum = "sha256:d24d92356347c3215b9db87eaddac821e689d71708761624ad61e698e19d15dd" [[file]] path = "justfiles/anvil/checks/bench.just" diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just index 2e65f9dee..85e2ac7b2 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -138,7 +138,7 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # hardware. Unrecognised non-empty values still gate, keeping the CI side # fail-closed. function Test-Flag([string]$value) { - if (-not $value) { return $false } + if ([string]::IsNullOrWhiteSpace($value)) { return $false } return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') } $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index abe92a07a..969dde8bf 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -1651,6 +1651,21 @@ fn bench_history_gate_reads_ci_markers_as_boolean_like() { both_streams(&output) ); + // A present-but-blank marker is not a marker either. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new(" ")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "a whitespace-only marker must not gate:{}", + both_streams(&output) + ); + // Anything else non-empty still gates: the CI side stays fail-closed. let (_tmp, output) = run_bench_history( ACTIVE_REGRESSION, diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just index 2e65f9dee..85e2ac7b2 100644 --- a/justfiles/anvil/checks/bench-history.just +++ b/justfiles/anvil/checks/bench-history.just @@ -138,7 +138,7 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # hardware. Unrecognised non-empty values still gate, keeping the CI side # fail-closed. function Test-Flag([string]$value) { - if (-not $value) { return $false } + if ([string]::IsNullOrWhiteSpace($value)) { return $false } return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') } $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) From 0c8e47c9974baceac826992d8fde6a3f1e1e3bbf Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 20:57:09 +0200 Subject: [PATCH 23/24] test(cargo-anvil): refresh snapshots for the blank-marker gate Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap | 2 +- .../cargo-anvil/tests/snapshots/snapshots__github_backend.snap | 2 +- crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index f246fd216..3e1c1313b 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -2206,7 +2206,7 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # hardware. Unrecognised non-empty values still gate, keeping the CI side # fail-closed. function Test-Flag([string]$value) { - if (-not $value) { return $false } + if ([string]::IsNullOrWhiteSpace($value)) { return $false } return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') } $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 4bf6f3832..6e7054dd0 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -1909,7 +1909,7 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # hardware. Unrecognised non-empty values still gate, keeping the CI side # fail-closed. function Test-Flag([string]$value) { - if (-not $value) { return $false } + if ([string]::IsNullOrWhiteSpace($value)) { return $false } return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') } $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index 41f600f48..e9546889c 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -595,7 +595,7 @@ anvil-bench-history: anvil-bench-history-validate-prereqs # hardware. Unrecognised non-empty values still gate, keeping the CI side # fail-closed. function Test-Flag([string]$value) { - if (-not $value) { return $false } + if ([string]::IsNullOrWhiteSpace($value)) { return $false } return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') } $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) From cbd9b9f1c570739ab30b7ef366b0ab22cd796e5e Mon Sep 17 00:00:00 2001 From: "Martin Kolinek (from Dev Box)" Date: Mon, 31 Aug 2026 22:08:31 +0200 Subject: [PATCH 24/24] fix(cargo-anvil): tolerate a nested artifact layout when restoring history Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: a3ecc845-0526-499a-a233-6b82909c4517 --- .anvil.lock | 4 +- .github/actions/anvil-run-group/action.yml | 13 ++++- .../templates/github/run-group-action.yml | 13 ++++- crates/cargo-anvil/tests/recipe_contracts.rs | 48 ++++++++++++++++--- .../snapshots/snapshots__github_backend.snap | 13 ++++- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/.anvil.lock b/.anvil.lock index c8e804209..ed5007806 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:efb5685148ddff925d233577f497d4e76dc2570667eee47d44218b1370bf39f7" +catalog_checksum = "sha256:1a7aa05ccee55e5bad46ea953d460a3c4cad4e2737ccec514bfdfa4736558dc9" [[file]] path = ".anvil/container/Dockerfile.dockerignore" @@ -17,7 +17,7 @@ checksum = "sha256:9940d1947482150ac08fcb9b4150da99f5ae60642f4caeea137577ce0e709 [[file]] path = ".github/actions/anvil-run-group/action.yml" -checksum = "sha256:d637b1bdc00eb6601f0006f851064618a5ab2b2f0c5c8fd2c1abc07be18e5035" +checksum = "sha256:23a323e87f5ac0bb72a4e64e33692a71acf827956356e74698ac1b97a5e0da0b" [[file]] path = ".github/actions/anvil-setup/action.yml" diff --git a/.github/actions/anvil-run-group/action.yml b/.github/actions/anvil-run-group/action.yml index 2abccd40e..67257a906 100644 --- a/.github/actions/anvil-run-group/action.yml +++ b/.github/actions/anvil-run-group/action.yml @@ -97,8 +97,17 @@ runs: complete_restore() { mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" # The recipe refuses to write anywhere but here, so an override diff --git a/crates/cargo-anvil/templates/github/run-group-action.yml b/crates/cargo-anvil/templates/github/run-group-action.yml index 2abccd40e..67257a906 100644 --- a/crates/cargo-anvil/templates/github/run-group-action.yml +++ b/crates/cargo-anvil/templates/github/run-group-action.yml @@ -97,8 +97,17 @@ runs: complete_restore() { mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" # The recipe refuses to write anywhere but here, so an override diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index 969dde8bf..d20b9875b 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -1930,7 +1930,13 @@ fn git_bash() -> Option<&'static str> { /// /// `runs` are the run ids the listing yields, newest first; `artifact_runs` /// are those the artifacts API reports as carrying the artifact. -fn run_github_restore(runs: &str, artifact_runs: &str, download_exit: &str, runs_exit: &str) -> (TempDir, Output, String, String) { +fn run_github_restore( + runs: &str, + artifact_runs: &str, + download_exit: &str, + runs_exit: &str, + nested: bool, +) -> (TempDir, Output, String, String) { let bash = git_bash().expect("git bash checked by caller"); let tmp = TempDir::new().unwrap(); let root = tmp.path(); @@ -1961,7 +1967,19 @@ if [ "$1" = "api" ]; then exit 0 fi if [ "$1" = "run" ] && [ "$2" = "download" ]; then - exit "${FAKE_GH_DOWNLOAD_EXIT:-0}" + if [ "${FAKE_GH_DOWNLOAD_EXIT:-0}" != "0" ]; then exit "$FAKE_GH_DOWNLOAD_EXIT"; fi + # Mimic the payload layout: flat by default (what a single --name gives), + # nested under the artifact name when the scenario asks for it. + dest="" + prev="" + for arg in "$@"; do + if [ "$prev" = "--dir" ]; then dest="$arg"; fi + prev="$arg" + done + if [ "$FAKE_GH_NESTED" = "1" ]; then dest="$dest/$ARTIFACT"; fi + mkdir -p "$dest" + echo "{}" > "$dest/run.json" + exit 0 fi exit 0 "#, @@ -1991,6 +2009,7 @@ exit 0 .env("FAKE_GH_ARTIFACT_RUNS", artifact_runs) .env("FAKE_GH_DOWNLOAD_EXIT", download_exit) .env("FAKE_GH_RUNS_EXIT", runs_exit) + .env("FAKE_GH_NESTED", if nested { "1" } else { "0" }) .env("ARTIFACT", "bench-history-linux") .env("DEFAULT_BRANCH", "main") .env("WORKFLOW", "anvil-scheduled") @@ -2015,7 +2034,7 @@ fn github_restore_separates_absence_from_failure() { // (1) The newest run has no artifact; an older one does. The walk must // reach it rather than cold-starting on the first miss. - let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "0"); + let (tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "0", false); assert!( output.status.success(), "walking back should succeed:\n{}", @@ -2027,10 +2046,27 @@ fn github_restore_separates_absence_from_failure() { "should name the run it restored from:{}", both_streams(&output) ); + // The payload has to land *in* the store, not one level under it. A + // nested copy loses the history silently: every run then cold-starts + // and analyzes to a clean no-op. + let store = tmp.path().join("target/anvil/bench-history/run.json"); + assert!(store.is_file(), "the restored payload must land in the store root"); + + // `gh run download` extracts a single `--name` straight into `--dir`, + // but nests one directory per artifact when several are requested. + // Tolerate both, so a change in that behaviour cannot silently empty + // the store. + let (tmp_nested, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "0", true); + assert!(output.status.success(), "nested layout:{}", both_streams(&output)); + assert!(env.contains("ANVIL_BENCH_RESTORE=restored"), "env:\n{env}"); + assert!( + tmp_nested.path().join("target/anvil/bench-history/run.json").is_file(), + "a nested artifact directory must be lifted into the store root" + ); // (2) No run in the window carries it: a genuine cold start, and it must // be visible on the summary rather than only in the log. - let (_tmp, output, env, summary) = run_github_restore("30 20 10", "", "0", "0"); + let (_tmp, output, env, summary) = run_github_restore("30 20 10", "", "0", "0", false); assert!(output.status.success()); assert!(env.contains("ANVIL_BENCH_RESTORE=cold-start"), "env:\n{env}"); assert!(summary.contains("cold start"), "summary:\n{summary}"); @@ -2038,7 +2074,7 @@ fn github_restore_separates_absence_from_failure() { // (3) The artifact exists but the download fails. This is the branch whose // silent reintroduction re-creates the history-loss bug: it must fail and // leave no publishable restore state. - let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "30", "1", "0"); + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "30", "1", "0", false); assert_failed(&output, "a failing download"); assert!( !env.contains("ANVIL_BENCH_RESTORE"), @@ -2049,7 +2085,7 @@ fn github_restore_separates_absence_from_failure() { // command substitution consumed as a `for` word list, so this branch // would otherwise fall through to a cold start and publish a truncated // store over the chain while reporting green. - let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "1"); + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "1", false); assert_failed(&output, "a failing run listing"); assert!(!env.contains("ANVIL_BENCH_RESTORE"), "a failed listing is not a cold start:\n{env}"); } diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index 6e7054dd0..be744880f 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -497,8 +497,17 @@ runs: complete_restore() { mkdir -p target/anvil/bench-history - if [ -n "$(ls -A "$staging" 2>/dev/null)" ]; then - cp -R "$staging/." target/anvil/bench-history/ + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ fi echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" # The recipe refuses to write anywhere but here, so an override