diff --git a/.anvil.lock b/.anvil.lock index 1a40a7296..ed5007806 100644 --- a/.anvil.lock +++ b/.anvil.lock @@ -1,7 +1,7 @@ version = 1 tool = "anvil" tool_version = "0.5.0" -catalog_checksum = "sha256:1388eee0dd1074adb96e6a944c65f2542ec4d6da90d17ad401ee453270359d3b" +catalog_checksum = "sha256:1a7aa05ccee55e5bad46ea953d460a3c4cad4e2737ccec514bfdfa4736558dc9" [[file]] path = ".anvil/container/Dockerfile.dockerignore" @@ -17,7 +17,7 @@ checksum = "sha256:9940d1947482150ac08fcb9b4150da99f5ae60642f4caeea137577ce0e709 [[file]] path = ".github/actions/anvil-run-group/action.yml" -checksum = "sha256:ff8def6c0786b6e146c4b633dfe38cb9b8ede398345516cca32bbcd5586087af" +checksum = "sha256:23a323e87f5ac0bb72a4e64e33692a71acf827956356e74698ac1b97a5e0da0b" [[file]] path = ".github/actions/anvil-setup/action.yml" @@ -37,11 +37,11 @@ checksum = "sha256:0c2530d9a38e6a74e0a7fd4f999b4a1790f97de30b58b68c6c2344600da19 [[file]] path = ".github/workflows/anvil-scheduled-impl.yml" -checksum = "sha256:ac70061acf594c8c212c45ed97c3b653e7b8de68f4e1dcc9695a2628e4e2596d" +checksum = "sha256:1c5250ead07c7915c978b52e425cc950e2ebe151a30cfc9a1398f354d1209ee9" [[file]] path = ".github/workflows/anvil-scheduled.yml" -checksum = "sha256:d4d3bd645a5586e9a1cc3a5fc27e38c93e59b1e29f83ede0ccd610273eec00f4" +checksum = "sha256:bc8120b035051db9e3db875495d6570238824339b9694234e388e759c0a84ff0" [[file]] path = "justfiles/anvil/checks/aprz.just" @@ -51,6 +51,10 @@ checksum = "sha256:0f9f3dd3c8a2f034ebf4f2ac0344cba3db79612ff2fc2ea037bfabbaacb1b path = "justfiles/anvil/checks/audit.just" checksum = "sha256:54abf96a320bb4b35a3c0ddf2f30b0f4a30e0673e482ca3a71242fa383536415" +[[file]] +path = "justfiles/anvil/checks/bench-history.just" +checksum = "sha256:d24d92356347c3215b9db87eaddac821e689d71708761624ad61e698e19d15dd" + [[file]] path = "justfiles/anvil/checks/bench.just" checksum = "sha256:50f04b4ea6c99df8ad7434d320f34dccdb6db6a77b83e3090e35de0ca3a15f83" @@ -191,6 +195,10 @@ checksum = "sha256:ea96d29e261b454a585c0ba3dc7954a35d0c726e9e94f6bb7c82f15261531 path = "justfiles/anvil/groups/scheduled-advisories.just" checksum = "sha256:4f9940bb54fd7cd1d622f3207f7c43f38f95232a370f7537b15546271d88805d" +[[file]] +path = "justfiles/anvil/groups/scheduled-benchmarks.just" +checksum = "sha256:38aa0554499568b97be95dd8c73227f5a097e9e4a1c0fa144af8544c22d3be12" + [[file]] path = "justfiles/anvil/groups/scheduled-exhaustive.just" checksum = "sha256:0b9023c614ae400c30f7131fc939b318bf6ee2ba086b18d45491fa30691694e7" @@ -213,19 +221,19 @@ checksum = "sha256:5714a138135154b91b234f6bad06ec3ade0e5780f761d631c1eca92b6010e [[file]] path = "justfiles/anvil/mod.just" -checksum = "sha256:2f7f0187f8c45716a1bed85f0c45ffbc71cdf592ed6414c9bc4cd72cf716a3a5" +checksum = "sha256:3cace5e08fd88200b1feac2345ec697d63993819dc46ee2dfdde0dad29cfa055" [[file]] path = "justfiles/anvil/tiers.just" -checksum = "sha256:00453a12cbb34811ee6a2c083dade5f6198575e3b0610f49e4743366326cdd18" +checksum = "sha256:a63a2cd4b5a63353c08a7d98e8b23683c87ef7ad60a23c33e4dd789204b0480d" [[file]] path = "justfiles/anvil/tools.just" -checksum = "sha256:a1e44ca16f172b487afa3997f102512733d3b65a4418cf894cbd749a3abc17dc" +checksum = "sha256:d7dc849363748f40df2768f13b364a3237767eda84525a1c901d4caddbacec3c" [[file]] path = "justfiles/anvil/versions.just" -checksum = "sha256:983e6732348188becb8f22a4af5237a13ac2579d566a9c4ff4bab6e724336077" +checksum = "sha256:69d843dd7fdfee808d74d46485ab3b290a839418db390233eeb89ef8893e9f5d" [[region]] host = ".anvil/container/Dockerfile" diff --git a/.github/actions/anvil-run-group/action.yml b/.github/actions/anvil-run-group/action.yml index ce3c159fe..67257a906 100644 --- a/.github/actions/anvil-run-group/action.yml +++ b/.github/actions/anvil-run-group/action.yml @@ -34,6 +34,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -43,6 +66,110 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -53,6 +180,7 @@ runs: # target/anvil/impact cache (read via `_anvil-impact-include`), not # threaded --package strings. This action only fixes the mode. ANVIL_IMPACT: ${{ inputs.impact_mode }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -75,6 +203,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true diff --git a/.github/workflows/anvil-scheduled-impl.yml b/.github/workflows/anvil-scheduled-impl.yml index 982dc1ae6..7071d04f1 100644 --- a/.github/workflows/anvil-scheduled-impl.yml +++ b/.github/workflows/anvil-scheduled-impl.yml @@ -24,6 +24,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -141,6 +149,35 @@ jobs: with: group: scheduled-exhaustive + scheduled-benchmarks: + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. + fetch-depth: 0 + lfs: true + - uses: ./.github/actions/anvil-run-group + with: + group: scheduled-benchmarks + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} + publish-failure: name: Publish scheduled failure needs: @@ -148,6 +185,7 @@ jobs: - scheduled-advisories - scheduled-runtime-analysis - scheduled-exhaustive + - scheduled-benchmarks if: ${{ always() && vars.ANVIL_PUBLISH_FAILURE_ISSUE != 'false' && contains(needs.*.result, 'failure') }} runs-on: ${{ inputs.linux_runner }} diff --git a/.github/workflows/anvil-scheduled.yml b/.github/workflows/anvil-scheduled.yml index 06a001521..52830ab86 100644 --- a/.github/workflows/anvil-scheduled.yml +++ b/.github/workflows/anvil-scheduled.yml @@ -21,5 +21,8 @@ jobs: # and restores that scope only on publish-failure. permissions: contents: read + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. Narrowed to that job inside. + actions: read issues: write secrets: inherit diff --git a/Cargo.lock b/Cargo.lock index 2ff420acf..834e687ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -32,6 +41,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -419,6 +434,7 @@ version = "0.5.0" dependencies = [ "assert_cmd", "clap", + "criterion", "insta", "mutants", "ohno", @@ -708,6 +724,12 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "castaway" version = "0.2.4" @@ -779,6 +801,33 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "clap" version = "4.6.6" @@ -960,6 +1009,39 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -994,6 +1076,12 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -2322,6 +2410,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hash32" version = "0.3.1" @@ -2729,6 +2828,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -3246,6 +3354,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -3274,6 +3388,16 @@ version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c45bb4a6ae1280ec0803b1ef9d3455eb50f01efbbe1447ab020f1d54fba9d8" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -3558,7 +3682,7 @@ checksum = "e0ebcaa9da49ee86809a3c5ff57c0af35530951ded4e7aad371839f173da7ef7" dependencies = [ "crossbeam-channel", "crossbeam-utils", - "itertools", + "itertools 0.14.0", "jod-thread", "libc", "miow", @@ -3573,7 +3697,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a584372630d85436a206d362b26e5156367f348043151b132011f360c63fa18" dependencies = [ "either", - "itertools", + "itertools 0.14.0", "ra_ap_parser", "ra_ap_stdx", "rowan", @@ -4657,6 +4781,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.12.0" diff --git a/Cargo.toml b/Cargo.toml index 2d11ae6e6..423080f5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -47,6 +47,7 @@ chrono = { version = "0.4.45", default-features = false } clap = { version = "4.6.6", default-features = false } clap_complete = { version = "4.6.9", default-features = false } compact_str = { version = "0.10.0", default-features = false } +criterion = { version = "0.8.1", default-features = false } csv = { version = "1.4.0", default-features = false } directories = { version = "6.0.0", default-features = false } duct = { version = "1.1.1", default-features = false } diff --git a/crates/cargo-anvil/Cargo.toml b/crates/cargo-anvil/Cargo.toml index 6214d4b85..a940c285f 100644 --- a/crates/cargo-anvil/Cargo.toml +++ b/crates/cargo-anvil/Cargo.toml @@ -39,12 +39,17 @@ tracing-subscriber = { workspace = true, features = ["fmt"] } [dev-dependencies] assert_cmd = { workspace = true } +criterion = { workspace = true } insta = { workspace = true, features = ["filters"] } predicates = { workspace = true } serial_test = { workspace = true } tempfile = { workspace = true } walkdir = { workspace = true } +[[bench]] +name = "catalog" +harness = false + # >>> anvil-managed: anvil-lints [lints] workspace = true diff --git a/crates/cargo-anvil/README.md b/crates/cargo-anvil/README.md index 9f1d6114a..4c39227a7 100644 --- a/crates/cargo-anvil/README.md +++ b/crates/cargo-anvil/README.md @@ -318,6 +318,7 @@ schedule against the default branch, not on PRs: scheduled-exhaustivemutants-full cargo-hackfeature powerset benchcompile-only + scheduled-benchmarksbench-historyregression detection over the accumulated benchmark history @@ -482,7 +483,7 @@ And `docs/verification.md` for the continuous-validation strategy. This crate was developed as part of The Oxidizer Project. Browse this crate's source code. - [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQb0-jr0JbRf5oblTuBxsHXMUIbSWUr86twfxQbfGu3X0VHoephZIGDa2NhcmdvLWFudmlsZTAuNS4wa2NhcmdvX2Fudmls + [__cargo_doc2readme_dependencies_info]: ggGmYW0CYXZlMC43LjNhdIQbFhzZ8rzWNNYbuRaDSGWynFgbH4PMdoT7GNcbVwNPtPjAhvFhYvRhcoQb0L5vQEL99hIbckxeGo6y7-gb-vDIK2TdUgkbRCeB1R9breNhZIGDa2NhcmdvLWFudmlsZTAuNS4wa2NhcmdvX2Fudmls [__link0]: https://crates.io/crates/cargo-delta [__link1]: https://docs.rs/cargo-anvil/0.5.0/cargo_anvil/?search=artifacts::container [__link10]: https://docs.rs/cargo-anvil/0.5.0/cargo_anvil/?search=artifacts diff --git a/crates/cargo-anvil/benches/catalog.rs b/crates/cargo-anvil/benches/catalog.rs new file mode 100644 index 000000000..e4aff3efc --- /dev/null +++ b/crates/cargo-anvil/benches/catalog.rs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Benchmarks for assembling the built-in catalog. +//! +//! `Catalog::anvil()` builds the whole artifact set — every embedded +//! template, every per-group fan-out — and `checksum()` renders and hashes +//! all of it. Both run on every `cargo anvil` invocation before anything is +//! written, so their cost is paid by every adopter on every update, and both +//! grow with the catalog: this crate's own history is a steady accretion of +//! checks, groups and backend files. +//! +//! The two are timed separately rather than end-to-end, so a move can be +//! attributed to assembly or to rendering rather than leaving the reader to +//! guess which half shifted. +//! +//! They are also the shape a trend watch handles well — pure, deterministic, +//! no I/O, no network — so a move here is a change in the code rather than in +//! the environment. + +use cargo_anvil::Catalog; +use criterion::{Criterion, criterion_group, criterion_main}; + +fn catalog(c: &mut Criterion) { + let mut group = c.benchmark_group("catalog"); + + // Assembly alone: the embedded templates and the per-group expansions. + group.bench_function("anvil", |b| b.iter(Catalog::anvil)); + + // Rendering and hashing every artifact body, over an already-assembled + // catalog. Assembly is deliberately outside the timed closure; the + // benchmark above covers it, and the update path's total cost is the + // two together. + group.bench_function("checksum", |b| { + let catalog = Catalog::anvil(); + b.iter(|| catalog.checksum()); + }); + + group.finish(); +} + +criterion_group!(benches, catalog); +criterion_main!(benches); diff --git a/crates/cargo-anvil/docs/design/README.md b/crates/cargo-anvil/docs/design/README.md index 015aa6332..798289aa3 100644 --- a/crates/cargo-anvil/docs/design/README.md +++ b/crates/cargo-anvil/docs/design/README.md @@ -15,6 +15,8 @@ user-visible shape of the tool. Detail lives in companion documents: - [ado.md](./ado.md) — Azure DevOps Pipelines emission, 1ESPT/msrustup composition. - [containers.md](./containers.md) — containerized execution: the explicit `anvil-container` recipe, the content-addressed image, and the credential hook. +- [benchmarks.md](./benchmarks.md) — scheduled benchmark regression detection via + `cargo-bench-history`: history persistence, fail-the-build surfacing, and bless. - [../implementation.md](../implementation.md) — internal implementation guidance. - [../verification.md](../verification.md) — continuous-validation strategy: dogfooding, fixture tests, schema validation. @@ -403,6 +405,7 @@ pipeline. | `pr-test`, `pr-runtime-analysis`, `scheduled-test` | All legs above | Where compile-time and runtime OS / arch bugs actually surface. The three `pr-slow*` groups run as parallel cloud-workflow jobs (split out from a former single `pr-slow`) for shorter wall-clock per leg. | | `pr-mutants` | GH: Linux x86_64 + Windows x86_64 + Linux aarch64 (windows-arm self-skips). ADO: Linux x86_64 + Windows x86_64 | Diff-scoped mutation testing. cargo-mutants doesn't build on `aarch64-pc-windows-msvc`; the recipe self-skips so the windows-arm leg is a no-op. | | `scheduled-exhaustive` | Linux x86_64 + Windows x86_64 | Full `cargo-mutants` / `cargo-hack` / `bench`. cargo-mutants doesn't build on `aarch64-pc-windows-msvc`; rather than splitting the matrix to add an ARM-Linux leg for cargo-hack and bench, the whole group is x86-only. Adopters with ARM-specific concerns extend the matrix in their root workflow. | +| `scheduled-benchmarks` | Linux x86_64 + Windows x86_64 | Benchmark regression detection over the accumulated history. Matches `scheduled-exhaustive`; each leg carries its own history artifact because the series are partitioned per machine. See [benchmarks.md](./benchmarks.md). | macOS is not in the default matrix — adopters who need it fork the owned reusable workflow (GH) or override `testPools` (ADO). The GH-side knob set is intentionally diff --git a/crates/cargo-anvil/docs/design/ado.md b/crates/cargo-anvil/docs/design/ado.md index c985a3433..52ff6ad5a 100644 --- a/crates/cargo-anvil/docs/design/ado.md +++ b/crates/cargo-anvil/docs/design/ado.md @@ -170,6 +170,12 @@ flowchart LR sadv_setup ==> sadv_setup_just sexh_setup ==> sexh_setup_just + sched_stages --> sbench_s["stage: scheduled_benchmarks
linux + windows jobs"]:::stage + sbench_s ==> sbench_step[".pipelines/anvil/
steps/scheduled-benchmarks.yml"]:::step + sbench_step ==> sbench_setup[".pipelines/anvil/
steps/setup.yml"]:::step + sbench_step ==> sbench_just["just anvil-scheduled-benchmarks"]:::recipe + sbench_setup ==> sbench_setup_just["just anvil-setup"]:::recipe + classDef trigger fill:#fff4d6,stroke:#b08800,stroke-width:1px; classDef root fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef impl fill:#dff0d8,stroke:#28a745,stroke-width:1px; @@ -208,6 +214,8 @@ Note the ADO topology differs from GitHub Actions in two places: │ `pool`, `steps`, `artifacts`; │ users edit to inject 1ESPT │ `templateContext:` etc.) + ├── bench-history-restore.yml owned (restore the benchmark history artifact) + ├── bench-history-summary.yml owned (attach benchmark findings to the build summary) ├── pr-fast.yml owned (one step template per group) ├── pr-test.yml owned ├── pr-runtime-analysis.yml owned @@ -215,7 +223,8 @@ Note the ADO topology differs from GitHub Actions in two places: ├── scheduled-test.yml owned ├── scheduled-advisories.yml owned ├── scheduled-runtime-analysis.yml owned - └── scheduled-exhaustive.yml owned + ├── scheduled-exhaustive.yml owned + └── scheduled-benchmarks.yml owned ``` All files are regular owned files tracked by the sidecar `.anvil.lock` manifest @@ -371,7 +380,13 @@ The contract is intentionally small and stable: | `pool` | `object` | yes | Pool block, passed verbatim to ADO's `pool:` key. `linuxPool` and `windowsPool` at the stage level are object parameters, so users can override their shape (e.g. `{ name, os, image }` for 1ESPT). | | `steps` | `stepList` | yes | Body of the job. Templated step lists are fine — the wrapper splices them in via `${{ each step in parameters.steps }}: - ${{ step }}`. | | `inputArtifacts` | `object` | no | List of pipeline artifacts to download *before* the steps run. Each item: `{ name: string, path: string }`. Default wrapper prepends one `DownloadPipelineArtifact@2` per entry; 1ESPT wrappers translate the same list into their own download mechanism (e.g. `templateContext.inputs`). This is how the impact set is shared — each PR group job downloads its OS's `anvil-impact-` artifact into `target/anvil/impact` and its checks read the cache exactly as a local run. | -| `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. The stages templates don't need to know which backend they're targeting. | +| `artifacts` | `object` | no | List of pipeline artifacts to publish. Each item: `{ name: string, path: string, condition: string (optional) }`. Default wrapper appends one `PublishPipelineArtifact@1` per entry; 1ESPT wrappers translate the same list into `templateContext.outputs.pipelineArtifact` blocks. `condition` defaults to `succeededOrFailed()`; a caller that must not publish in some states sets it, and a fork carries it through to whatever output shape it emits. The stages templates don't need to know which backend they're targeting. | + +A job that needs a non-default checkout (depth, LFS) puts an explicit `checkout` +step at the head of its own `steps` list rather than growing this contract. The +contract stays frozen because a forked wrapper cannot be expected to declare a +parameter added after the fork, and a stages template binding one would fail +expansion for the entire pipeline. The default wrapper anvil ships downloads any `inputArtifacts`, splices in the `steps`, then publishes any `artifacts`: @@ -398,7 +413,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} @@ -424,7 +442,10 @@ jobs: - output: pipelineArtifact targetPath: ${{ artifact.path }} artifactName: ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() steps: - ${{ each step in parameters.steps }}: - ${{ step }} @@ -469,6 +490,19 @@ keyword, which for parameters defined at the call site is the stages template itself — so the path is written relative to `pr.yml` / `scheduled.yml`, *not* relative to `steps/job.yml`. +**Per-group steps.** Some groups need steps around the uniform runner — the +benchmark group checks out at full depth and round-trips its history store, +`scheduled-test` publishes coverage. Those are spliced into the group's own +emitted step template at generation time, not written at the call site, so the +stages templates stay a plain list of groups. Moving a check between groups, or +giving a group extra steps, never edits `pr.yml` / `scheduled.yml`. + +The one thing that cannot move is a **job-level output**: `artifacts` is declared +where the job is constructed, because a forked wrapper translates that list into +its own output shape and a publish task inside the step list would bypass the +translation. That declaration is therefore the only per-group content the stages +templates carry. + ### 4.2 Stages template shape Approximate shape (anvil writes this verbatim; users normally don't edit it): @@ -900,3 +934,67 @@ Adding a new advisory check is a two-step change: the recipe writes entry. There's deliberately no auto-discovery loop over the convention dir — explicit per-check entries keep stale comments deterministically clearable when a check is removed from the catalog. + +## 12. Benchmark regression detection + +The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs +`cargo-bench-history`, whose history persists across scheduled runs as **pipeline +artifacts**, reusing the §4.1 job-wrapper `artifacts` contract to publish. The +history is partitioned per machine, so each leg of the group's matrix carries its +own artifact (`bench-history-`). + +Each scheduled benchmark job: + +1. checks out with full history and LFS, and restores, applies blessings, runs + the analysis and attaches findings — all inside `steps/scheduled-benchmarks.yml`, + the group's own emitted step template (§4.1), so the stages template stays a + plain list of groups; +2. the restore walks the pipeline's own builds on the branch newest-first + (`_apis/build/builds?…&queryOrder=finishTimeDescending`) and, per build, queries + the artifacts endpoint for this leg's artifact. A `404` means that build simply + has no such artifact and the walk continues; any other status is an operational + failure and fails the job. Finding none across the whole window is a genuine cold + start; +3. **publishes** the updated store through the wrapper's `artifacts` parameter + (`{ name: bench-history-$(Agent.JobName), path: , condition: … }`), which + the default wrapper emits as `PublishPipelineArtifact@1` and 1ESPT wrappers as a + `pipelineArtifact` output. The name is keyed on the job, which ADO guarantees is + unique within a stage, so two legs can never declare one artifact and merge two + machines' samples into a single series. + +The restore stages its download and creates the store path **only once it has +reached a known state** — restored, or a positively identified cold start. An +operational failure therefore leaves no store directory at all, so a publisher that +runs unconditionally has nothing to upload and the chain survives. That matters +because `artifacts[].condition` is an optional field: a wrapper forked before it +existed accepts the entry and ignores the condition silently, and this is exactly +the case where ignoring it would overwrite good history. The condition remains as a +second line of defence, not the only one. + +`DownloadPipelineArtifact@2` is not used for the restore: `latestFromBranch` resolves +a single build and does not walk, so a cancelled or never-publishing latest build +would cold-start a store that still has usable history. + +The walk is outcome-agnostic, which is what keeps the chain intact across a +regression: a flagged regression fails the stage, so a success-only restore would +discard every sample taken while the pipeline stayed red. Publishing is likewise not +limited to green runs. The `condition` on the artifact entry lives in the `artifacts` +contract (§4.1) rather than in a publish task the benchmark group emits itself, which +would bypass the translation a forked wrapper performs. + +Surfacing is by **build failure**, not a PR comment — the regression is discovered +after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits +non-zero on an active regression, failing the stage; ADO's existing failed-build +**notification subscriptions** fire, and the findings live in the build summary. + +Because the build status is one bit, while the pipeline is already red from one +regression a newly appearing second one does not re-fire the native notification; +the findings remain in the build summary. + +Blessings are applied from a committed `.config/bench-blessings.toml` before analyze +(step 3) — a reviewed pull request, not an out-of-band action. + +The machine key the history is partitioned by is a `benchMachineKey` parameter on the +stages template, surfaced as a stage-level `ANVIL_BENCH_MACHINE_KEY` variable. Empty +uses the hardware fingerprint; a stable pool label trades partition fidelity for +density on a heterogeneous pool. diff --git a/crates/cargo-anvil/docs/design/benchmarks.md b/crates/cargo-anvil/docs/design/benchmarks.md new file mode 100644 index 000000000..82f178a6e --- /dev/null +++ b/crates/cargo-anvil/docs/design/benchmarks.md @@ -0,0 +1,195 @@ +# cargo-anvil benchmark regression detection + +This document describes `cargo-anvil`'s support for detecting performance +regressions from a repo's benchmarks over time. It wraps +[`cargo-bench-history`][cbh] (cbh) — which stores each benchmark run as an +immutable record, reconstructs per-benchmark series in git first-parent order, +partitions series by a hardware machine key, and reports level shifts and drift +with noise-aware, false-discovery-controlled statistics — and integrates it into +the opinionated catalog across both cloud-workflow backends. + +The intended audience is `cargo-anvil` maintainers and downstream catalog +authors. The phased build-out is tracked in +[implementation-plans/0003.md](../implementation-plans/0003.md). + +## 1. Problem + +A repo's benchmarks are only useful as a regression signal if a slowdown shows +up as a break in a *trend*, well before it would reach any expensive, +bespoke-hardware load test. Turning per-run numbers into a reliable trend is the +hard part: results must be ordered by how the code evolved (not by when a +benchmark happened to run), compared only against like hardware (CI runs on a +heterogeneous, rotating pool whose machine-to-machine variance dwarfs the +measurement), and judged with noise-aware statistics (a fixed percentage +threshold on that noise fires constantly). A repo with benchmarks gets none of +this today — the existing `bench` check only compiles them. + +A repo without benchmarks is a first-class case, not an oversight: the recipe +runs, records nothing, and analyzes an empty history to a clean no-op, so the +capability costs such a repo a green job and nothing else. + +## 2. Design principles + +- **Detection is cbh's, not anvil's.** anvil installs and drives the tool; it + does not reimplement change-point detection, machine-key normalization, or + history storage. The opinionated contribution is *where* the tool runs, *how* + its history persists, and *how* a regression is surfaced and accepted. +- **Regression detection is a scheduled concern.** It obeys the catalog's + standing rule — a check belongs in scheduled iff its outcome can change without + a commit to this repo (see [checks.md §4](./checks.md)). Benchmark results + accrue over time and the trend verdict at the tip changes as history grows, so + detection runs on the scheduled tier. On pull requests the benchmark check + stays **compile-only**; running benches per-PR is too costly and, on shared + runners, too noisy to gate a merge. +- **History is cross-run state, kept in CI-native storage.** cbh's local backend + is an immutable, key-addressed directory that round-trips between scheduled runs + through each backend's native build **artifacts**, whose retention and + fetch-the-latest-from-the-default-branch semantics keep the subsystem + self-contained in the pipeline with no external store to provision by default. +- **A regression fails the scheduled build.** Detection happens after the + offending change merged, so there is no pull request to annotate; the signal is + a failed scheduled build, which each backend's native failure notifications + carry. This sits within the "advisory, never fail" tenet, which governs *PR + gating* — a scheduled build blocks no one's merge. +- **Intentional changes are accepted through a reviewed file.** A regression is + cleared by fixing it or by *blessing* it; blessing is expressed as a committed, + reviewed entry rather than an out-of-band action, so accepting a slowdown is an + audited decision. +- **One catalog, both backends.** As with every other check, the capability is + generated for GitHub Actions and Azure DevOps from the same source, so adding + it once reaches every consuming repo. + +## 3. Place in the catalog + +The compile-only `bench` check is unchanged. A new analyzing check — +`bench-history` — runs the benchmarks, records this commit's results into the +history, applies any pending blessings, and analyzes the accumulated +series. It lives in its own scheduled group, `scheduled-benchmarks`, so its +history round-trip and failure semantics stay isolated, and it **exits non-zero +when cbh reports an active regression**; locally and in cloud it behaves +identically (always writes its findings; only the exit code gates), matching the +local-vs-cloud parity every recipe keeps. + +The check is **unscoped**: it ignores the impact-analysis include contract every +other check honors. A series is only comparable when the same suite is measured at +every commit, so measuring an impact-scoped subset would punch holes in the history +that detection cannot tell apart from a benchmark being deleted. + +## 4. History as cross-run state + +Each scheduled run checks out with full history (analysis reads the commit graph +to order series and locate the base merge-base), restores the history published by +the most recent run on the **default branch that carries one**, runs collect → +apply-blessings → analyze, and publishes the updated history as this run's +artifact. Because each run republishes the whole accumulated directory, only the +newest snapshot is ever needed. + +Both the restore and the publish are indifferent to the run's outcome. A flagged +regression fails the build, so restoring only from green runs — or publishing only +on success — would throw away exactly the samples taken while the pipeline stayed +red, which is the stretch of history a reviewer most needs. + +Every series is partitioned by a machine key, so each leg of the group's matrix +carries its own artifact rather than sharing one. + +The persistence is a **rolling window** on CI-native artifacts — portable and +zero-config, and on eviction it degrades to a harmless cold start, since +detection is advisory-by-design and gates nobody's merge. The backend-specific +restore/save building blocks live in [github.md](./github.md) and +[ado.md](./ado.md). cbh's own durable backends (for example Azure Blob) are +outside anvil's scope: the artifact rolling window is the supported store. + +The window's *depth* is not symmetric between backends, and cannot be made so +from here. GitHub's save sets an explicit retention, so the horizon is a +property of the emitted workflow. ADO publishes through the job wrapper's +artifact contract, which exposes no retention control, so the horizon there is +whatever the project or organization build-retention policy grants — commonly +shorter. An ADO chain can therefore lapse sooner than a GitHub one for the same +schedule. This is a visible cold start rather than a silent wrong answer, so it +is documented as a caveat rather than worked around: an adopter who needs a +guaranteed horizon on ADO raises the project's build retention. + +## 5. Surfacing: failing the scheduled build + +An active regression fails the scheduled build; the findings — each benchmark, +its magnitude, and the commit cbh attributes the change-point to — are written to +the build summary and to a findings file. The emitted findings include cbh's +topology-accurate trend chart, so the reviewer's surface is self-contained — +enough to decide *fix or bless* without reproducing the run. + +The failure itself is reported by whatever mechanism the backend already uses for a +failed scheduled build: notification subscriptions on Azure DevOps, and on GitHub +Actions the scheduled tier's own `publish-failure` job, which `scheduled-benchmarks` +is one of the dependencies of. This subsystem contributes the *detail* on the build +summary rather than a notification channel of its own, so a regression reaches a +human exactly the way every other scheduled failure does. + +A sustained regression re-fails every run until it is fixed or blessed, so red +stays meaningful only under the discipline that the build is always returned to +green by one of those two actions. + +## 6. Local behavior + +The `bench-history` recipe is the same command locally and in cloud, but its +*input* differs: the history lives only in the CI artifact store, so a local run +has no shared trend to analyze. Locally the recipe runs the benches, records them +into a gitignored local store, and analyzes that store — which on a fresh checkout +is empty, so analysis is a clean no-op reporting that there is no local history +yet. A developer thus gets the current run's numbers (and a single-machine local +trend if they run it repeatedly), not the shared regression signal. + +For the same reason the *gate* is CI-only by default. The recipe reports +identically in both places, but only exits non-zero under CI (or with an explicit +local opt-in): a laptop's measurement noise is not the shared trend, and failing +a pre-release `anvil-full` on thermal throttling would invite silencing it with a +committed blessing — polluting a reviewed, audited file with one machine's +artifacts. + +Regression detection is therefore a scheduled, shared concern, and the failure +surface (§5) — cbh's finding and trend chart on the build summary — is +the interface a developer acts on. Reproducing a CI finding locally is not a +first-class workflow: it would require downloading that run's `bench-history` +artifact into the local store and running cbh's `examine` with the run's machine +key (a developer's machine has a different hardware fingerprint). That remains a +documented manual escape hatch rather than a generated recipe. + +## 7. Accepting intentional changes: bless + +cbh's blessing re-baselines a series from a commit forward via an append-only +sidecar written into the *history store*. To keep the store single-writer and to +make acceptance reviewable, blessing is expressed as a committed entry — the +benchmark, the attributed commit, and a human reason — that the scheduled job +applies (idempotently) before analyzing. The workflow is therefore: red build → +a reviewed pull request accepting the change → the next scheduled run applies it +and the build returns to green. The accumulated entries are an audit trail of +every deliberate tradeoff. + +Idempotence comes from reconciling the committed entries against the blessings +already recorded in the store, so re-running the job never re-appends a sidecar +that is already in effect. + +## 8. Boundaries and caveats + +- **Hosted-runner machine-key density.** cbh partitions by a hardware + fingerprint; a heterogeneous hosted pool can split a series into per-key + partitions too sparse to analyze. Whether a hosted pool stays dense enough + depends on its hardware homogeneity; self-hosted or dedicated runners avoid the + concern. An adopter who knows their pool is uniform enough can substitute a + stable pool label for the fingerprint, trading partition fidelity for density; + both backends expose that as an input on their scheduled wiring. +- **Attribution is coarse under sparse benchmarking.** Benches do not run on + every commit, so the attributed commit is the first *benchmarked* one after a + regression and may bundle several changes — an honest range, not always a + single culprit. +- **The scheduled status is one bit.** It collapses several concurrent + regressions into one red, and a second regression appearing while the build is + already red re-fires no notification. The per-regression detail is recovered + from the build summary, which lists every finding on every run. +- **Uncalibrated thresholds.** cbh's gating thresholds are defaults rather than + values calibrated to every consumer's data; pinning the tool version contains + the resulting risk, and the signal only gates the scheduled build. +- **History horizon.** CI-native artifacts hold a rolling window, not unbounded + history; there is no anvil-supported durable store, so the horizon is the + artifact retention. + +[cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history diff --git a/crates/cargo-anvil/docs/design/checks.md b/crates/cargo-anvil/docs/design/checks.md index 3496e885c..d20fba15b 100644 --- a/crates/cargo-anvil/docs/design/checks.md +++ b/crates/cargo-anvil/docs/design/checks.md @@ -52,6 +52,7 @@ flowchart LR sched --> s_adv[anvil-scheduled-advisories]:::group sched --> s_runtime[anvil-scheduled-runtime-analysis]:::group sched --> s_exh[anvil-scheduled-exhaustive]:::group + sched --> s_bench[anvil-scheduled-benchmarks]:::group pr_fast --> fmt[fmt]:::check pr_fast --> clippy[clippy]:::check @@ -99,6 +100,8 @@ flowchart LR s_exh --> cargo_hack[cargo-hack]:::check s_exh --> bench[bench]:::check + s_bench --> bench_history[bench-history]:::check + classDef tier fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef group fill:#f6f8fa,stroke:#586069,stroke-width:1px; classDef check fill:#f3e8ff,stroke:#6f42c1,stroke-width:1px,font-size:10px; @@ -117,7 +120,7 @@ flowchart LR The three `pr-slow*` groups are independent: failures in `pr-test` don't block `pr-runtime-analysis` or `pr-mutants` from running, and overall PR wall-clock is `max(pr-test, pr-runtime-analysis, pr-mutants)` per leg rather than the sum. Locally, `just anvil-pr-slow` is an umbrella recipe that runs all three sub-recipes sequentially so adopters who want "run everything slow" don't have to type three commands. -### scheduled tier (4 groups) +### scheduled tier (5 groups) | Group | OS scope | Purpose | |----------------------|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------| @@ -125,6 +128,7 @@ The three `pr-slow*` groups are independent: failures in `pr-test` don't block ` | `scheduled-advisories` | Same default as `pr-fast` | Runs checks whose outcome can change without a commit to this repo: `deny`, `audit`, `aprz` (external databases), `clippy` (lint set evolves with toolchain). Cross-OS because clippy compiles per host. | | `scheduled-runtime-analysis` | Same default as `pr-runtime-analysis` | Whole-workspace runtime correctness under `miri`, tree-borrows, strict-provenance, and race-coverage. One job per OS leg runs the four profiles sequentially so they share setup and cache state; parallelism is across OS legs. | | `scheduled-exhaustive` | Linux x86_64 + Windows x86_64 | Full `cargo mutants`, cargo-hack feature powerset, and benchmark compilation. The default matrix is x86-only because cargo-mutants is unsupported on Windows ARM. | +| `scheduled-benchmarks` | Same default as `scheduled-exhaustive` | Runs the benchmarks and analyzes the accumulated history with `cargo-bench-history` to detect performance regressions, restoring and publishing that history as a build artifact and failing on an active regression. Its own group so the history round-trip and fail-on-regression semantics stay isolated from the other exhaustive work. See [benchmarks.md](./benchmarks.md). | **Backend asymmetry on ARM coverage.** The GitHub backend ships a four-leg default matrix (Linux/Windows × x86_64/aarch64) because GH has Microsoft-hosted ARM runners @@ -264,7 +268,16 @@ The `miri` row above is the one place the catalog deliberately duplicates a chec |-----------------------|--------------------------------------------------------------------------------------------------------------|--------| | `mutants-full` | `cargo mutants --workspace --no-shuffle --jobs 0` | oxidizer-github, oxidizer (sharded cross-OS) | | `cargo-hack` powerset | `cargo hack --workspace --feature-powerset --depth 2 check` | oxidizer, oxidizer-github | -| `bench` | `cargo bench --workspace --all-features --no-run`; benchmark execution is intentionally outside the catalog because runtime requirements are repository-specific | oxidizer | +| `bench` | `cargo bench --workspace --all-features --no-run`; compile-only, so a broken bench target is caught without paying for a measurement run | oxidizer | + +### `scheduled-benchmarks` + +| Check | Invocation | Source | +|----------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------|--------| +| `bench-history` | `cargo bench-history` collect → apply-blessings → analyze over the history restored from the previous scheduled run; exits non-zero on an active regression. | new | + +Unlike the compile-only `bench` above, this check actually *runs* the benchmarks and +judges the resulting trend. See [benchmarks.md](./benchmarks.md). ## 3. Per-check vs grouped cloud workflows execution @@ -312,7 +325,10 @@ What that means concretely: - **Run only in scheduled** -- the expensive whole-workspace work that doesn't fit a PR budget: the non-stacked miri profiles `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage` (in `scheduled-runtime-analysis`); full `mutants`, - `cargo-hack --feature-powerset`, `bench` (in `scheduled-exhaustive`). + `cargo-hack --feature-powerset`, and the compile-only `bench` (in + `scheduled-exhaustive`). Benchmark *regression detection* — running the + benches and analyzing the accumulated history — is also scheduled-only and is + designed in [benchmarks.md](./benchmarks.md). The single-tier-per-group rule still holds: when a check appears in both tiers it lives in two different groups (one PR group, one scheduled group). Repos that want a @@ -357,7 +373,7 @@ Bucket assignments per check: | modified | `fmt`, `cargo-sort`, `license-headers`, `ensure-no-cyclic-deps`, `ensure-no-default-features` | | affected | `clippy`*, `llvm-cov`, `doc-test`, `examples`, `mutants-diff`, `miri`, `miri-tree-borrows`, `miri-strict-provenance`, `miri-race-coverage`, `careful`, `loom`, `bolero`, `semver-check`, `external-types`, `bench` | | required | `doc-build`, `udeps`, `cargo-hack` (feature powerset) | -| unscoped | `pr-title`, `deny`, `audit`, `aprz`, `mutants-full`, `readme-check`, `spellcheck` | +| unscoped | `pr-title`, `deny`, `audit`, `aprz`, `mutants-full`, `readme-check`, `spellcheck`, `bench-history` | \* cargo-delta's README recommends `clippy` with the modified tier. anvil deliberately runs it on the affected set instead: a change in a crate's API can introduce clippy lints @@ -376,8 +392,9 @@ deps), `cargo udeps` (unused-deps detection needs the resolved graph), `cargo ha external risk DB. `readme-check` and `spellcheck` also belong here: their inputs include repo-level files cargo-delta does not map to any package — the workspace-level README template (`crates/README.j2` / `README.j2`) and the root `.spelling` dictionary — so a -change to one of those would be silently scoped out. These ignore impact scoping and -always run. +change to one of those would be silently scoped out. `bench-history` belongs here too: it +needs the same suite measured at every commit for its series to stay comparable. These +ignore impact scoping and always run. The sentinel `--skip` is a magic string that cannot be a valid cargo argument, so there is no collision with real package names. Recipes test for it with diff --git a/crates/cargo-anvil/docs/design/github.md b/crates/cargo-anvil/docs/design/github.md index ec9fe6963..19fb6b971 100644 --- a/crates/cargo-anvil/docs/design/github.md +++ b/crates/cargo-anvil/docs/design/github.md @@ -20,6 +20,9 @@ need to change: runs setup plus the matching `just anvil--` recipe and surfaces the concrete failure without duplicating group membership. See [Failure attribution and commit statuses](#failure-attribution-and-commit-statuses). + A group needing steps around the runner (the benchmark group's history + round-trip) turns them on with an input, so the workflows stay a plain + list of groups and the other groups skip the steps. See also: @@ -153,6 +156,12 @@ flowchart LR run_group_act ==> sexh_just setup_act ==> setup_just + sched_impl --> sbench_job["scheduled-benchmarks
matrix: linux, windows"]:::job + sbench_job ==> sbench_act[".github/actions/
anvil-scheduled-benchmarks"]:::action + sbench_act ==> sbench_setup[".github/actions/
anvil-setup"]:::action + sbench_act ==> sbench_just["just anvil-scheduled-benchmarks"]:::recipe + sbench_setup ==> sbench_setup_just["just anvil-setup"]:::recipe + classDef trigger fill:#fff4d6,stroke:#b08800,stroke-width:1px; classDef root fill:#e6f0ff,stroke:#0366d6,stroke-width:2px; classDef impl fill:#dff0d8,stroke:#28a745,stroke-width:1px; @@ -420,15 +429,17 @@ is the canonical YAML: ```text caller anvil-scheduled.yml - permissions upper bound: contents:read + issues:write + permissions upper bound: contents:read + actions:read + issues:write └─ called anvil-scheduled-impl.yml default reset: contents:read ├─ scheduled-test (Linux/Windows × x64/ARM64) ├─ scheduled-advisories (Linux/Windows × x64/ARM64) ├─ scheduled-runtime-analysis (Linux/Windows × x64/ARM64) ├─ scheduled-exhaustive (Linux/Windows x64) + ├─ scheduled-benchmarks (Linux/Windows x64) + │ job override: actions:read (history-artifact restore) └─ publish-failure - needs: all four scheduled groups + needs: all five scheduled groups condition: at least one failure and publication not disabled job override: issues:write only ``` @@ -451,11 +462,30 @@ The reusable workflow declares a small input set so the root workflow can pass o | `windows_runner` | string | `windows-latest` | Runner label for x86_64 Windows jobs. | | `linux_arm_runner` | string | `ubuntu-24.04-arm` | Runner label for aarch64 Linux jobs. | | `windows_arm_runner` | string | `windows-11-arm` | Runner label for aarch64 Windows jobs. | - -The input surface is intentionally narrow: only per-leg *runner labels* are exposed, -because swapping in self-hosted runners is the one common need that doesn't require -otherwise touching the workflow. The OS matrix shape (which legs run) is fixed in the -workflow source — see the discussion under the PR snippet above. +| `bench_machine_key` | string | *(empty)* | Machine key the benchmark history is partitioned by (scheduled workflow only). | + +The input surface is **per-leg runner labels plus a per-capability knob where the +capability's behaviour depends on the runner fleet rather than on the source tree**. +Runner labels are exposed because swapping in self-hosted runners is the one common +need that doesn't require otherwise touching the workflow. `bench_machine_key` earns +an input on the same test: benchmark history is partitioned by a hardware +fingerprint, so a heterogeneous pool can fragment a series into partitions too sparse +to analyze, and only the adopter knows whether their fleet is uniform enough to +substitute a stable pool label. That is a property of *their* runners, invisible to +the catalog, so no recipe default or env var can supply it. + +A knob that fails that test — anything the catalog could decide, or that varies per +developer rather than per fleet — stays an env var read by the recipe +(`ANVIL_BENCH_HISTORY_STORE` is the local-only counterexample) rather than growing +this surface. + +Setting an input means editing the generated root workflow, which takes ownership of +it through the dirty-file flow (§3): subsequent updates Propose into an +`.anvil-proposed` sibling instead of overwriting. That cost is real and is why the +surface stays small. + +The OS matrix shape (which legs run) is fixed in the workflow source — see the +discussion under the PR snippet above. The reusable workflows also declare an optional `workflow_call` secret `CODECOV_TOKEN`. See §10 (Coverage upload) for how it's used. @@ -1057,3 +1087,61 @@ a matching `Upsert anvil-` / `Clear anvil-` pair with `header: anvil-`. There's deliberately no auto-discovery loop over the convention dir — explicit per-check steps keep stale comments deterministically clearable when a check is removed from the catalog. + +## 12. Benchmark regression detection + +The scheduled benchmark group (see [benchmarks.md](./benchmarks.md)) runs +`cargo-bench-history`, whose history persists across scheduled runs as GitHub +**Actions artifacts**. The history is partitioned per machine, so each leg of the +group's matrix carries its own artifact (`bench-history-`). + +The round-trip lives in the **shared group action**, behind an input, rather than +in the workflow or in an action of its own. Actions are the only reuse primitive +that shares a runner with the group run, which restore and save must: a reusable +workflow would put them on a different machine from the store they manage. +Groups that leave the input off skip the steps entirely, so the scheduled +workflow stays a plain list of groups. + +The action cannot supply everything, and the remainder is exactly what GitHub +scopes to the job: an action cannot request `permissions`, and the checkout has +already happened before it starts. So the `actions: read` grant and the +full-depth checkout stay in the workflow, along with the matrix. The artifact +name is passed in for the same reason — the matrix value is in scope only at the +call site. + +Each scheduled benchmark job: + +1. checks out with `fetch-depth: 0` and `lfs: true` (analysis reads the commit + graph; benchmark inputs may be LFS-tracked); +2. **restores** the history by walking back from the newest `anvil-scheduled` run + on the default branch and taking the first that carries the leg's artifact; the + first run finds none and starts empty; +3. applies any pending blessings, runs collect + analyze, writing findings to the + job summary and to a findings file; +4. **saves** the updated store with `actions/upload-artifact`, whatever the job's + outcome, so the samples collected while the pipeline is red are not lost. + Retention is set so the latest artifact outlives the gap to the next scheduled + run. + +The restore step queries the runs and artifacts APIs, so the job needs +`actions: read`. A reusable workflow cannot grant itself more than its caller, so +the root workflow passes it through and the impl workflow narrows it to the +benchmark job; the PR workflow keeps `contents: read`. + +Restoring from the newest run that *carries* the artifact rather than the newest +*successful* one is what keeps the chain intact across a regression: a flagged +regression fails the job, so a success-only restore would discard every sample +taken while the pipeline stayed red. + +Surfacing is by **build failure**, not a PR comment — the regression is discovered +after merge (see [benchmarks.md §5](./benchmarks.md)). The benchmark recipe exits +non-zero on an active regression, failing the job, and `scheduled-benchmarks` is +one of the jobs `publish-failure` depends on (§11), so the regression reaches a +human through the same tracking issue as any other scheduled failure. The +per-finding detail — each benchmark, its magnitude, its attributed commit, and +cbh's trend chart — is written to the job summary, which the issue links to, so +the failed run carries everything a reviewer needs to decide *fix or bless*. + +Blessings are applied from a committed `.config/bench-blessings.toml` before analyze +(step 3), so accepting an intentional change is a reviewed pull request rather than an +out-of-band action. diff --git a/crates/cargo-anvil/docs/design/local.md b/crates/cargo-anvil/docs/design/local.md index e22da4a2d..cc22554f7 100644 --- a/crates/cargo-anvil/docs/design/local.md +++ b/crates/cargo-anvil/docs/design/local.md @@ -146,8 +146,13 @@ anvil-scheduled-advisories: anvil-deny anvil-audit anvil-aprz anvil-clippy anvil-scheduled-runtime-analysis: anvil-miri anvil-miri-tree-borrows \ anvil-miri-strict-provenance anvil-miri-race-coverage anvil-scheduled-exhaustive: anvil-mutants-full anvil-cargo-hack anvil-bench +anvil-scheduled-benchmarks: anvil-bench-history ``` +The `scheduled-benchmarks` group runs the same recipe locally, but benchmark +regression analysis needs the shared CI history that a local checkout does not +have; its local behavior is described in [benchmarks.md §6](./benchmarks.md). + ### tiers.just Three tier aggregators. Each tier is a recipe that depends on the appropriate set of groups @@ -157,7 +162,7 @@ in a deterministic order: anvil-pr: anvil-pr-validate-prereqs anvil-pr-fast anvil-pr-slow anvil-scheduled: anvil-scheduled-validate-prereqs anvil-scheduled-test \ anvil-scheduled-advisories anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive anvil-scheduled-benchmarks anvil-full: anvil-pr anvil-scheduled ``` diff --git a/crates/cargo-anvil/docs/design/updates.md b/crates/cargo-anvil/docs/design/updates.md index 0b071d1d1..f0a0724fb 100644 --- a/crates/cargo-anvil/docs/design/updates.md +++ b/crates/cargo-anvil/docs/design/updates.md @@ -180,6 +180,23 @@ different base or extra packages edits them in place, and drift handling preserves the edit (see [containers.md](./containers.md#8-customization)). A "DO NOT EDIT" marker would contradict the customization path both are for. +### 2.1 Growing a contract a fork implements + +`steps/job.yml` is the one owned file adopters are expected to fork, so its +parameter contract is the one place where a change can silently do nothing. A +generated file that binds a *new parameter* the fork does not declare fails +template expansion loudly. A new *optional field inside an existing object* +parameter is the dangerous case: the fork accepts the object, ignores the field, +and diverges in behaviour with no error anywhere — the emitter, `.anvil.lock` and +the update flow cannot see it. + +The rule that follows: **a behaviour that protects data must not depend on a fork +having adopted a field.** Express it so the generated side is safe on its own, and +let the contract field be a second line of defence. `artifacts[].condition` is the +worked example — it guards against publishing a truncated benchmark history, and +the restore step is written so a failed restore leaves nothing to publish at all, +which holds even for a wrapper that predates the field. + ## 3. Managed regions Co-owned files with one or more tool-managed sections delimited by sentinel comments. diff --git a/crates/cargo-anvil/docs/implementation-plans/0003.md b/crates/cargo-anvil/docs/implementation-plans/0003.md new file mode 100644 index 000000000..00e912b92 --- /dev/null +++ b/crates/cargo-anvil/docs/implementation-plans/0003.md @@ -0,0 +1,79 @@ +# Implementation Plan 0003 — Benchmark regression detection via cargo-bench-history + +The design for this capability is [../design/benchmarks.md](../design/benchmarks.md) +— with the catalog placement in [../design/checks.md](../design/checks.md) and the +backend wiring in [../design/github.md §12](../design/github.md) and +[../design/ado.md §12](../design/ado.md). This plan does not restate *what* the +capability is; it sequences *how* and *in what order* it lands. + +The detection engine is [`cargo-bench-history`][cbh]; anvil supplies only the +wiring. The approach was de-risked ahead of this plan by a backtest over three +months of real benchmark history — cbh flagged three known regressions at their +exact attributed commits with no false positives on dense series — so the plan +proceeds to wiring rather than re-validating detection. + +## Sequencing principles + +1. **Additive first.** Tool registration and the local recipe land before any + cloud-workflow change, so they review without touching backend generation. +2. **Local before cloud.** The check runs and gates locally before any scheduled + wiring exists (anvil's local-vs-cloud parity); the cloud phases only add + persistence, surfacing, and scheduling around an already-working recipe. +3. **Both backends per concern.** Persistence and failure surfacing each land as a + single commit spanning GitHub and ADO, so the two never drift. +4. **Snapshots travel with each phase** (see [../verification.md](../verification.md)): + the catalog and emitted-file snapshots grow with the new tool, group, + actions/steps, and workflows as each phase adds them. + +## Phase 1 — Tool registration + +Pin `cargo-bench-history` in `versions.just` and add an +`anvil-tool-cargo-bench-history-install` recipe to `tools.just`. No group +references it yet. Pure additive; catalog snapshots gain the recipe. + +## Phase 2 — The `bench-history` check recipe (local) + +Add the `bench-history` check recipe (`checks/bench-history.just`) and the +`scheduled-benchmarks` group recipe. The recipe resolves a local history +directory, applies any pending blessings, runs `cbh collect` then `cbh analyze`, +writes findings to the build-summary path and a findings file, and exits non-zero +on an active regression — behaving identically with no cloud context. Catalog +snapshots gain the group and check. + +## Phase 3 — Scheduled wiring and history persistence (both backends) + +Emit the `scheduled-benchmarks` job (GitHub reusable scheduled workflow) and stage +(ADO scheduled stages), each running `anvil-setup` then +`just anvil-scheduled-benchmarks`, and add the artifact round-trip. Each leg +carries its own artifact, and both backends walk back from the newest run to the +first that actually carries it — restoring only from *successful* runs would +discard the samples taken while the pipeline was red, which is the stretch that +matters. A restore that fails operationally must fail the job rather than +publishing a truncated store over the chain. + +- **GitHub** — `fetch-depth: 0` + `lfs: true` checkout; a step resolving the run + via the runs/artifacts API and downloading with `gh run download`; + `actions/upload-artifact` with retention, guarded on the restore outcome. +- **ADO** — full-history + LFS checkout as an explicit `checkout` step in the + group's own step list (the §4.1 wrapper contract stays frozen, since adopters + fork that file); a restore step resolving the build through the build/artifacts + REST API; a publish step carrying the same guard. + +Emitted-file snapshots gain the actions/steps and workflow/stage entries. + +## Phase 4 — Failure surfacing + +Write the findings to each backend's build summary and rely on the scheduled tier's +existing failure reporting to notify a human — ADO notification subscriptions, and +on GitHub the `publish-failure` job, which gains `scheduled-benchmarks` as a +dependency. No notification channel of this subsystem's own. + +## Phase 5 — Bless application + +Define the `.config/bench-blessings.toml` schema and the idempotent apply step that +runs ahead of analyze in the scheduled job, reconciling the committed entries against +what `cbh list blessings` reports as already recorded. A fixture repo exercises +red → bless → green. + +[cbh]: https://github.com/folo-rs/folo/tree/main/packages/cargo-bench-history + diff --git a/crates/cargo-anvil/src/anvil/artifacts/ado.rs b/crates/cargo-anvil/src/anvil/artifacts/ado.rs index 596888f65..01d63fd20 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/ado.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/ado.rs @@ -23,6 +23,12 @@ const ADVISORY_COMMENTS_STEP: &str = include_str!("../../../templates/ado/steps/ /// Embedded body of the dirty-file job wrapper. const JOB_WRAPPER: &str = include_str!("../../../templates/ado/steps/job.yml"); +/// Embedded body of the benchmark-history restore step template. +const BENCH_HISTORY_RESTORE_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-restore.yml"); + +/// Embedded body of the benchmark-findings build-summary step template. +const BENCH_HISTORY_SUMMARY_STEP: &str = include_str!("../../../templates/ado/steps/bench-history-summary.yml"); + /// Embedded body of the PR-tier stages template. const PR_STAGES: &str = include_str!("../../../templates/ado/pr-stages.yml"); @@ -55,6 +61,7 @@ const GROUPS: &[&str] = &[ "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ]; /// Embedded template for one per-group step. `__GROUP__` is substituted with @@ -67,6 +74,54 @@ const GROUP_PLACEHOLDER: &str = "__GROUP__"; /// Placeholder token the per-group template uses for the impact-mode selection. const IMPACT_MODE_PLACEHOLDER: &str = "__IMPACT_MODE__"; +/// Placeholder lines for steps a group needs around the uniform runner. +/// Substituted away entirely for groups that need none. +const PRE_STEPS_PLACEHOLDER: &str = "__PRE_STEPS__\n"; +const POST_STEPS_PLACEHOLDER: &str = "__POST_STEPS__\n"; + +/// Steps that run before the uniform group runner, per group. +/// +/// These live in the group's own emitted step template rather than at the +/// call site, so `pr.yml` / `scheduled.yml` stay a plain list of groups. A +/// group absent from the table gets nothing. +/// +/// The bodies are template files like every other emitted YAML, rather than +/// Rust string literals: they are fragments of a `steps:` list, so they are +/// not standalone-valid templates, but keeping them in `templates/ado/` +/// means the indentation is real YAML instead of escapes. +const GROUP_PRE_STEPS: &[(&str, &str)] = &[( + "scheduled-benchmarks", + include_str!("../../../templates/ado/steps/scheduled-benchmarks-pre.yml"), +)]; + +/// Steps that run after the uniform group runner, per group. +const GROUP_POST_STEPS: &[(&str, &str)] = &[ + ( + "scheduled-test", + include_str!("../../../templates/ado/steps/scheduled-test-post.yml"), + ), + ( + "scheduled-benchmarks", + include_str!("../../../templates/ado/steps/scheduled-benchmarks-post.yml"), + ), +]; + +/// The extra steps registered for `group`, with the fragment's own license +/// header removed: the emitted file already carries one, and a second copy +/// mid-list is noise. The explanatory comments are kept. +fn extra_steps(table: &'static [(&'static str, &'static str)], group: &str) -> String { + let Some(text) = table.iter().find_map(|&(name, steps)| (name == group).then_some(steps)) else { + return String::new(); + }; + let body: Vec<&str> = text + .lines() + .skip_while(|line| line.starts_with("# Copyright") || line.starts_with("# Licensed") || *line == "#") + .collect(); + let mut out = body.join("\n"); + out.push('\n'); + out +} + /// Render the step template for one group. #[must_use] fn render_group_step(group: &str) -> String { @@ -78,6 +133,8 @@ fn render_group_step(group: &str) -> String { rendered.replace_range(pos..pos + IMPACT_MODE_PLACEHOLDER.len(), impact_mode(group)); } rendered + .replace(PRE_STEPS_PLACEHOLDER, &extra_steps(GROUP_PRE_STEPS, group)) + .replace(POST_STEPS_PLACEHOLDER, &extra_steps(GROUP_POST_STEPS, group)) } /// Repo-root-relative path for one group's step template. @@ -111,6 +168,28 @@ pub fn job_wrapper() -> Artifact { Artifact::backend_file(Backend::Ado, ".pipelines/anvil/steps/job.yml", JOB_WRAPPER) } +/// `.pipelines/anvil/steps/bench-history-restore.yml` — restores the +/// benchmark history the previous scheduled run published. +#[must_use] +pub fn bench_history_restore() -> Artifact { + Artifact::backend_file( + Backend::Ado, + ".pipelines/anvil/steps/bench-history-restore.yml", + BENCH_HISTORY_RESTORE_STEP, + ) +} + +/// `.pipelines/anvil/steps/bench-history-summary.yml` — attaches the +/// benchmark findings to the build summary. +#[must_use] +pub fn bench_history_summary() -> Artifact { + Artifact::backend_file( + Backend::Ado, + ".pipelines/anvil/steps/bench-history-summary.yml", + BENCH_HISTORY_SUMMARY_STEP, + ) +} + /// `.pipelines/anvil/pr.yml` — the PR-tier stages template. #[must_use] pub fn pr_stages() -> Artifact { @@ -175,12 +254,20 @@ pub(crate) const GROUP_STEPS: &[(&str, &str)] = &[ ".pipelines/anvil/steps/scheduled-runtime-analysis.yml", ), ("scheduled-exhaustive", ".pipelines/anvil/steps/scheduled-exhaustive.yml"), + ("scheduled-benchmarks", ".pipelines/anvil/steps/scheduled-benchmarks.yml"), ]; /// All ADO backend artifacts in emission order. #[must_use] pub(crate) fn all() -> Vec { - let mut out = vec![setup_step(), impact_step(), advisory_comments(), job_wrapper()]; + let mut out = vec![ + setup_step(), + impact_step(), + advisory_comments(), + job_wrapper(), + bench_history_restore(), + bench_history_summary(), + ]; for (group, path) in GROUP_STEPS { out.push(Artifact::backend_file(Backend::Ado, path, render_group_step(group))); } @@ -387,16 +474,36 @@ mod tests { } #[test] - fn scheduled_stages_has_four_groups() { + fn per_group_step_keys_name_real_groups() { + // `extra_steps` matches by string equality and falls back to "", so a + // renamed or dropped group would silently lose its steps -- and for + // the benchmark group that means analyzing an empty store and + // reporting "no regressions", the one outcome it must never produce + // by accident. + for (table, label) in [(GROUP_PRE_STEPS, "pre"), (GROUP_POST_STEPS, "post")] { + let mut seen = Vec::new(); + for &(group, _) in table { + assert!(GROUPS.contains(&group), "{label}-step key '{group}' is not a catalog group"); + assert!(!seen.contains(&group), "{label}-step key '{group}' is registered twice"); + seen.push(group); + } + } + } + + #[test] + fn scheduled_stages_has_five_groups() { for needle in [ "stage: scheduled_test", "stage: scheduled_advisories", "stage: scheduled_runtime_analysis", "stage: scheduled_exhaustive", + "stage: scheduled_benchmarks", ] { assert!(SCHEDULED_STAGES.contains(needle), "scheduled stages missing '{needle}'"); } - assert!(SCHEDULED_STAGES.contains("PublishCodeCoverageResults@2")); + // Coverage publication lives in the group's own step template now; + // the stages file is a plain list of groups. + assert!(render_group_step("scheduled-test").contains("PublishCodeCoverageResults@2")); assert!(SCHEDULED_STAGES.contains("- template: steps/job.yml")); assert!( !SCHEDULED_STAGES.contains("\n - job: "), @@ -404,6 +511,76 @@ mod tests { ); } + #[test] + fn scheduled_benchmarks_stage_round_trips_the_history_artifact() { + // The stage is a plain list of groups; the round-trip lives in the + // group's own step template. + let group_step = render_group_step("scheduled-benchmarks"); + assert!( + !JOB_WRAPPER.contains("fetchDepth"), + "the job wrapper contract must stay frozen; put checkout in the group's step template" + ); + assert!(group_step.contains("- checkout: self")); + assert!(group_step.contains("fetchDepth: 0")); + // Benchmark inputs can be LFS-tracked. + assert!(group_step.contains("lfs: true")); + assert!(group_step.contains("template: bench-history-restore.yml")); + assert!(group_step.contains("template: bench-history-summary.yml")); + // A group with no registered extras gets none of this. + assert!(!render_group_step("scheduled-exhaustive").contains("bench-history")); + assert!(!render_group_step("scheduled-exhaustive").contains("checkout: self")); + // Coverage publication likewise moved off the call site. + assert!(render_group_step("scheduled-test").contains("PublishCodeCoverageResults@2")); + assert!( + !SCHEDULED_STAGES.contains("PublishCodeCoverageResults@2"), + "the stages template must not carry per-group steps" + ); + // The publish stays a job-level output so a forked (1ESPT) wrapper + // still translates it; the guard rides along as a `condition`. + assert_eq!( + SCHEDULED_STAGES + .matches("condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], ''))") + .count(), + 2, + "each leg's artifact entry carries the restore guard" + ); + assert!(JOB_WRAPPER.contains("${{ if artifact.condition }}")); + // Take the newest build carrying the artifact whatever its outcome: + // restoring only from green builds would drop every sample collected + // while the pipeline was red from a regression. + assert!(BENCH_HISTORY_RESTORE_STEP.contains("queryOrder=finishTimeDescending")); + // Absence and operational failure must stay distinguishable, or one + // transient error publishes an empty store over a good history. + assert!(BENCH_HISTORY_RESTORE_STEP.contains("if ($status -eq 404) { continue }")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("Complete-Restore 'restored'")); + assert!(BENCH_HISTORY_RESTORE_STEP.contains("Complete-Restore 'cold-start'")); + // Fail-closed by construction, and atomically: the store is assembled + // beside the published path and moved into place in one step, so + // neither an operational failure nor a copy that dies partway leaves + // anything for a wrapper that ignores the artifact condition to + // upload over the accumulated chain. + assert_eq!( + BENCH_HISTORY_RESTORE_STEP + .matches("Move-Item -LiteralPath $pending -Destination $path") + .count(), + 1, + "the store path must appear only via the atomic move" + ); + assert!( + !BENCH_HISTORY_RESTORE_STEP.contains("Copy-Item -Path (Join-Path $staging '*') -Destination $path"), + "copying straight into the published path can leave a partial store there" + ); + assert!( + !BENCH_HISTORY_RESTORE_STEP.contains("continueOnError"), + "a blanket continueOnError would read every failure as a cold start" + ); + assert!(BENCH_HISTORY_SUMMARY_STEP.contains("##vso[task.uploadsummary]")); + assert!(BENCH_HISTORY_SUMMARY_STEP.contains("condition: succeededOrFailed()")); + // The machine-key escape hatch is an input on this backend too. + assert!(SCHEDULED_STAGES.contains("name: benchMachineKey")); + assert!(SCHEDULED_STAGES.contains("ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }}")); + } + #[test] fn setup_caches_cargo_install_metadata_without_eager_libclang_install() { assert!(SETUP_STEP.contains("$(HOME)/.cargo/.crates.toml")); diff --git a/crates/cargo-anvil/src/anvil/artifacts/github.rs b/crates/cargo-anvil/src/anvil/artifacts/github.rs index e4977457d..4938c36a0 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/github.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/github.rs @@ -143,6 +143,7 @@ mod tests { "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ]; #[test] @@ -485,6 +486,81 @@ export -f just ); } + #[test] + fn scheduled_benchmarks_job_round_trips_the_history_artifact() { + // The round-trip is a parameter of the shared group action, not a + // second way to run a group: the eight other jobs skip it, and the + // scheduled workflow stays a plain list of groups. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_history: true")); + // Per-leg identity is supplied by the caller. The matrix value is in + // scope only in the workflow, and merging two runners' samples into + // one series would destroy the comparison. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_artifact: bench-history-${{ matrix.os }}")); + assert_eq!( + SCHEDULED_IMPL_WORKFLOW.matches("bench_history: true").count(), + 1, + "only the benchmark group pays for the history round-trip" + ); + // Neither of these can be expressed inside an action: an action + // cannot request permissions, and the checkout has already run by + // the time it starts. They are the whole per-group remainder. + assert!(SCHEDULED_IMPL_WORKFLOW.contains("fetch-depth: 0")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("actions: read")); + assert!(SCHEDULED_ROOT_WORKFLOW.contains("actions: read")); + assert!(SCHEDULED_IMPL_WORKFLOW.contains("bench_machine_key:")); + + // Everything else lives in the action, guarded so the other groups + // skip it. + assert!(RUN_GROUP_ACTION.contains("if: inputs.bench_history == 'true'")); + assert!(RUN_GROUP_ACTION.contains("ARTIFACT: ${{ inputs.bench_artifact }}")); + assert!(RUN_GROUP_ACTION.contains("name: ${{ inputs.bench_artifact }}")); + assert!(RUN_GROUP_ACTION.contains("ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }}")); + assert!(RUN_GROUP_ACTION.contains("actions/upload-artifact@")); + assert!(RUN_GROUP_ACTION.contains("gh run download")); + assert!(RUN_GROUP_ACTION.contains("GITHUB_STEP_SUMMARY")); + // An enabled round-trip with no artifact name would restore nothing + // and publish nothing, reporting a false clean every run. + assert!(RUN_GROUP_ACTION.contains("if [ -z \"$ARTIFACT\" ]; then")); + // The workflow is identified by its runtime name, not a literal + // filename: the root workflow is owned and renameable, and a rename + // must not silently reset the series. + assert!(RUN_GROUP_ACTION.contains("WORKFLOW: ${{ github.workflow }}")); + assert!( + !RUN_GROUP_ACTION.contains("--workflow anvil-scheduled.yml"), + "a hardcoded workflow filename breaks on rename" + ); + // Absence and operational failure must stay distinguishable. The run + // listing is assigned rather than consumed by `for`, because `set -e` + // ignores a command substitution used as a word list -- a failed + // listing would otherwise read as a cold start. + assert!(RUN_GROUP_ACTION.contains("if ! run_ids=")); + assert!(RUN_GROUP_ACTION.contains("select(.name == \\\"$ARTIFACT\\\" and .expired == false)")); + assert!(RUN_GROUP_ACTION.contains("complete_restore restored")); + assert!(RUN_GROUP_ACTION.contains("complete_restore cold-start")); + // Fail-closed by construction: the store path is created only inside + // complete_restore, so an operational failure leaves nothing to + // upload over the accumulated chain. + assert_eq!( + RUN_GROUP_ACTION.matches("mkdir -p target/anvil/bench-history").count(), + 1, + "the store path must be created only on a completed restore" + ); + assert!(RUN_GROUP_ACTION.contains("if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != ''")); + // Notifying a human is the scheduled tier's own publish-failure job, + // which this group must be a dependency of -- otherwise a benchmark + // regression fails the run without ever reaching the tracking issue. + assert!(!SCHEDULED_IMPL_WORKFLOW.contains("gh issue create")); + let publish_needs = SCHEDULED_IMPL_WORKFLOW + .split_once(" publish-failure:") + .and_then(|(_, rest)| rest.split_once(" if:")) + .map(|(needs, _)| needs) + .expect("publish-failure declares its needs before its if"); + assert!( + publish_needs.contains("- scheduled-benchmarks"), + "publish-failure must depend on scheduled-benchmarks:\n{publish_needs}" + ); + } + #[test] #[cfg_attr(miri, ignore = "uses filesystem and subprocesses; miri isolation forbids them")] #[expect(clippy::too_many_lines, reason = "keeps the reporter lifecycle scenarios together")] diff --git a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs index 8546d786b..70b9c139e 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/justfile.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/justfile.rs @@ -95,6 +95,7 @@ const CHECK_FILES: &[(&str, &str)] = split_recipe_files!( "aprz", "audit", "bench", + "bench-history", "bolero", "careful", "cargo-hack", @@ -139,6 +140,7 @@ const GROUP_FILES: &[(&str, &str)] = split_recipe_files!( "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ] ); @@ -253,6 +255,7 @@ mod tests { ("aprz", Unscoped), ("audit", Unscoped), ("bench", Affected), + ("bench-history", Unscoped), ("bolero", Affected), ("careful", Affected), ("cargo-hack", Required), @@ -502,7 +505,7 @@ mod tests { let unscoped = EXPECTED_CHECK_POLICY.len() - scoped; assert_eq!( (scoped, unscoped), - (23, 7), + (23, 8), "impact scoped/unscoped split changed; update EXPECTED_CHECK_POLICY deliberately" ); } @@ -539,6 +542,7 @@ mod tests { "anvil-scheduled-test:", "anvil-scheduled-advisories:", "anvil-scheduled-exhaustive:", + "anvil-scheduled-benchmarks:", ] { assert!(groups.contains(needle), "groups tree missing '{needle}'"); } @@ -568,6 +572,7 @@ mod tests { "scheduled-advisories", "scheduled-runtime-analysis", "scheduled-exhaustive", + "scheduled-benchmarks", ] { assert!( groups.contains(&format!("anvil-{g}: (_anvil-unscoped \"{g}\")")), @@ -629,6 +634,36 @@ mod tests { } } + #[test] + fn bench_history_gates_on_active_regressions_and_applies_blessings() { + let (_, body) = CHECK_FILES + .iter() + .find(|(path, _)| *path == "justfiles/anvil/checks/bench-history.just") + .expect("bench-history check template is registered"); + + // cargo-bench-history never fails on findings, so the recipe must + // read the JSON report and gate on active regressions itself. + for needle in [ + "cargo bench-history collect", + "--skip-existing", + "cargo bench-history analyze", + "_anvil-bench-history-bless", + "cargo bench-history bless", + "cargo bench-history list blessings", + ".config/bench-blessings.toml", + "$_.direction -eq 'regression' -and $_.active", + ] { + assert!(body.contains(needle), "bench-history template missing '{needle}'"); + } + + // The series is only comparable when the same suite is measured at + // every commit, so the recipe must not honour impact scoping. + assert!( + !body.contains("ANVIL_INCLUDE_AFFECTED"), + "bench-history must measure the whole workspace, not an impact-scoped subset" + ); + } + #[test] fn tiers_just_template_has_three_tiers() { for needle in ["anvil-pr:", "anvil-scheduled:", "anvil-full:", "_anvil-scheduled:", "_anvil-full:"] { @@ -665,6 +700,7 @@ mod tests { "anvil-scheduled-advisories", "anvil-scheduled-runtime-analysis", "anvil-scheduled-exhaustive", + "anvil-scheduled-benchmarks", ] { assert!(TIERS_JUST.contains(needle), "scheduled tier must reference group '{needle}'"); } @@ -707,6 +743,7 @@ mod tests { "import? 'container.just'", "import 'groups/pr-fast.just'", "import 'groups/scheduled-exhaustive.just'", + "import 'groups/scheduled-benchmarks.just'", "import 'tiers.just'", "import 'tools.just'", "import 'versions.just'", diff --git a/crates/cargo-anvil/src/anvil/artifacts/mod.rs b/crates/cargo-anvil/src/anvil/artifacts/mod.rs index 5d1773bcf..56b352ca4 100644 --- a/crates/cargo-anvil/src/anvil/artifacts/mod.rs +++ b/crates/cargo-anvil/src/anvil/artifacts/mod.rs @@ -43,7 +43,7 @@ use crate::catalog::{Artifact, ComposedHost}; pub(crate) fn impact_mode(group: &str) -> &'static str { match group { "pr-fast" | "pr-test" | "pr-runtime-analysis" | "pr-mutants" => "consume", - "scheduled-test" | "scheduled-advisories" | "scheduled-runtime-analysis" | "scheduled-exhaustive" => "off", + "scheduled-test" | "scheduled-advisories" | "scheduled-runtime-analysis" | "scheduled-exhaustive" | "scheduled-benchmarks" => "off", other => { panic!("impact_mode: unclassified group '{other}'; add it to the pr/scheduled arms in artifacts::impact_mode") } diff --git a/crates/cargo-anvil/src/lib.rs b/crates/cargo-anvil/src/lib.rs index 285dfcd66..20d6534c7 100644 --- a/crates/cargo-anvil/src/lib.rs +++ b/crates/cargo-anvil/src/lib.rs @@ -319,6 +319,7 @@ //! scheduled-exhaustivemutants-full //! cargo-hackfeature powerset //! benchcompile-only +//! scheduled-benchmarksbench-historyregression detection over the accumulated benchmark history //! //! //! diff --git a/crates/cargo-anvil/src/run.rs b/crates/cargo-anvil/src/run.rs index 228305fce..27de5fe31 100644 --- a/crates/cargo-anvil/src/run.rs +++ b/crates/cargo-anvil/src/run.rs @@ -1141,6 +1141,7 @@ mod tests { "justfiles/anvil/checks/miri.just", "justfiles/anvil/groups/pr-fast.just", "justfiles/anvil/groups/scheduled-exhaustive.just", + "justfiles/anvil/groups/scheduled-benchmarks.just", "justfiles/anvil/tiers.just", "justfiles/anvil/tools.just", "justfiles/anvil/versions.just", @@ -1540,6 +1541,9 @@ mod tests { ".pipelines/anvil/steps/scheduled-advisories.yml", ".pipelines/anvil/steps/scheduled-runtime-analysis.yml", ".pipelines/anvil/steps/scheduled-exhaustive.yml", + ".pipelines/anvil/steps/scheduled-benchmarks.yml", + ".pipelines/anvil/steps/bench-history-restore.yml", + ".pipelines/anvil/steps/bench-history-summary.yml", ".pipelines/anvil/pr.yml", ".pipelines/anvil/scheduled.yml", ".pipelines/anvil-pr.yml", diff --git a/crates/cargo-anvil/templates/ado/scheduled-stages.yml b/crates/cargo-anvil/templates/ado/scheduled-stages.yml index f1cb349d7..c7aefb07e 100644 --- a/crates/cargo-anvil/templates/ado/scheduled-stages.yml +++ b/crates/cargo-anvil/templates/ado/scheduled-stages.yml @@ -13,37 +13,32 @@ parameters: - name: windowsPool type: object default: { vmImage: windows-latest } + - name: benchMachineKey + type: string + # Machine key the benchmark history is partitioned by. Empty uses + # cargo-bench-history's hardware fingerprint; set a stable pool label + # when the agent pool is heterogeneous enough to fragment a series into + # partitions too sparse to analyze. + default: '' stages: - stage: scheduled_test displayName: anvil scheduled-test jobs: - # Publish coverage from both legs so OS-gated code is fully - # represented (see the pr-stages.yml comment for the rationale). + # Coverage is published from both legs so OS-gated code is fully + # represented; that step lives in steps/scheduled-test.yml. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (linux) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (windows) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - stage: scheduled_advisories displayName: anvil scheduled-advisories @@ -106,3 +101,42 @@ stages: pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-exhaustive.yml + + - stage: scheduled_benchmarks + displayName: anvil scheduled-benchmarks + dependsOn: [] + variables: + # Read by the bench-history recipe; empty means "use the hardware + # fingerprint". ADO exports pipeline variables as environment + # variables, so this reaches the recipe without further plumbing. + ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} + jobs: + # The history round-trip lives in steps/scheduled-benchmarks.yml. Only + # the artifact declaration is here, because publishing is a job-level + # output: a forked (1ESPT) wrapper translates this list into its own + # output shape, which a task inside the step list would bypass. + # + # The condition is the guard that keeps an operational restore failure + # from overwriting a good chain with a truncated store; failed runs + # still publish, since a flagged regression fails the job and those + # samples belong in the history. + - template: steps/job.yml + parameters: + name: linux + pool: ${{ parameters.linuxPool }} + artifacts: + - name: bench-history-$(Agent.JobName) + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) + steps: + - template: steps/scheduled-benchmarks.yml + - template: steps/job.yml + parameters: + name: windows + pool: ${{ parameters.windowsPool }} + artifacts: + - name: bench-history-$(Agent.JobName) + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) + steps: + - template: steps/scheduled-benchmarks.yml diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml new file mode 100644 index 000000000..d32bf2bba --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-restore.yml @@ -0,0 +1,147 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Restores the benchmark history published by the most recent build that +# carries this leg's artifact. +# +# `DownloadPipelineArtifact@2` with `latestFromBranch` resolves a single +# build and yields nothing if that build has no such artifact -- it does +# not walk back. A cancelled or never-publishing latest build would +# therefore cold-start a store that actually has history. This step +# resolves the build itself, so absence of the artifact throughout the +# window (a genuine cold start) stays distinct from an operational +# failure. +# +# The store path is created only once the restore has reached a known +# state -- restored, or a positively identified cold start. An operational +# failure therefore leaves *no store directory at all*, so a publisher +# that runs unconditionally has nothing to upload and the accumulated +# chain survives. That makes the guard structural rather than dependent on +# every job wrapper honouring the `artifacts` condition: a fork predating +# that field ignores it silently, and this is the case where ignoring it +# would overwrite good history with a truncated store. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history + - name: window + type: number + default: 30 +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $artifact = '${{ parameters.artifact }}' + $path = '${{ parameters.path }}' + $window = ${{ parameters.window }} + + if (-not $env:SYSTEM_ACCESSTOKEN) { + Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." + exit 1 + } + + # Everything lands here first; $path is created only on success. + # $PID-scoped: on a self-hosted agent two concurrent jobs share the + # temp directory, and an artifact name is not unique across + # overlapping pipeline runs. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging-$PID" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + # Publishes the staged store as the restore's result and records how + # the restore ended, so the publish step can tell a continuation of + # the chain from a job that never established one. + function Complete-Restore([string]$state) { + # Assembled beside the published path and moved into place in one + # step. A copy that failed partway would otherwise leave a partial + # store exactly where a wrapper forked before `artifacts[].condition` + # existed -- which ignores that field silently -- would publish it + # over the accumulated chain. + $parent = Split-Path -Parent $path + if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } + $pending = "$path.pending-$PID" + Remove-Item -Recurse -Force $pending -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $pending | Out-Null + if (Test-Path (Join-Path $staging '*')) { + Copy-Item -Path (Join-Path $staging '*') -Destination $pending -Recurse -Force + } + Remove-Item -Recurse -Force $path -ErrorAction SilentlyContinue + Move-Item -LiteralPath $pending -Destination $path + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + # The recipe refuses to write anywhere but here, so a wrapper that + # retargets `path` cannot silently detach the recipe's store from + # the one that is actually published. + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_WIRED_STORE]$path" + } + + $collection = $env:SYSTEM_COLLECTIONURI + $project = $env:SYSTEM_TEAMPROJECTID + $definition = $env:SYSTEM_DEFINITIONID + $headers = @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + + # The series is keyed on the repository's default branch, matching the + # GitHub backend and the documented contract. Keying on the branch the + # build happened to run on would fork a fresh, one-sample history for + # every manual or release-branch run and report that as clean. + $definitionUri = "$collection$project/_apis/build/definitions/$definition" + "?api-version=7.0" + $branch = (Invoke-RestMethod -Uri $definitionUri -Headers $headers).repository.defaultBranch + if (-not $branch) { + Write-Error "anvil: could not resolve the pipeline's default branch, so the history series cannot be identified; refusing to continue rather than fork a new chain under the current branch." + exit 1 + } + # Newest first, whatever the outcome: a flagged regression fails the + # build, so restricting to successful builds would discard exactly + # the stretch of history that matters most. + $runsUri = "$collection$project/_apis/build/builds?definitions=$definition&branchName=$branch&`$top=$window&queryOrder=finishTimeDescending&api-version=7.0" + $runs = (Invoke-RestMethod -Uri $runsUri -Headers $headers).value + + foreach ($run in $runs) { + if ($run.id -eq $env:BUILD_BUILDID) { continue } + + $artifactUri = "$collection$project/_apis/build/builds/$($run.id)/artifacts?artifactName=$artifact&api-version=7.0" + try { + $found = Invoke-RestMethod -Uri $artifactUri -Headers $headers + } catch { + # 404 is "this build has no such artifact" -- the expected, + # common case while walking back. Anything else is an + # operational failure and must not be read as absence. + $status = $_.Exception.Response.StatusCode.value__ + if ($status -eq 404) { continue } + Write-Error "anvil: querying artifacts of build $($run.id) failed with HTTP $status; refusing to continue with an empty history." + exit 1 + } + if (-not $found.resource.downloadUrl) { continue } + + Write-Host "anvil: restoring $artifact from build $($run.id)" + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-$PID.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract-$PID" + Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue + Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip + Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force + # The archive nests its contents under a directory named for the + # artifact; lift them up into the staging directory. An artifact + # that is present but empty is a valid restore -- a workspace with + # no benchmarks yet publishes exactly that -- and `Copy-Item` over + # an empty wildcard throws, so the copy is guarded. + $inner = Join-Path $extract $artifact + $source = if (Test-Path $inner) { $inner } else { $extract } + if (Test-Path (Join-Path $source '*')) { + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + } + Complete-Restore 'restored' + exit 0 + } + + # Nothing in the window carried the artifact: a genuine cold start, + # which is a valid empty store rather than a failure. + Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" + Complete-Restore 'cold-start' + displayName: Restore ${{ parameters.artifact }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml b/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml new file mode 100644 index 000000000..c01ed11fe --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/bench-history-summary.yml @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Attaches the benchmark findings to the build summary, so a failed +# scheduled build carries the per-finding detail its one-bit status +# cannot. Runs whether or not the analysis flagged anything. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: findings + type: string + default: target/anvil/bench/findings.md +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $findings = '${{ parameters.findings }}' + if (-not (Test-Path -LiteralPath $findings)) { + Write-Host "anvil: no $findings to attach" + exit 0 + } + $full = (Resolve-Path -LiteralPath $findings).Path + Write-Host "##vso[task.uploadsummary]$full" + displayName: Publish benchmark findings + condition: succeededOrFailed() diff --git a/crates/cargo-anvil/templates/ado/steps/group.yml b/crates/cargo-anvil/templates/ado/steps/group.yml index 173e18946..f5d9d5601 100644 --- a/crates/cargo-anvil/templates/ado/steps/group.yml +++ b/crates/cargo-anvil/templates/ado/steps/group.yml @@ -12,7 +12,12 @@ # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: +__PRE_STEPS__ - template: setup.yml parameters: group: __GROUP__ @@ -61,3 +66,4 @@ steps: # pure cost with no cross-run benefit. Mirrors the GitHub impl workflows' # workflow-level CARGO_INCREMENTAL=0. CARGO_INCREMENTAL: "0" +__POST_STEPS__ diff --git a/crates/cargo-anvil/templates/ado/steps/job.yml b/crates/cargo-anvil/templates/ado/steps/job.yml index 8e4c40b02..a7dd7de30 100644 --- a/crates/cargo-anvil/templates/ado/steps/job.yml +++ b/crates/cargo-anvil/templates/ado/steps/job.yml @@ -17,6 +17,10 @@ # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# A step list may lead with its own `checkout: self` +# (the benchmark group does, for depth and LFS), so a +# wrapper must NOT add a checkout of its own -- doing +# so would give those jobs two. # - inputArtifacts (object) Optional list of pipeline artifacts to download # BEFORE the steps run. Each item: { name, path }. # The default wrapper prepends one @@ -27,14 +31,18 @@ # `anvil-impact-` artifact into target/anvil/impact # and its checks read the cache exactly as a local run. # - artifacts (object) Optional list of pipeline artifacts to publish. -# Each item: { name: string, path: string }. -# The default wrapper appends one -# PublishPipelineArtifact@1 task per entry; 1ESPT -# wrappers translate these into +# Each item: { name: string, path: string, +# condition: string (optional) }. The default wrapper +# appends one PublishPipelineArtifact@1 task per +# entry; 1ESPT wrappers translate these into # templateContext.outputs.pipelineArtifact blocks # at job level. The contract is the same either way, # so the stages templates don't need to know which # backend they're targeting. +# `condition` defaults to succeededOrFailed(); a +# caller that must not publish in some states sets +# it, and a fork is expected to carry it through to +# whatever output shape it emits. parameters: - name: name type: string @@ -70,7 +78,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml new file mode 100644 index 000000000..a6c54ee5e --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-post.yml @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, after the uniform runner. Not a standalone template: this is +# part of a `steps:` list. + - template: bench-history-summary.yml diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml new file mode 100644 index 000000000..c0d9a982a --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-benchmarks-pre.yml @@ -0,0 +1,26 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, ahead of the uniform runner. Not a standalone template: this is +# part of a `steps:` list, which is why it has no `steps:` key of its own. +# +# The analysis orders each series by first-parent commit topology and +# locates the merge-base, so it needs the whole commit graph. LFS matters +# because benchmark inputs can be LFS-tracked and would otherwise arrive as +# pointer files. +# +# The checkout is explicit rather than a job-wrapper parameter: job.yml is +# the file adopters fork, so binding a parameter their copy lacks would fail +# expansion for the whole pipeline. +# +# The artifact is named for the *job*, which ADO guarantees is unique within +# a stage. Naming it for the agent OS instead would let two legs on like-OS +# pools declare one artifact, colliding on publish and merging two machines' +# samples into a single series. + - checkout: self + fetchDepth: 0 + lfs: true + - template: bench-history-restore.yml + parameters: + artifact: bench-history-$(Agent.JobName) diff --git a/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml b/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml new file mode 100644 index 000000000..519f79120 --- /dev/null +++ b/crates/cargo-anvil/templates/ado/steps/scheduled-test-post.yml @@ -0,0 +1,15 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# +# Fragment spliced into the scheduled-test group's emitted step template, +# after the uniform runner. Not a standalone template: this is part of a +# `steps:` list. +# +# Coverage is published from every leg so OS-gated code is fully +# represented. + - task: PublishCodeCoverageResults@2 + condition: succeededOrFailed() + displayName: Publish coverage + inputs: + summaryFileLocation: target/coverage/cobertura-*.xml + failIfCoverageEmpty: false diff --git a/crates/cargo-anvil/templates/github/run-group-action.yml b/crates/cargo-anvil/templates/github/run-group-action.yml index ce3c159fe..67257a906 100644 --- a/crates/cargo-anvil/templates/github/run-group-action.yml +++ b/crates/cargo-anvil/templates/github/run-group-action.yml @@ -34,6 +34,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -43,6 +66,110 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -53,6 +180,7 @@ runs: # target/anvil/impact cache (read via `_anvil-impact-include`), not # threaded --package strings. This action only fixes the mode. ANVIL_IMPACT: ${{ inputs.impact_mode }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -75,6 +203,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true diff --git a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml index 982dc1ae6..7071d04f1 100644 --- a/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-impl-workflow.yml @@ -24,6 +24,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -141,6 +149,35 @@ jobs: with: group: scheduled-exhaustive + scheduled-benchmarks: + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. + fetch-depth: 0 + lfs: true + - uses: ./.github/actions/anvil-run-group + with: + group: scheduled-benchmarks + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} + publish-failure: name: Publish scheduled failure needs: @@ -148,6 +185,7 @@ jobs: - scheduled-advisories - scheduled-runtime-analysis - scheduled-exhaustive + - scheduled-benchmarks if: ${{ always() && vars.ANVIL_PUBLISH_FAILURE_ISSUE != 'false' && contains(needs.*.result, 'failure') }} runs-on: ${{ inputs.linux_runner }} diff --git a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml index 06a001521..52830ab86 100644 --- a/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml +++ b/crates/cargo-anvil/templates/github/scheduled-root-workflow.yml @@ -21,5 +21,8 @@ jobs: # and restores that scope only on publish-failure. permissions: contents: read + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. Narrowed to that job inside. + actions: read issues: write secrets: inherit diff --git a/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just new file mode 100644 index 000000000..85e2ac7b2 --- /dev/null +++ b/crates/cargo-anvil/templates/justfiles/anvil/checks/bench-history.just @@ -0,0 +1,320 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh", "-NoProfile")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on + # every platform, so on Linux two genuinely different directories would + # compare equal and the detached-store failure this guard exists to + # catch would slip through. + $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } + if (-not $same) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if ([string]::IsNullOrWhiteSpace($value)) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh", "-NoProfile")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + # `benchmark` and `commit` are passed to cbh and git. A leading `-` + # would be read as an option there, so reject it here rather than let + # the file's documented subset be reinterpreted: `benchmark = "--all"` + # would otherwise accept every benchmark at the commit while the log + # claimed a benchmark of that name. + foreach ($guarded in @('benchmark', 'commit')) { + if ($e[$guarded].StartsWith('-')) { + Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + # A blessing can only be applied at a commit the store still holds a clean + # run for: cbh rejects a context commit with no data point, and `collect` + # only ever records the current commit, so a commit the store has forgotten + # can never be re-established by a later run. Without this, any cold start + # or artifact eviction past a blessed commit would turn the group + # permanently red until a human edited the ledger -- a blessing whose + # commit has aged out is inapplicable, not invalid. + $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" + & cargo bench-history list runs --local="$store" ` + --since 1970-01-01 --no-text --json $runsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $stored = [System.Collections.Generic.List[string]]::new() + foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { + foreach ($c in $set.commits) { + if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } + } + } + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Compare the persisted identity exactly. `list blessings --all` is the + # window view: it reports resolved, concrete benchmark ids and never + # populates `prefixes`. Since cbh's positional is a prefix, a ledger + # entry naming a family expands to several concrete rows and matches + # none of them, so such an entry is re-applied each run. That is the + # safe direction to be wrong in: re-blessing is idempotent in effect, + # whereas a loose prefix test would skip an entry that is only + # partially applied and leave the build red while claiming otherwise. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { + Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + # `--` so a benchmark id is never parsed as a flag: bless's positional + # takes no hyphen values, and it has an `--all` sibling that would + # otherwise accept every benchmark at the commit. + & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs diff --git a/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just b/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just new file mode 100644 index 000000000..9a4e119aa --- /dev/null +++ b/crates/cargo-anvil/templates/justfiles/anvil/groups/scheduled-benchmarks.just @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs diff --git a/crates/cargo-anvil/templates/justfiles/anvil/mod.just b/crates/cargo-anvil/templates/justfiles/anvil/mod.just index ee29054c9..8f937ae22 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/mod.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/mod.just @@ -40,6 +40,7 @@ import 'impact.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -81,6 +82,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'tiers.just' import 'tools.just' import 'versions.just' diff --git a/crates/cargo-anvil/templates/justfiles/anvil/tiers.just b/crates/cargo-anvil/templates/justfiles/anvil/tiers.just index a06e87b4f..9e0822b6a 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/tiers.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/tiers.just @@ -39,7 +39,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full-workspace for the same reason as the scheduled tier. @@ -82,7 +83,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -90,7 +92,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] diff --git a/crates/cargo-anvil/templates/justfiles/anvil/tools.just b/crates/cargo-anvil/templates/justfiles/anvil/tools.just index cb6aab145..9b0c751dc 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/tools.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/tools.just @@ -550,6 +550,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the diff --git a/crates/cargo-anvil/templates/justfiles/anvil/versions.just b/crates/cargo-anvil/templates/justfiles/anvil/versions.just index 0a4aeaa13..c9ad5d32c 100644 --- a/crates/cargo-anvil/templates/justfiles/anvil/versions.just +++ b/crates/cargo-anvil/templates/justfiles/anvil/versions.just @@ -51,6 +51,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.1.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.10" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/recipe_contracts.rs b/crates/cargo-anvil/tests/recipe_contracts.rs index fb4d8349b..d20b9875b 100644 --- a/crates/cargo-anvil/tests/recipe_contracts.rs +++ b/crates/cargo-anvil/tests/recipe_contracts.rs @@ -109,6 +109,44 @@ if ($args -contains 'nextest') { } exit [int]$env:FAKE_NEXTEST_EXIT } +if ($args -contains 'bench-history') { + # $args inside a function refers to that function's own arguments, so + # the script's are captured here and passed in explicitly. + $cbhArgs = $args + # Writes whatever report the scenario asked for to the path the recipe + # passed, so the recipe's own parsing and gating are what get exercised. + function Write-Report([string[]]$all, [string]$flag, [string]$content) { + $index = [array]::IndexOf($all, $flag) + if ($index -ge 0 -and $content) { + $target = $all[$index + 1] + $parent = Split-Path -Parent $target + if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } + Set-Content -LiteralPath $target -Value $content -Encoding UTF8 + } + } + if ($cbhArgs -contains 'collect') { exit [int]$env:FAKE_CBH_COLLECT_EXIT } + if ($cbhArgs -contains 'analyze') { + Write-Report $cbhArgs '--json' $env:FAKE_CBH_FINDINGS + Write-Report $cbhArgs '--markdown' 'findings' + Write-Report $cbhArgs '--markdown-summary' 'summary' + exit [int]$env:FAKE_CBH_ANALYZE_EXIT + } + if ($cbhArgs -contains 'list') { + if ($cbhArgs -contains 'runs') { + $runs = if ($env:FAKE_CBH_RUNS) { + $env:FAKE_CBH_RUNS + } else { + '{"sets":[{"commits":[{"commit":"8392995a3b94","runs":1,"clean":1,"dirty":0}]}]}' + } + Write-Report $cbhArgs '--json' $runs + } else { + Write-Report $cbhArgs '--json' $env:FAKE_CBH_BLESSINGS + } + exit 0 + } + if ($cbhArgs -contains 'bless') { exit [int]$env:FAKE_CBH_BLESS_EXIT } + exit 0 +} if ($args -contains 'binstall') { exit [int]$env:FAKE_BINSTALL_EXIT } @@ -164,7 +202,7 @@ fn fixture(imports: &[(&str, &str)], dependency_recipes: &[&str]) -> TempDir { let bin = tmp.path().join("fake-bin"); fs::create_dir_all(&bin).unwrap(); write(&bin.join("cargo.ps1"), FAKE_CARGO_PS1); - write(&bin.join("git.ps1"), "exit 0\n"); + write(&bin.join("git.ps1"), FAKE_GIT); tmp } @@ -1452,3 +1490,794 @@ fn symlink_file(target: &Path, link: &Path) -> std::io::Result<()> { fn symlink_dir(target: &Path, link: &Path) -> std::io::Result<()> { std::os::unix::fs::symlink(target, link) } + +const BENCH_HISTORY: &str = include_str!("../templates/justfiles/anvil/checks/bench-history.just"); + +/// A findings report with one active regression — the state that must gate. +const ACTIVE_REGRESSION: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"regression", + "active":true,"relative_delta":0.4966,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// The same finding after it recovered: reported, but nothing to act on. +const INACTIVE_REGRESSION: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"regression", + "active":false,"relative_delta":0.4966,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// An improvement — never a reason to fail. +const IMPROVEMENT: &str = r#"{"notable":true,"findings":[ + {"segments":["emit_alloc","churn"],"kind":"wall_time","direction":"improvement", + "active":true,"relative_delta":-0.31,"confidence":1.0,"commit":"8392995a"}]}"#; + +/// What an empty workspace analyzes to: no runs, no findings. +const NO_FINDINGS: &str = r#"{"notable":false,"findings":[]}"#; + +/// The stand-in `git`, covering the commit resolution the blessing +/// reconciliation performs. +const FAKE_GIT: &str = r" +if ($args -contains 'rev-parse') { + # The recipe resolves a possibly-abbreviated commit before blessing; + # FAKE_GIT_UNKNOWN_COMMIT makes that resolution fail. + if ($env:FAKE_GIT_UNKNOWN_COMMIT) { exit 128 } + Write-Output '8392995a3b94218612437d0b868df2a48029b6ea' + exit 0 +} +exit 0 +"; + +/// Both streams of a recipe run, for assertion messages. +/// +/// A recipe that dies before producing output says why on stderr, so a +/// failure message carrying only stdout hides the actual cause. +fn both_streams(output: &Output) -> String { + format!( + "\n--- stdout ---\n{}\n--- stderr ---\n{}\n--- status: {:?} ---", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + output.status + ) +} + +/// Runs `anvil-bench-history` in a fixture, with the fake cbh returning +/// `findings` and the given extra environment. Gating is CI-only, so the +/// scenarios that assert on the exit code set `ANVIL_BENCH_GATE`. +fn run_bench_history(findings: &str, extra: &[(&str, &OsStr)]) -> (TempDir, Output) { + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + let log = tmp.path().join("cargo.log"); + let mut environment: Vec<(&str, &OsStr)> = vec![ + ("FAKE_CBH_FINDINGS", OsStr::new(findings)), + ("FAKE_CARGO_LOG", log.as_os_str()), + ("ANVIL_BENCH_GATE", OsStr::new("1")), + ]; + environment.extend_from_slice(extra); + let output = run_just(tmp.path(), &["anvil-bench-history"], &environment); + (tmp, output) +} + +fn cargo_calls(root: &Path) -> String { + std::fs::read_to_string(root.join("cargo.log")).unwrap_or_default() +} + +#[test] +fn bench_history_gates_on_active_regressions_only() { + if !tools_available() { + return; + } + + // An active regression is the one state that gates. + let (tmp, output) = run_bench_history(ACTIVE_REGRESSION, &[]); + assert_failed(&output, "an active regression"); + let text = String::from_utf8_lossy(&output.stdout); + assert!(text.contains("emit_alloc/churn"), "names the benchmark:{}", both_streams(&output)); + assert!(text.contains("8392995a"), "names the attributed commit:{}", both_streams(&output)); + let calls = cargo_calls(tmp.path()); + assert!(calls.contains("bench-history collect"), "calls:\n{calls}"); + assert!(calls.contains("bench-history analyze"), "calls:\n{calls}"); + + // A recovered regression and an improvement both need no action. + for (findings, label) in [(INACTIVE_REGRESSION, "inactive"), (IMPROVEMENT, "improvement")] { + let (_tmp, output) = run_bench_history(findings, &[]); + assert!(output.status.success(), "{label} finding must not gate:{}", both_streams(&output)); + } + + // A workspace with no benchmarks analyzes to nothing and stays green: + // adopting the capability must not turn such a repo permanently red. + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[]); + assert!( + output.status.success(), + "an empty history must be a clean no-op:{}", + both_streams(&output) + ); +} + +#[test] +fn bench_history_reports_without_gating_outside_ci() { + if !tools_available() { + return; + } + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + // No ANVIL_BENCH_GATE, and the CI markers explicitly cleared: a laptop's + // measurement noise must not fail a pre-release `anvil-full` and invite + // silencing it with a committed blessing. + let output = run_just( + tmp.path(), + &["anvil-bench-history"], + &[ + ("FAKE_CBH_FINDINGS", OsStr::new(ACTIVE_REGRESSION)), + ("CI", OsStr::new("")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "a local run reports but does not gate:{}", + both_streams(&output) + ); + let text = String::from_utf8_lossy(&output.stdout); + assert!(text.contains("emit_alloc/churn"), "still reports the finding:\n{text}"); + assert!(text.contains("ANVIL_BENCH_GATE"), "points at the opt-in:\n{text}"); +} + +#[test] +fn bench_history_gate_reads_ci_markers_as_boolean_like() { + if !tools_available() { + return; + } + // PowerShell treats every non-empty string as true, so an exported + // CI=false would otherwise gate a local run on a shared trend measured + // on that developer's own hardware. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new("false")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "CI=false must not gate a local run:{}", + both_streams(&output) + ); + + // A present-but-blank marker is not a marker either. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new(" ")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert!( + output.status.success(), + "a whitespace-only marker must not gate:{}", + both_streams(&output) + ); + + // Anything else non-empty still gates: the CI side stays fail-closed. + let (_tmp, output) = run_bench_history( + ACTIVE_REGRESSION, + &[ + ("ANVIL_BENCH_GATE", OsStr::new("")), + ("CI", OsStr::new("true")), + ("TF_BUILD", OsStr::new("")), + ], + ); + assert_failed(&output, "CI=true"); +} + +#[test] +fn bench_history_propagates_tool_failure() { + if !tools_available() { + return; + } + // A tool that fails to *run* is not "no regressions". + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[("FAKE_CBH_COLLECT_EXIT", OsStr::new("3"))]); + assert_failed(&output, "a failing collect"); +} + +#[test] +fn bench_history_refuses_a_store_the_wiring_does_not_publish() { + if !tools_available() { + return; + } + + // The wiring announces the one path it restores into and publishes from. + // A store pointing anywhere else would never persist, so every run would + // cold-start and analyze to a clean no-op -- reporting green exactly when + // the history needed to report red has been lost. + let (_tmp, output) = run_bench_history( + NO_FINDINGS, + &[ + ("ANVIL_BENCH_WIRED_STORE", OsStr::new("target/anvil/bench-history")), + ("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/somewhere-else")), + ], + ); + assert_failed(&output, "a store the wiring does not publish"); + + // Agreement is not a desync, however it is spelled: the comparison is on + // the resolved path, not the literal string. + let (_tmp, output) = run_bench_history( + NO_FINDINGS, + &[ + ("ANVIL_BENCH_WIRED_STORE", OsStr::new("target/anvil/bench-history")), + ("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/anvil/../anvil/bench-history")), + ], + ); + assert!( + output.status.success(), + "the same path spelled differently is not a desync:{}", + both_streams(&output) + ); + + // Without wiring there is nothing to disagree with: a local run may put + // its store wherever it likes. + let (_tmp, output) = run_bench_history(NO_FINDINGS, &[("ANVIL_BENCH_HISTORY_STORE", OsStr::new("target/somewhere-else"))]); + assert!( + output.status.success(), + "an unwired run may choose its own store:{}", + both_streams(&output) + ); +} + +/// Runs the private blessing reconciliation directly, so the prefix-matching +/// boundary is pinned without going through a whole analysis. +fn run_bless(blessings_file: &str, applied: &str) -> (TempDir, Output) { + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + write(&tmp.path().join(".config/bench-blessings.toml"), blessings_file); + let log = tmp.path().join("cargo.log"); + let output = run_just( + tmp.path(), + &["_anvil-bench-history-bless", "store", ".config/bench-blessings.toml"], + &[("FAKE_CBH_BLESSINGS", OsStr::new(applied)), ("FAKE_CARGO_LOG", log.as_os_str())], + ); + (tmp, output) +} + +#[test] +fn bench_history_bless_reconciles_on_exact_prefix_identity() { + if !tools_available() { + return; + } + let requested = "[[blessing]]\n\ + benchmark = \"emit_alloc\"\n\ + commit = \"8392995a\"\n\ + reason = \"arena allocator tradeoff\"\n"; + + // A stored blessing of the *narrower* `emit_alloc/churn` does not cover + // the requested broader `emit_alloc`. Treating it as already-applied + // would leave the build red while claiming nothing needed doing. + let narrower = r#"{"blessings":[{"commit":"8392995a3b94","benchmark":"emit_alloc/churn"}]}"#; + let (tmp, output) = run_bless(requested, narrower); + assert!( + output.status.success(), + "reconciliation failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + cargo_calls(tmp.path()).contains("bench-history bless"), + "a narrower stored blessing must not satisfy a broader request:{}", + both_streams(&output) + ); + + // The exact same id already recorded is a no-op, so a scheduled run never + // re-appends a sidecar that is already in effect. Window-shaped, because + // `list blessings --all` reports concrete ids and never `prefixes` -- + // fixtures in the HEAD shape would cover a branch that never runs here. + let exact = r#"{"blessings":[{"commit":"8392995a3b94","benchmark":"emit_alloc"}]}"#; + let (tmp, output) = run_bless(requested, exact); + assert!(output.status.success()); + assert!( + !cargo_calls(tmp.path()).contains("bench-history bless"), + "an already-applied blessing must not be re-appended:{}", + both_streams(&output) + ); +} + +#[test] +fn bench_history_bless_skips_commits_the_store_has_forgotten() { + if !tools_available() { + return; + } + let requested = "[[blessing]]\n\ + benchmark = \"emit_alloc\"\n\ + commit = \"8392995a\"\n\ + reason = \"arena allocator tradeoff\"\n"; + + // cbh refuses to bless a context commit it holds no run for, and `collect` + // only ever records the current commit, so a commit lost to a cold start + // or artifact eviction can never come back. Applying it anyway would fail + // the recipe before analyze on every run, leaving the group permanently + // red until a human edited the ledger. + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + write(&tmp.path().join(".config/bench-blessings.toml"), requested); + let log = tmp.path().join("cargo.log"); + let output = run_just( + tmp.path(), + &["_anvil-bench-history-bless", "store", ".config/bench-blessings.toml"], + &[ + ("FAKE_CBH_BLESSINGS", OsStr::new(r#"{"blessings":[]}"#)), + ("FAKE_CBH_RUNS", OsStr::new(r#"{"sets":[]}"#)), + ("FAKE_CARGO_LOG", log.as_os_str()), + ], + ); + assert!( + output.status.success(), + "a forgotten commit must not fail the recipe:{}", + both_streams(&output) + ); + assert!( + !cargo_calls(tmp.path()).contains("bench-history bless"), + "nothing to accept at a commit the store has forgotten:{}", + both_streams(&output) + ); +} + +#[test] +fn bench_history_bless_rejects_malformed_entries() { + if !tools_available() { + return; + } + let applied = r#"{"blessings":[]}"#; + + // A `#` inside a value is content, not a comment: silently truncating a + // reason citing an issue number would lose exactly what makes the audit + // trail worth keeping. + let hashed = "[[blessing]]\n\ + benchmark = \"emit_alloc\"\n\ + commit = \"8392995a\"\n\ + reason = \"accepted in #1234\"\n"; + let (tmp, output) = run_bless(hashed, applied); + assert!( + output.status.success(), + "a # inside a quoted value is content:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + String::from_utf8_lossy(&output.stdout).contains("#1234"), + "the reason must survive intact:{}", + both_streams(&output) + ); + assert!(cargo_calls(tmp.path()).contains("bench-history bless")); + + // Anything the subset cannot represent is rejected, not reinterpreted. + for (body, label) in [ + ("[[blessing]]\nbenchmark = emit_alloc\n", "unquoted value"), + ("[[blessing]]\nbenchmark = \"a\"\ncommit = \"b\"\n", "missing reason"), + ("[[other]]\nbenchmark = \"a\"\n", "unexpected table"), + // A leading `-` would reach cbh as a flag rather than a benchmark id. + // `--all` in particular has a real meaning there: accept *every* + // benchmark at the commit, while the log claims one named `--all`. + ( + "[[blessing]]\nbenchmark = \"--all\"\ncommit = \"8392995a\"\nreason = \"r\"\n", + "an option-shaped benchmark", + ), + ( + "[[blessing]]\nbenchmark = \"emit_alloc\"\ncommit = \"--all\"\nreason = \"r\"\n", + "an option-shaped commit", + ), + ] { + let (_tmp, output) = run_bless(body, applied); + assert_failed(&output, label); + } +} + +// --------------------------------------------------------------------------- +// Benchmark-history restore blocks. +// +// These carry the most control flow in the benchmark wiring, and the invariant +// they exist for -- an operational failure must never be mistaken for "no +// history yet" -- is invisible to a `contains` assertion on the emitted YAML. +// Both are therefore extracted from their template and executed against mocked +// transports. +// --------------------------------------------------------------------------- + +const GH_BENCH_RESTORE: &str = include_str!("../templates/github/run-group-action.yml"); + +const ADO_RESTORE: &str = include_str!("../templates/ado/steps/bench-history-restore.yml"); + +/// Extracts a block scalar (`run: |` / `pwsh: |`) from `yaml`, starting the +/// search at `after` and dedenting the body. +fn block_scalar(yaml: &str, after: &str, key: &str) -> String { + let start = yaml.find(after).unwrap_or_else(|| panic!("marker '{after}' not found")); + let rest = &yaml[start..]; + let key_offset = rest.find(key).unwrap_or_else(|| panic!("key '{key}' not found after '{after}'")); + let key_line_start = rest[..key_offset].rfind('\n').map_or(0, |index| index + 1); + let indent = key_offset - key_line_start; + + let body = &rest[key_offset + key.len()..]; + let mut lines = Vec::new(); + for line in body.lines().skip(1) { + let line_indent = line.len() - line.trim_start().len(); + if !line.trim().is_empty() && line_indent <= indent { + break; + } + lines.push(if line.len() > indent + 2 { &line[indent + 2..] } else { "" }); + } + lines.join("\n") +} + +fn git_bash() -> Option<&'static str> { + let candidate = r"C:\Program Files\Git\bin\bash.exe"; + Path::new(candidate).is_file().then_some(candidate) +} + +/// Runs the GitHub restore block with a stubbed `gh`. +/// +/// `runs` are the run ids the listing yields, newest first; `artifact_runs` +/// are those the artifacts API reports as carrying the artifact. +fn run_github_restore( + runs: &str, + artifact_runs: &str, + download_exit: &str, + runs_exit: &str, + nested: bool, +) -> (TempDir, Output, String, String) { + let bash = git_bash().expect("git bash checked by caller"); + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + + write( + &root.join("bin/gh"), + r#"#!/usr/bin/env bash +if [ "$1" = "run" ] && [ "$2" = "list" ]; then + if [ -n "$FAKE_GH_RUNS_EXIT" ] && [ "$FAKE_GH_RUNS_EXIT" != "0" ]; then + echo "gh: HTTP 502 Bad Gateway (api.github.com)" >&2 + exit "$FAKE_GH_RUNS_EXIT" + fi + for id in $FAKE_GH_RUNS; do echo "$id"; done + exit 0 +fi +if [ "$1" = "api" ]; then + for arg in "$@"; do + case "$arg" in + */actions/runs/*/artifacts) + rid="${arg##*/runs/}" + rid="${rid%%/artifacts}" + for id in $FAKE_GH_ARTIFACT_RUNS; do + if [ "$id" = "$rid" ]; then echo "artifact-$id"; exit 0; fi + done + ;; + esac + done + exit 0 +fi +if [ "$1" = "run" ] && [ "$2" = "download" ]; then + if [ "${FAKE_GH_DOWNLOAD_EXIT:-0}" != "0" ]; then exit "$FAKE_GH_DOWNLOAD_EXIT"; fi + # Mimic the payload layout: flat by default (what a single --name gives), + # nested under the artifact name when the scenario asks for it. + dest="" + prev="" + for arg in "$@"; do + if [ "$prev" = "--dir" ]; then dest="$arg"; fi + prev="$arg" + done + if [ "$FAKE_GH_NESTED" = "1" ]; then dest="$dest/$ARTIFACT"; fi + mkdir -p "$dest" + echo "{}" > "$dest/run.json" + exit 0 +fi +exit 0 +"#, + ); + + let script = block_scalar(GH_BENCH_RESTORE, "- name: Restore benchmark history", "run: |"); + write(&root.join("restore.sh"), &script); + + // `gh` is bound as a shell function rather than found on PATH. Git for + // Windows' bash prepends its own entries to whatever PATH it is handed, + // so a stub placed on PATH can be shadowed by a real `gh` earlier in the + // resolution order -- which silently exercises the developer's GitHub CLI + // instead of the fixture. Function lookup precedes PATH, so this cannot + // be shadowed. + write(&root.join("run.sh"), "gh() { bash \"$STUB_GH\" \"$@\"; }\n. ./restore.sh\n"); + + let github_env = root.join("env.txt"); + let summary = root.join("summary.md"); + write(&github_env, ""); + write(&summary, ""); + + let output = Command::new(bash) + .arg("run.sh") + .current_dir(root) + .env("STUB_GH", root.join("bin/gh").display().to_string().replace('\\', "/")) + .env("FAKE_GH_RUNS", runs) + .env("FAKE_GH_ARTIFACT_RUNS", artifact_runs) + .env("FAKE_GH_DOWNLOAD_EXIT", download_exit) + .env("FAKE_GH_RUNS_EXIT", runs_exit) + .env("FAKE_GH_NESTED", if nested { "1" } else { "0" }) + .env("ARTIFACT", "bench-history-linux") + .env("DEFAULT_BRANCH", "main") + .env("WORKFLOW", "anvil-scheduled") + .env("REPO", "owner/repo") + .env("WINDOW", "30") + .env("GITHUB_ENV", &github_env) + .env("GITHUB_STEP_SUMMARY", &summary) + .output() + .expect("bash runs the extracted restore block"); + + let env_text = std::fs::read_to_string(&github_env).unwrap_or_default(); + let summary_text = std::fs::read_to_string(&summary).unwrap_or_default(); + (tmp, output, env_text, summary_text) +} + +#[test] +fn github_restore_separates_absence_from_failure() { + if git_bash().is_none() { + eprintln!("skipping: git bash not installed"); + return; + } + + // (1) The newest run has no artifact; an older one does. The walk must + // reach it rather than cold-starting on the first miss. + let (tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "0", false); + assert!( + output.status.success(), + "walking back should succeed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(env.contains("ANVIL_BENCH_RESTORE=restored"), "env:\n{env}"); + assert!( + String::from_utf8_lossy(&output.stdout).contains("run 10"), + "should name the run it restored from:{}", + both_streams(&output) + ); + // The payload has to land *in* the store, not one level under it. A + // nested copy loses the history silently: every run then cold-starts + // and analyzes to a clean no-op. + let store = tmp.path().join("target/anvil/bench-history/run.json"); + assert!(store.is_file(), "the restored payload must land in the store root"); + + // `gh run download` extracts a single `--name` straight into `--dir`, + // but nests one directory per artifact when several are requested. + // Tolerate both, so a change in that behaviour cannot silently empty + // the store. + let (tmp_nested, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "0", true); + assert!(output.status.success(), "nested layout:{}", both_streams(&output)); + assert!(env.contains("ANVIL_BENCH_RESTORE=restored"), "env:\n{env}"); + assert!( + tmp_nested.path().join("target/anvil/bench-history/run.json").is_file(), + "a nested artifact directory must be lifted into the store root" + ); + + // (2) No run in the window carries it: a genuine cold start, and it must + // be visible on the summary rather than only in the log. + let (_tmp, output, env, summary) = run_github_restore("30 20 10", "", "0", "0", false); + assert!(output.status.success()); + assert!(env.contains("ANVIL_BENCH_RESTORE=cold-start"), "env:\n{env}"); + assert!(summary.contains("cold start"), "summary:\n{summary}"); + + // (3) The artifact exists but the download fails. This is the branch whose + // silent reintroduction re-creates the history-loss bug: it must fail and + // leave no publishable restore state. + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "30", "1", "0", false); + assert_failed(&output, "a failing download"); + assert!( + !env.contains("ANVIL_BENCH_RESTORE"), + "a failed restore must not mark a publishable state:\n{env}" + ); + + // (4) Listing the runs fails outright. `set -e` does not apply to a + // command substitution consumed as a `for` word list, so this branch + // would otherwise fall through to a cold start and publish a truncated + // store over the chain while reporting green. + let (_tmp, output, env, _summary) = run_github_restore("30 20 10", "10", "0", "1", false); + assert_failed(&output, "a failing run listing"); + assert!(!env.contains("ANVIL_BENCH_RESTORE"), "a failed listing is not a cold start:\n{env}"); +} + +/// Runs the ADO restore block with mocked REST/download/extract cmdlets. +/// +/// `artifact_builds` are the build ids whose artifact query succeeds; every +/// other build answers 404 (absence). `failure` injects an operational fault: +/// "query" (non-404 status), "download", or "extract". +fn run_ado_restore(builds: &str, artifact_builds: &str, failure: &str, no_default_branch: bool) -> (TempDir, Output, String) { + let tmp = TempDir::new().unwrap(); + let root = tmp.path(); + + let body = block_scalar(ADO_RESTORE, "steps:", "pwsh: |") + .replace("${{ parameters.artifact }}", "bench-history-linux") + .replace("${{ parameters.path }}", "store") + .replace("${{ parameters.window }}", "30"); + + // Function definitions shadow cmdlets of the same name, so the block runs + // unchanged against these stand-ins. + let prelude = r#" +# An exception whose Response.StatusCode.value__ the block can read, which is +# how it tells absence (404) from an operational failure (anything else). +class FakeHttpException : System.Exception { + [object]$Response + FakeHttpException([int]$status) : base("http $status") { + $this.Response = [pscustomobject]@{ StatusCode = [pscustomobject]@{ value__ = $status } } + } +} +function Invoke-RestMethod { + param([string]$Uri, $Headers) + if ($Uri -like '*/definitions/*') { + # The pipeline definition carries the default branch the series is + # keyed on. FAKE_ADO_NO_DEFAULT_BRANCH withholds it. + if ($env:FAKE_ADO_NO_DEFAULT_BRANCH) { + return [pscustomobject]@{ repository = [pscustomobject]@{ defaultBranch = $null } } + } + return [pscustomobject]@{ repository = [pscustomobject]@{ defaultBranch = 'refs/heads/main' } } + } + if ($Uri -notlike '*/artifacts*') { + $ids = $env:FAKE_ADO_BUILDS -split ' ' | Where-Object { $_ } + return [pscustomobject]@{ value = @($ids | ForEach-Object { [pscustomobject]@{ id = $_ } }) } + } + $buildId = ($Uri -replace '.*/builds/', '') -replace '/artifacts.*', '' + $carries = ($env:FAKE_ADO_ARTIFACT_BUILDS -split ' ') -contains $buildId + if (-not $carries) { throw [FakeHttpException]::new(404) } + if ($env:FAKE_ADO_FAILURE -eq 'query') { throw [FakeHttpException]::new(500) } + return [pscustomobject]@{ resource = [pscustomobject]@{ downloadUrl = "https://example/$buildId" } } +} +function Invoke-WebRequest { + param([string]$Uri, $Headers, [string]$OutFile) + if ($env:FAKE_ADO_FAILURE -eq 'download') { throw 'download failed' } + Set-Content -LiteralPath $OutFile -Value 'zip' +} +function Expand-Archive { + param([string]$LiteralPath, [string]$DestinationPath, [switch]$Force) + if ($env:FAKE_ADO_FAILURE -eq 'extract') { throw 'corrupt archive' } + New-Item -ItemType Directory -Force -Path $DestinationPath | Out-Null + Set-Content -LiteralPath (Join-Path $DestinationPath 'run.json') -Value '{}' +} +"#; + + write(&root.join("restore.ps1"), &format!("{prelude}\n{body}")); + + let output = Command::new("pwsh") + .args(["-NoProfile", "-File", "restore.ps1"]) + .current_dir(root) + .env("FAKE_ADO_BUILDS", builds) + .env("FAKE_ADO_ARTIFACT_BUILDS", artifact_builds) + .env("FAKE_ADO_FAILURE", failure) + .env("FAKE_ADO_NO_DEFAULT_BRANCH", if no_default_branch { "1" } else { "" }) + .env("SYSTEM_ACCESSTOKEN", "token") + .env("SYSTEM_COLLECTIONURI", "https://example/") + .env("SYSTEM_TEAMPROJECTID", "project") + .env("SYSTEM_DEFINITIONID", "7") + .env("BUILD_SOURCEBRANCH", "refs/heads/main") + .env("BUILD_BUILDID", "999") + .output() + .expect("pwsh runs the extracted restore block"); + + let stdout = String::from_utf8_lossy(&output.stdout).to_string(); + (tmp, output, stdout) +} + +#[test] +fn ado_restore_separates_absence_from_failure() { + if Command::new("pwsh").arg("--version").output().is_err() { + eprintln!("skipping: pwsh not installed"); + return; + } + + // A 404 on the newest build walks back to an older one that has it. + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "10", "", false); + assert!( + output.status.success(), + "walking back should succeed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(stdout.contains("restoring"), "stdout:\n{stdout}"); + assert!(stdout.contains("ANVIL_BENCH_RESTORE]restored"), "stdout:\n{stdout}"); + + // Nothing in the window carries it: a genuine cold start. + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "", "", false); + assert!(output.status.success()); + assert!(stdout.contains("ANVIL_BENCH_RESTORE]cold-start"), "stdout:\n{stdout}"); + + // Every operational fault must fail without marking a publishable state, + // which is what the guarded publish depends on to avoid overwriting a + // good chain with a truncated store. + for failure in ["query", "download", "extract"] { + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "30", failure, false); + assert_failed(&output, failure); + assert!( + !stdout.contains("ANVIL_BENCH_RESTORE]"), + "{failure} must not set a restore state:\n{stdout}" + ); + } + + // The series is keyed on the default branch. If that cannot be resolved + // the leg does not know which chain it is extending, and silently keying + // on the branch this build ran on would fork a one-sample history and + // report it as clean. + let (_tmp, output, stdout) = run_ado_restore("30 20 10", "10", "", true); + assert_failed(&output, "an unresolvable default branch"); + assert!( + !stdout.contains("ANVIL_BENCH_RESTORE]"), + "an unidentified series must not set a restore state:\n{stdout}" + ); +} + +#[test] +fn bench_history_bless_listings_are_process_scoped() { + if !tools_available() { + return; + } + // The blessing reconciliation writes `list blessings` output to a temp + // file before reading it back. A fixed name lets two jobs sharing a + // machine -- matrix legs on a self-hosted agent, concurrent local runs -- + // read each other's listing. Reverting the process scoping must trip + // this deterministically rather than by chance under parallel runs. + let shared_temp = TempDir::new().unwrap(); + + // Each invocation asks for a different benchmark and is told a different + // already-applied set, so consuming the other's listing is observable: + // each would then think its blessing was already in effect and skip it. + let cases = [ + ("alpha", r#"{"blessings":[{"commit":"8392995a3b94","prefixes":["beta"]}]}"#), + ("beta", r#"{"blessings":[{"commit":"8392995a3b94","prefixes":["alpha"]}]}"#), + ]; + + for (benchmark, applied) in cases { + let file = format!("[[blessing]]\nbenchmark = \"{benchmark}\"\ncommit = \"8392995a\"\nreason = \"deliberate\"\n"); + let tmp = fixture( + &[("bench-history.just", BENCH_HISTORY)], + &[ + "anvil-tool-cargo-bench-history-validate-prereqs", + "anvil-tool-cargo-bench-history-install installer=\"install\"", + ], + ); + write(&tmp.path().join(".config/bench-blessings.toml"), &file); + let log = tmp.path().join("cargo.log"); + let output = run_just( + tmp.path(), + &["_anvil-bench-history-bless", "store", ".config/bench-blessings.toml"], + &[ + ("FAKE_CBH_BLESSINGS", OsStr::new(applied)), + ("FAKE_CARGO_LOG", log.as_os_str()), + // Both invocations share one temp directory, which is what a + // fixed listing filename would collide in. + ("RUNNER_TEMP", shared_temp.path().as_os_str()), + ], + ); + assert!( + output.status.success(), + "reconciliation failed for {benchmark}:{}", + both_streams(&output) + ); + // Its own listing says a *different* benchmark is blessed, so this + // one must still be applied. + assert!( + cargo_calls(tmp.path()).contains("bench-history bless"), + "{benchmark} must be blessed from its own listing:{}", + both_streams(&output) + ); + } + + // One listing file per process, so the two runs never shared one. + let listings: Vec<_> = std::fs::read_dir(shared_temp.path()) + .unwrap() + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.starts_with("anvil-bench-blessings")) + .collect(); + assert_eq!(listings.len(), 2, "each invocation must write its own listing, got: {listings:?}"); +} diff --git a/crates/cargo-anvil/tests/schemas.rs b/crates/cargo-anvil/tests/schemas.rs index 83580c5a3..175f55adf 100644 --- a/crates/cargo-anvil/tests/schemas.rs +++ b/crates/cargo-anvil/tests/schemas.rs @@ -97,6 +97,18 @@ fn taplo_validates_emitted_toml_files() { #[test] fn actionlint_validates_emitted_workflows() { let tmp = run_with_backend("github"); + // actionlint refuses to run outside a git repository ("no project was + // found in any parent directories"), so the generated tree has to look + // like one before the workflows can be validated. + let Some(init) = try_run(Command::new("git").args(["init", "--quiet"]).current_dir(tmp.path())) else { + eprintln!("skipping: git not installed"); + return; + }; + assert!( + init.status.success(), + "git init failed in the fixture:\n{}", + String::from_utf8_lossy(&init.stderr) + ); let mut cmd = Command::new("actionlint"); cmd.current_dir(tmp.path()); let Some(out) = try_run(&mut cmd) else { @@ -141,6 +153,7 @@ fn just_lists_emitted_recipes() { ("anvil-scheduled", "# Run all scheduled checks."), ("anvil-scheduled-advisories", "# Run the scheduled advisory checks."), ("anvil-scheduled-exhaustive", "# Run the scheduled exhaustive checks."), + ("anvil-scheduled-benchmarks", "# Run the scheduled benchmark regression detection."), ("anvil-scheduled-runtime-analysis", "# Run the scheduled runtime analysis."), ("anvil-scheduled-test", "# Run the scheduled tests."), ] { diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap index 597aa901c..3e1c1313b 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__ado_backend.snap @@ -503,37 +503,32 @@ parameters: - name: windowsPool type: object default: { vmImage: windows-latest } + - name: benchMachineKey + type: string + # Machine key the benchmark history is partitioned by. Empty uses + # cargo-bench-history's hardware fingerprint; set a stable pool label + # when the agent pool is heterogeneous enough to fragment a series into + # partitions too sparse to analyze. + default: '' stages: - stage: scheduled_test displayName: anvil scheduled-test jobs: - # Publish coverage from both legs so OS-gated code is fully - # represented (see the pr-stages.yml comment for the rationale). + # Coverage is published from both legs so OS-gated code is fully + # represented; that step lives in steps/scheduled-test.yml. - template: steps/job.yml parameters: name: linux pool: ${{ parameters.linuxPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (linux) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - template: steps/job.yml parameters: name: windows pool: ${{ parameters.windowsPool }} steps: - template: steps/scheduled-test.yml - - task: PublishCodeCoverageResults@2 - condition: succeededOrFailed() - displayName: Publish coverage (windows) - inputs: - summaryFileLocation: target/coverage/cobertura-*.xml - failIfCoverageEmpty: false - stage: scheduled_advisories displayName: anvil scheduled-advisories @@ -597,6 +592,45 @@ stages: steps: - template: steps/scheduled-exhaustive.yml + - stage: scheduled_benchmarks + displayName: anvil scheduled-benchmarks + dependsOn: [] + variables: + # Read by the bench-history recipe; empty means "use the hardware + # fingerprint". ADO exports pipeline variables as environment + # variables, so this reaches the recipe without further plumbing. + ANVIL_BENCH_MACHINE_KEY: ${{ parameters.benchMachineKey }} + jobs: + # The history round-trip lives in steps/scheduled-benchmarks.yml. Only + # the artifact declaration is here, because publishing is a job-level + # output: a forked (1ESPT) wrapper translates this list into its own + # output shape, which a task inside the step list would bypass. + # + # The condition is the guard that keeps an operational restore failure + # from overwriting a good chain with a truncated store; failed runs + # still publish, since a flagged regression fails the job and those + # samples belong in the history. + - template: steps/job.yml + parameters: + name: linux + pool: ${{ parameters.linuxPool }} + artifacts: + - name: bench-history-$(Agent.JobName) + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) + steps: + - template: steps/scheduled-benchmarks.yml + - template: steps/job.yml + parameters: + name: windows + pool: ${{ parameters.windowsPool }} + artifacts: + - name: bench-history-$(Agent.JobName) + path: target/anvil/bench-history + condition: and(succeededOrFailed(), ne(variables['ANVIL_BENCH_RESTORE'], '')) + steps: + - template: steps/scheduled-benchmarks.yml + === .pipelines/anvil/steps/advisory-comments.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -695,6 +729,184 @@ steps: } } +=== .pipelines/anvil/steps/bench-history-restore.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Restores the benchmark history published by the most recent build that +# carries this leg's artifact. +# +# `DownloadPipelineArtifact@2` with `latestFromBranch` resolves a single +# build and yields nothing if that build has no such artifact -- it does +# not walk back. A cancelled or never-publishing latest build would +# therefore cold-start a store that actually has history. This step +# resolves the build itself, so absence of the artifact throughout the +# window (a genuine cold start) stays distinct from an operational +# failure. +# +# The store path is created only once the restore has reached a known +# state -- restored, or a positively identified cold start. An operational +# failure therefore leaves *no store directory at all*, so a publisher +# that runs unconditionally has nothing to upload and the accumulated +# chain survives. That makes the guard structural rather than dependent on +# every job wrapper honouring the `artifacts` condition: a fork predating +# that field ignores it silently, and this is the case where ignoring it +# would overwrite good history with a truncated store. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: artifact + type: string + - name: path + type: string + default: target/anvil/bench-history + - name: window + type: number + default: 30 +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $artifact = '${{ parameters.artifact }}' + $path = '${{ parameters.path }}' + $window = ${{ parameters.window }} + + if (-not $env:SYSTEM_ACCESSTOKEN) { + Write-Error "anvil: SYSTEM_ACCESSTOKEN is not exposed to this job, so the benchmark history cannot be restored." + exit 1 + } + + # Everything lands here first; $path is created only on success. + # $PID-scoped: on a self-hosted agent two concurrent jobs share the + # temp directory, and an artifact name is not unique across + # overlapping pipeline runs. + $staging = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-staging-$PID" + Remove-Item -Recurse -Force $staging -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $staging | Out-Null + + # Publishes the staged store as the restore's result and records how + # the restore ended, so the publish step can tell a continuation of + # the chain from a job that never established one. + function Complete-Restore([string]$state) { + # Assembled beside the published path and moved into place in one + # step. A copy that failed partway would otherwise leave a partial + # store exactly where a wrapper forked before `artifacts[].condition` + # existed -- which ignores that field silently -- would publish it + # over the accumulated chain. + $parent = Split-Path -Parent $path + if ($parent) { New-Item -ItemType Directory -Force -Path $parent | Out-Null } + $pending = "$path.pending-$PID" + Remove-Item -Recurse -Force $pending -ErrorAction SilentlyContinue + New-Item -ItemType Directory -Force -Path $pending | Out-Null + if (Test-Path (Join-Path $staging '*')) { + Copy-Item -Path (Join-Path $staging '*') -Destination $pending -Recurse -Force + } + Remove-Item -Recurse -Force $path -ErrorAction SilentlyContinue + Move-Item -LiteralPath $pending -Destination $path + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_RESTORE]$state" + # The recipe refuses to write anywhere but here, so a wrapper that + # retargets `path` cannot silently detach the recipe's store from + # the one that is actually published. + Write-Host "##vso[task.setvariable variable=ANVIL_BENCH_WIRED_STORE]$path" + } + + $collection = $env:SYSTEM_COLLECTIONURI + $project = $env:SYSTEM_TEAMPROJECTID + $definition = $env:SYSTEM_DEFINITIONID + $headers = @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + + # The series is keyed on the repository's default branch, matching the + # GitHub backend and the documented contract. Keying on the branch the + # build happened to run on would fork a fresh, one-sample history for + # every manual or release-branch run and report that as clean. + $definitionUri = "$collection$project/_apis/build/definitions/$definition" + "?api-version=7.0" + $branch = (Invoke-RestMethod -Uri $definitionUri -Headers $headers).repository.defaultBranch + if (-not $branch) { + Write-Error "anvil: could not resolve the pipeline's default branch, so the history series cannot be identified; refusing to continue rather than fork a new chain under the current branch." + exit 1 + } + # Newest first, whatever the outcome: a flagged regression fails the + # build, so restricting to successful builds would discard exactly + # the stretch of history that matters most. + $runsUri = "$collection$project/_apis/build/builds?definitions=$definition&branchName=$branch&`$top=$window&queryOrder=finishTimeDescending&api-version=7.0" + $runs = (Invoke-RestMethod -Uri $runsUri -Headers $headers).value + + foreach ($run in $runs) { + if ($run.id -eq $env:BUILD_BUILDID) { continue } + + $artifactUri = "$collection$project/_apis/build/builds/$($run.id)/artifacts?artifactName=$artifact&api-version=7.0" + try { + $found = Invoke-RestMethod -Uri $artifactUri -Headers $headers + } catch { + # 404 is "this build has no such artifact" -- the expected, + # common case while walking back. Anything else is an + # operational failure and must not be read as absence. + $status = $_.Exception.Response.StatusCode.value__ + if ($status -eq 404) { continue } + Write-Error "anvil: querying artifacts of build $($run.id) failed with HTTP $status; refusing to continue with an empty history." + exit 1 + } + if (-not $found.resource.downloadUrl) { continue } + + Write-Host "anvil: restoring $artifact from build $($run.id)" + $zip = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-$PID.zip" + $extract = Join-Path ([System.IO.Path]::GetTempPath()) "$artifact-extract-$PID" + Remove-Item -Recurse -Force $extract -ErrorAction SilentlyContinue + Invoke-WebRequest -Uri $found.resource.downloadUrl -Headers $headers -OutFile $zip + Expand-Archive -LiteralPath $zip -DestinationPath $extract -Force + # The archive nests its contents under a directory named for the + # artifact; lift them up into the staging directory. An artifact + # that is present but empty is a valid restore -- a workspace with + # no benchmarks yet publishes exactly that -- and `Copy-Item` over + # an empty wildcard throws, so the copy is guarded. + $inner = Join-Path $extract $artifact + $source = if (Test-Path $inner) { $inner } else { $extract } + if (Test-Path (Join-Path $source '*')) { + Copy-Item -Path (Join-Path $source '*') -Destination $staging -Recurse -Force + } + Complete-Restore 'restored' + exit 0 + } + + # Nothing in the window carried the artifact: a genuine cold start, + # which is a valid empty store rather than a failure. + Write-Host "anvil: no $artifact artifact in the last $window builds on $branch; starting a new history" + Complete-Restore 'cold-start' + displayName: Restore ${{ parameters.artifact }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +=== .pipelines/anvil/steps/bench-history-summary.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# +# Attaches the benchmark findings to the build summary, so a failed +# scheduled build carries the per-finding detail its one-bit status +# cannot. Runs whether or not the analysis flagged anything. +# +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md +parameters: + - name: findings + type: string + default: target/anvil/bench/findings.md +steps: + - pwsh: | + $ErrorActionPreference = 'Stop' + $findings = '${{ parameters.findings }}' + if (-not (Test-Path -LiteralPath $findings)) { + Write-Host "anvil: no $findings to attach" + exit 0 + } + $full = (Resolve-Path -LiteralPath $findings).Path + Write-Host "##vso[task.uploadsummary]$full" + displayName: Publish benchmark findings + condition: succeededOrFailed() + === .pipelines/anvil/steps/impact.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -746,6 +958,10 @@ steps: # - name (string) Job name; ADO derives the display name from it. # - pool (object) Pool block, passed verbatim to ADO's `pool:` key. # - steps (stepList) Body of the job. Templated step lists are fine. +# A step list may lead with its own `checkout: self` +# (the benchmark group does, for depth and LFS), so a +# wrapper must NOT add a checkout of its own -- doing +# so would give those jobs two. # - inputArtifacts (object) Optional list of pipeline artifacts to download # BEFORE the steps run. Each item: { name, path }. # The default wrapper prepends one @@ -756,14 +972,18 @@ steps: # `anvil-impact-` artifact into target/anvil/impact # and its checks read the cache exactly as a local run. # - artifacts (object) Optional list of pipeline artifacts to publish. -# Each item: { name: string, path: string }. -# The default wrapper appends one -# PublishPipelineArtifact@1 task per entry; 1ESPT -# wrappers translate these into +# Each item: { name: string, path: string, +# condition: string (optional) }. The default wrapper +# appends one PublishPipelineArtifact@1 task per +# entry; 1ESPT wrappers translate these into # templateContext.outputs.pipelineArtifact blocks # at job level. The contract is the same either way, # so the stages templates don't need to know which # backend they're targeting. +# `condition` defaults to succeededOrFailed(); a +# caller that must not publish in some states sets +# it, and a fork is expected to carry it through to +# whatever output shape it emits. parameters: - name: name type: string @@ -799,7 +1019,10 @@ jobs: - ${{ each artifact in parameters.artifacts }}: - task: PublishPipelineArtifact@1 displayName: Publish ${{ artifact.name }} - condition: succeededOrFailed() + ${{ if artifact.condition }}: + condition: ${{ artifact.condition }} + ${{ else }}: + condition: succeededOrFailed() inputs: targetPath: ${{ artifact.path }} artifact: ${{ artifact.name }} @@ -819,6 +1042,10 @@ jobs: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -884,6 +1111,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -949,6 +1180,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1014,6 +1249,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1079,6 +1318,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1129,6 +1372,102 @@ steps: # workflow-level CARGO_INCREMENTAL=0. CARGO_INCREMENTAL: "0" +=== .pipelines/anvil/steps/scheduled-benchmarks.yml === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md +# The token scheduled-benchmarks is substituted by cargo-anvil at emit time with +# the concrete check-group name (pr-fast, pr-test, scheduled-runtime-analysis, ...). +# +# The impact set reaches this group as the downloaded target/anvil/impact/ +# cache. A PR group job downloads the `anvil-impact-` artifact into +# target/anvil/impact/ (via job.yml's inputArtifacts) and the scoped checks +# read that cache via their `anvil-impact` dependency -- the same code path as +# a local run. Scheduled group jobs download nothing and run full-workspace +# (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. +steps: +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, ahead of the uniform runner. Not a standalone template: this is +# part of a `steps:` list, which is why it has no `steps:` key of its own. +# +# The analysis orders each series by first-parent commit topology and +# locates the merge-base, so it needs the whole commit graph. LFS matters +# because benchmark inputs can be LFS-tracked and would otherwise arrive as +# pointer files. +# +# The checkout is explicit rather than a job-wrapper parameter: job.yml is +# the file adopters fork, so binding a parameter their copy lacks would fail +# expansion for the whole pipeline. +# +# The artifact is named for the *job*, which ADO guarantees is unique within +# a stage. Naming it for the agent OS instead would let two legs on like-OS +# pools declare one artifact, colliding on publish and merging two machines' +# samples into a single series. + - checkout: self + fetchDepth: 0 + lfs: true + - template: bench-history-restore.yml + parameters: + artifact: bench-history-$(Agent.JobName) + - template: setup.yml + parameters: + group: scheduled-benchmarks + # ADO has no PR-title predefined variable: `System.PullRequest.Title` + # does NOT exist, so `$(System.PullRequest.Title)` would expand to the + # literal unexpanded macro text (non-empty) and make anvil-pr-title + # validate that string. Resolve the real title from the REST API on PR + # builds and publish it as the PR_TITLE pipeline variable. Only + # anvil-pr-title (in pr-fast) consults it; other groups ignore it, but we + # resolve uniformly to keep group.yml the same across groups. Non-PR builds + # skip; a known PR whose metadata cannot be retrieved fails closed. + - pwsh: | + $ErrorActionPreference = 'Stop' + $prId = $env:SYSTEM_PULLREQUEST_PULLREQUESTID + if (-not $prId) { + Write-Host 'anvil: not a PR build; leaving PR_TITLE empty' + Write-Host '##vso[task.setvariable variable=PR_TITLE]' + exit 0 + } + $uri = "$($env:SYSTEM_COLLECTIONURI)$($env:SYSTEM_TEAMPROJECTID)/_apis/git/repositories/$($env:BUILD_REPOSITORY_ID)/pullRequests/${prId}?api-version=7.0" + try { + $resp = Invoke-RestMethod -Uri $uri -Headers @{ Authorization = "Bearer $($env:SYSTEM_ACCESSTOKEN)" } + Write-Host "##vso[task.setvariable variable=PR_TITLE]$($resp.title)" + } catch { + Write-Error "anvil: could not resolve PR title for PR $prId ($_). Ensure the job can access the repository OAuth token." + exit 1 + } + displayName: anvil-scheduled-benchmarks (resolve PR title) + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + - bash: | + set -euo pipefail + # The impact mode is fixed by group class at emit time -- never probed from a + # marker file. PR group jobs downloaded the target/anvil/impact artifact + # and trust it verbatim (consume: anvil-impact no-ops -- no snapshot, + # cargo-delta, or base ref); scheduled group jobs force off so every + # tier runs full-workspace. + export ANVIL_IMPACT=off + just anvil-scheduled-benchmarks + displayName: anvil-scheduled-benchmarks + env: + # Resolved by the preceding step; empty only outside PR builds. + PR_TITLE: $(PR_TITLE) + # Disable cargo incremental compilation in CI: the incremental dir is + # not part of the anvil-setup cache (see setup.yml), so generating it is + # pure cost with no cross-run benefit. Mirrors the GitHub impl workflows' + # workflow-level CARGO_INCREMENTAL=0. + CARGO_INCREMENTAL: "0" +# Fragment spliced into the scheduled-benchmarks group's emitted step +# template, after the uniform runner. Not a standalone template: this is +# part of a `steps:` list. + - template: bench-history-summary.yml + === .pipelines/anvil/steps/scheduled-exhaustive.yml === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -1144,6 +1483,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1209,6 +1552,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1274,6 +1621,10 @@ steps: # read that cache via their `anvil-impact` dependency -- the same code path as # a local run. Scheduled group jobs download nothing and run full-workspace # (see the bash step). +# +# Steps that a particular group needs around the uniform runner are spliced +# in at the same time, so the stages templates stay a plain list of groups +# with no per-group customization at the call site. steps: - template: setup.yml parameters: @@ -1323,6 +1674,18 @@ steps: # pure cost with no cross-run benefit. Mirrors the GitHub impl workflows' # workflow-level CARGO_INCREMENTAL=0. CARGO_INCREMENTAL: "0" +# Fragment spliced into the scheduled-test group's emitted step template, +# after the uniform runner. Not a standalone template: this is part of a +# `steps:` list. +# +# Coverage is published from every leg so OS-gated code is fully +# represented. + - task: PublishCodeCoverageResults@2 + condition: succeededOrFailed() + displayName: Publish coverage + inputs: + summaryFileLocation: target/coverage/cobertura-*.xml + failIfCoverageEmpty: false === .pipelines/anvil/steps/setup.yml === # Copyright (c) Microsoft Corporation. @@ -1702,6 +2065,328 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh", "-NoProfile")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on + # every platform, so on Linux two genuinely different directories would + # compare equal and the detached-store failure this guard exists to + # catch would slip through. + $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } + if (-not $same) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if ([string]::IsNullOrWhiteSpace($value)) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh", "-NoProfile")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + # `benchmark` and `commit` are passed to cbh and git. A leading `-` + # would be read as an option there, so reject it here rather than let + # the file's documented subset be reinterpreted: `benchmark = "--all"` + # would otherwise accept every benchmark at the commit while the log + # claimed a benchmark of that name. + foreach ($guarded in @('benchmark', 'commit')) { + if ($e[$guarded].StartsWith('-')) { + Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + # A blessing can only be applied at a commit the store still holds a clean + # run for: cbh rejects a context commit with no data point, and `collect` + # only ever records the current commit, so a commit the store has forgotten + # can never be re-established by a later run. Without this, any cold start + # or artifact eviction past a blessed commit would turn the group + # permanently red until a human edited the ledger -- a blessing whose + # commit has aged out is inapplicable, not invalid. + $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" + & cargo bench-history list runs --local="$store" ` + --since 1970-01-01 --no-text --json $runsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $stored = [System.Collections.Generic.List[string]]::new() + foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { + foreach ($c in $set.commits) { + if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } + } + } + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Compare the persisted identity exactly. `list blessings --all` is the + # window view: it reports resolved, concrete benchmark ids and never + # populates `prefixes`. Since cbh's positional is a prefix, a ledger + # entry naming a family expands to several concrete rows and matches + # none of them, so such an entry is re-applied each run. That is the + # safe direction to be wrong in: re-blessing is idempotent in effect, + # whereas a loose prefix test would skip an entry that is only + # partially applied and leave the build red while claiming otherwise. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { + Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + # `--` so a benchmark id is never parsed as a flag: bless's positional + # takes no hyphen values, and it has an `--all` sibling that would + # otherwise accept every benchmark at the commit. + & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4946,6 +5631,42 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -5887,6 +6608,7 @@ import 'impact.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -5928,6 +6650,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'tiers.just' import 'tools.just' import 'versions.just' @@ -5977,7 +6700,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full-workspace for the same reason as the scheduled tier. @@ -6020,7 +6744,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -6028,7 +6753,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -6617,6 +7343,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -6861,6 +7595,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.1.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.10" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap index dd5965ea8..be744880f 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__github_backend.snap @@ -434,6 +434,29 @@ inputs: Clean runs only supersede prior failures. default: "false" required: false + bench_history: + description: >- + Round-trip a cargo-bench-history store around the group run, so the + regression analysis has cross-run history to compare against. The + caller must grant actions: read and check out full history + (fetch-depth: 0), neither of which an action can request for itself. + default: "false" + required: false + bench_artifact: + description: >- + Artifact name carrying this leg's history. Must be unique per matrix + leg: the history is partitioned per machine, and merging two runners' + samples into one series destroys the comparison. Supplied by the + caller, which is the only place the matrix value is in scope. + default: "" + required: false + bench_machine_key: + description: >- + Overrides cargo-bench-history's hardware fingerprint with a stable + pool label, for runner pools heterogeneous enough to fragment a series + into partitions too sparse to analyze. + default: "" + required: false runs: using: composite steps: @@ -443,6 +466,110 @@ runs: group: ${{ inputs.group }} free-disk-space: ${{ inputs.free-disk-space }} + - name: Restore benchmark history + if: inputs.bench_history == 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + ARTIFACT: ${{ inputs.bench_artifact }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + # The caller's workflow *name*, not a hardcoded filename: the root + # scheduled workflow is an owned, renameable file, and a rename must + # not silently reset the series. + WORKFLOW: ${{ github.workflow }} + REPO: ${{ github.repository }} + WINDOW: "30" + run: | + set -euo pipefail + + if [ -z "$ARTIFACT" ]; then + echo "::error::bench_history is enabled but bench_artifact is empty;" \ + "refusing to continue, since an unnamed store cannot be restored" \ + "or published and every run would report a false clean." + exit 1 + fi + + # Staged first. The store path is created only once the restore has + # reached a known state, so an operational failure leaves no store + # at all and a publisher that runs unconditionally has nothing to + # upload over the accumulated chain. + staging="$(mktemp -d)" + + complete_restore() { + mkdir -p target/anvil/bench-history + # `gh run download` with a single `--name` extracts into --dir + # directly, but with several it nests under one directory per + # artifact. Lift a nested layout if we ever see one: getting this + # wrong loses the history silently and every run reports a clean + # cold start, which is the one failure this feature must not have. + src="$staging" + if [ -d "$staging/$ARTIFACT" ]; then + src="$staging/$ARTIFACT" + fi + if [ -n "$(ls -A "$src" 2>/dev/null)" ]; then + cp -R "$src/." target/anvil/bench-history/ + fi + echo "ANVIL_BENCH_RESTORE=$1" >> "$GITHUB_ENV" + # The recipe refuses to write anywhere but here, so an override + # cannot silently detach it from the store that is published. + echo "ANVIL_BENCH_WIRED_STORE=target/anvil/bench-history" >> "$GITHUB_ENV" + } + + # Assigned rather than consumed directly by `for`: `set -e` ignores + # the exit status of a command substitution used as a word list, so + # a failed listing would yield an empty list and fall through to a + # cold start -- publishing a truncated store over the chain and + # reporting green for want of the history needed to report red. + if ! run_ids="$(gh run list --workflow "$WORKFLOW" \ + --branch "$DEFAULT_BRANCH" --limit "$WINDOW" \ + --json databaseId --jq '.[].databaseId')"; then + echo "::error::could not list $WORKFLOW runs on $DEFAULT_BRANCH;" \ + "refusing to continue, since treating this as a cold start" \ + "would publish a truncated store over the existing chain." + exit 1 + fi + + # Walk back from the newest run and take the first that carries this + # leg's artifact. Restoring from the latest *successful* run would + # drop every sample collected while the pipeline was red from a + # regression -- precisely the window that matters. + # + # Absence and failure are kept distinct. A run is only a candidate + # once the artifacts API confirms the artifact exists and has not + # expired; a download that then fails is an operational error + # (token, API, corrupt payload) and fails the job rather than being + # silently downgraded to a cold start. + for run_id in $run_ids; do + artifact_id=$(gh api --paginate \ + "repos/$REPO/actions/runs/$run_id/artifacts" \ + --jq ".artifacts[] | select(.name == \"$ARTIFACT\" and .expired == false) | .id" \ + | head -n1) + [ -n "$artifact_id" ] || continue + + if ! gh run download "$run_id" --name "$ARTIFACT" --dir "$staging"; then + echo "::error::found $ARTIFACT in run $run_id but could not download it;" \ + "failing rather than continuing with an empty history, which would" \ + "publish a truncated store over the existing chain." + exit 1 + fi + echo "restored benchmark history from run $run_id" + complete_restore restored + exit 0 + done + + # No run in the window carried the artifact: a genuine cold start + # (first run, or the chain lapsed), which is a valid empty store. + # Surfaced on the summary rather than only in this log -- "history + # quietly restarted" must not look like "no regressions". + complete_restore cold-start + echo "no $ARTIFACT artifact in the last $WINDOW scheduled runs; starting a new history" + { + printf '### Benchmark history: cold start\n\n' + printf 'No `%s` artifact was found in the last %s `%s` runs on `%s`, ' \ + "$ARTIFACT" "$WINDOW" "$WORKFLOW" "$DEFAULT_BRANCH" + printf 'so this run starts a new series. Trend detection needs several runs of history.\n' + } >> "$GITHUB_STEP_SUMMARY" + - name: Run Anvil group id: run if: steps.setup.outcome == 'success' @@ -453,6 +580,7 @@ runs: # target/anvil/impact cache (read via `_anvil-impact-include`), not # threaded --package strings. This action only fixes the mode. ANVIL_IMPACT: ${{ inputs.impact_mode }} + ANVIL_BENCH_MACHINE_KEY: ${{ inputs.bench_machine_key }} # Some checks (e.g. cargo-aprz) call GitHub's API. The built-in token # gives them the authenticated quota without adding group knowledge # to this action. @@ -475,6 +603,33 @@ runs: # Reporting is supplemental: run after success or failure, but never let # an API outage determine the authoritative workflow-job result. + - name: Save benchmark history + # always(): the run's own samples belong in the history even when the + # analysis flagged a regression and failed the group. + # + # Guarded on the restore having reached a known state: if the restore + # failed operationally the store is not a continuation of the chain, + # and publishing it would overwrite good history with a truncated + # snapshot. + if: always() && inputs.bench_history == 'true' && env.ANVIL_BENCH_RESTORE != '' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ inputs.bench_artifact }} + path: target/anvil/bench-history + # Comfortably longer than the scheduled cadence, so a paused or + # infrequent schedule does not break the chain. + retention-days: 90 + if-no-files-found: ignore + + - name: Publish benchmark findings + if: always() && inputs.bench_history == 'true' + shell: bash + run: | + set -euo pipefail + if [ -f target/anvil/bench/findings.md ]; then + cat target/anvil/bench/findings.md >> "$GITHUB_STEP_SUMMARY" + fi + - name: Publish supplemental Anvil commit status if: always() && inputs.publish_commit_statuses == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository continue-on-error: true @@ -1107,6 +1262,14 @@ on: description: Runner label for aarch64 Windows jobs. type: string default: windows-11-arm + bench_machine_key: + description: | + Machine key the benchmark history is partitioned by. Leave empty to + use cargo-bench-history's hardware fingerprint. Set a stable pool + label when the runner pool is heterogeneous enough to fragment a + series into partitions too sparse to analyze. + type: string + default: "" secrets: CODECOV_TOKEN: description: | @@ -1224,6 +1387,35 @@ jobs: with: group: scheduled-exhaustive + scheduled-benchmarks: + strategy: + fail-fast: false + matrix: + os: [linux, windows] + runs-on: ${{ matrix.os == 'linux' && inputs.linux_runner || inputs.windows_runner }} + permissions: + contents: read + # Restoring the history walks the Actions runs/artifacts API. An action + # cannot request permissions, so this has to be granted here. + actions: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The analysis orders each series by first-parent commit topology + # and locates the merge-base, so it needs the whole commit graph. + # The checkout has already happened by the time an action runs, so + # this too has to be set here. + fetch-depth: 0 + lfs: true + - uses: ./.github/actions/anvil-run-group + with: + group: scheduled-benchmarks + bench_history: true + # Per-leg identity: the matrix value is in scope here and nowhere + # inside the action. + bench_artifact: bench-history-${{ matrix.os }} + bench_machine_key: ${{ inputs.bench_machine_key }} + publish-failure: name: Publish scheduled failure needs: @@ -1231,6 +1423,7 @@ jobs: - scheduled-advisories - scheduled-runtime-analysis - scheduled-exhaustive + - scheduled-benchmarks if: ${{ always() && vars.ANVIL_PUBLISH_FAILURE_ISSUE != 'false' && contains(needs.*.result, 'failure') }} runs-on: ${{ inputs.linux_runner }} @@ -1316,6 +1509,9 @@ jobs: # and restores that scope only on publish-failure. permissions: contents: read + # The scheduled-benchmarks job restores its history artifact, which + # reads the Actions runs/artifacts API. Narrowed to that job inside. + actions: read issues: write secrets: inherit @@ -1581,6 +1777,328 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh", "-NoProfile")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on + # every platform, so on Linux two genuinely different directories would + # compare equal and the detached-store failure this guard exists to + # catch would slip through. + $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } + if (-not $same) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if ([string]::IsNullOrWhiteSpace($value)) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh", "-NoProfile")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + # `benchmark` and `commit` are passed to cbh and git. A leading `-` + # would be read as an option there, so reject it here rather than let + # the file's documented subset be reinterpreted: `benchmark = "--all"` + # would otherwise accept every benchmark at the commit while the log + # claimed a benchmark of that name. + foreach ($guarded in @('benchmark', 'commit')) { + if ($e[$guarded].StartsWith('-')) { + Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + # A blessing can only be applied at a commit the store still holds a clean + # run for: cbh rejects a context commit with no data point, and `collect` + # only ever records the current commit, so a commit the store has forgotten + # can never be re-established by a later run. Without this, any cold start + # or artifact eviction past a blessed commit would turn the group + # permanently red until a human edited the ledger -- a blessing whose + # commit has aged out is inapplicable, not invalid. + $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" + & cargo bench-history list runs --local="$store" ` + --since 1970-01-01 --no-text --json $runsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $stored = [System.Collections.Generic.List[string]]::new() + foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { + foreach ($c in $set.commits) { + if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } + } + } + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Compare the persisted identity exactly. `list blessings --all` is the + # window view: it reports resolved, concrete benchmark ids and never + # populates `prefixes`. Since cbh's positional is a prefix, a ledger + # entry naming a family expands to several concrete rows and matches + # none of them, so such an entry is re-applied each run. That is the + # safe direction to be wrong in: re-blessing is idempotent in effect, + # whereas a loose prefix test would skip an entry that is only + # partially applied and leave the build red while claiming otherwise. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { + Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + # `--` so a benchmark id is never parsed as a flag: bless's positional + # takes no hyphen values, and it has an `--all` sibling that would + # otherwise accept every benchmark at the commit. + & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4825,6 +5343,42 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -5766,6 +6320,7 @@ import 'impact.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -5807,6 +6362,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'tiers.just' import 'tools.just' import 'versions.just' @@ -5856,7 +6412,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full-workspace for the same reason as the scheduled tier. @@ -5899,7 +6456,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -5907,7 +6465,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -6496,6 +7055,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -6740,6 +7307,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.1.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.10" cargo_check_external_types_version := "0.5.0" diff --git a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap index 5e7f6d13e..e9546889c 100644 --- a/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap +++ b/crates/cargo-anvil/tests/snapshots/snapshots__local_only.snap @@ -454,6 +454,328 @@ anvil-audit-setup installer="install": (anvil-tool-cargo-audit-install installer [group("anvil-setup")] anvil-audit-validate-prereqs: anvil-tool-cargo-audit-validate-prereqs +=== justfiles/anvil/checks/bench-history.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh", "-NoProfile")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on + # every platform, so on Linux two genuinely different directories would + # compare equal and the detached-store failure this guard exists to + # catch would slip through. + $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } + if (-not $same) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if ([string]::IsNullOrWhiteSpace($value)) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh", "-NoProfile")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + # `benchmark` and `commit` are passed to cbh and git. A leading `-` + # would be read as an option there, so reject it here rather than let + # the file's documented subset be reinterpreted: `benchmark = "--all"` + # would otherwise accept every benchmark at the commit while the log + # claimed a benchmark of that name. + foreach ($guarded in @('benchmark', 'commit')) { + if ($e[$guarded].StartsWith('-')) { + Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + # A blessing can only be applied at a commit the store still holds a clean + # run for: cbh rejects a context commit with no data point, and `collect` + # only ever records the current commit, so a commit the store has forgotten + # can never be re-established by a later run. Without this, any cold start + # or artifact eviction past a blessed commit would turn the group + # permanently red until a human edited the ledger -- a blessing whose + # commit has aged out is inapplicable, not invalid. + $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" + & cargo bench-history list runs --local="$store" ` + --since 1970-01-01 --no-text --json $runsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $stored = [System.Collections.Generic.List[string]]::new() + foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { + foreach ($c in $set.commits) { + if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } + } + } + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Compare the persisted identity exactly. `list blessings --all` is the + # window view: it reports resolved, concrete benchmark ids and never + # populates `prefixes`. Since cbh's positional is a prefix, a ledger + # entry naming a family expands to several concrete rows and matches + # none of them, so such an entry is re-applied each run. That is the + # safe direction to be wrong in: re-blessing is idempotent in effect, + # whereas a loose prefix test would skip an entry that is only + # partially applied and leave the build red while claiming otherwise. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { + Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + # `--` so a benchmark id is never parsed as a flag: bless's positional + # takes no hyphen values, and it has an `--all` sibling that would + # otherwise accept every benchmark at the commit. + & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs + === justfiles/anvil/checks/bench.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -3698,6 +4020,42 @@ anvil-scheduled-advisories-validate-prereqs: \ anvil-aprz-validate-prereqs \ anvil-clippy-validate-prereqs +=== justfiles/anvil/groups/scheduled-benchmarks.just === +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs + === justfiles/anvil/groups/scheduled-exhaustive.just === # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. @@ -4639,6 +4997,7 @@ import 'impact.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -4680,6 +5039,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'tiers.just' import 'tools.just' import 'versions.just' @@ -4729,7 +5089,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full-workspace for the same reason as the scheduled tier. @@ -4772,7 +5133,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -4780,7 +5142,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] @@ -5369,6 +5732,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the @@ -5613,6 +5984,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.1.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.10" cargo_check_external_types_version := "0.5.0" diff --git a/justfiles/anvil/checks/bench-history.just b/justfiles/anvil/checks/bench-history.just new file mode 100644 index 000000000..85e2ac7b2 --- /dev/null +++ b/justfiles/anvil/checks/bench-history.just @@ -0,0 +1,320 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# Unscoped by design. A benchmark's series is only comparable when the +# same suite is measured at every commit, so impact-scoping the run would +# punch holes in the history that detection cannot distinguish from a +# benchmark being deleted. The recipe therefore ignores impact scoping and +# always measures the whole workspace. +# +# Environment contract (all optional): +# ANVIL_BENCH_HISTORY_STORE history directory (default target/anvil/bench-history). +# Local-only: the generated cloud wiring restores and +# publishes the default path, so overriding it in CI +# would leave the recipe reading a different directory +# than the one the artifact round-trip maintains. +# ANVIL_BENCH_MACHINE_KEY machine key overriding cbh's hardware fingerprint, +# for pools heterogeneous enough to fragment a series. +# Both backends plumb this through their scheduled +# wiring, so it is settable in CI as well as locally. +# ANVIL_BENCH_GATE "1" to make an active regression fail this recipe +# locally. CI sets it automatically; see below. +# +# The store is the cross-run state the cloud wiring restores before and +# publishes after this recipe; locally it is whatever has accumulated +# under target/, which on a fresh checkout is empty and analyzes to a +# clean no-op. +# +# Gating is CI-only by default. The recipe behaves identically either way +# --- it always runs the benches and always writes its findings --- but a +# laptop produces measurement noise that a shared, homogeneous runner pool +# does not, and `anvil-scheduled` / `anvil-full` are run locally before a +# release. Failing those on thermal throttling would invite committing a +# blessing to silence it, which would pollute the reviewed, audited +# blessings file with an artifact of one developer's hardware. + +# Run the benchmarks and analyze the accumulated history for regressions. +[script("pwsh", "-NoProfile")] +anvil-bench-history: anvil-bench-history-validate-prereqs + $ErrorActionPreference = 'Stop' + + # The CI wiring restores into, and publishes from, one fixed path. If + # the recipe wrote anywhere else the store would never persist: every + # run would cold-start and analyze to a clean no-op, reporting green + # precisely when it has lost the history it needs to report red. The + # wiring therefore announces its path and the recipe refuses to + # disagree with it. + $wired = $env:ANVIL_BENCH_WIRED_STORE + $store = if ($env:ANVIL_BENCH_HISTORY_STORE) { + $env:ANVIL_BENCH_HISTORY_STORE + } elseif ($wired) { + $wired + } else { + 'target/anvil/bench-history' + } + + if ($wired) { + $wantPath = [System.IO.Path]::GetFullPath($wired) + $gotPath = [System.IO.Path]::GetFullPath($store) + # Case-sensitively off Windows: PowerShell's -ne is case-insensitive on + # every platform, so on Linux two genuinely different directories would + # compare equal and the detached-store failure this guard exists to + # catch would slip through. + $same = if ($IsWindows) { $wantPath -eq $gotPath } else { $wantPath -ceq $gotPath } + if (-not $same) { + Write-Error "anvil: the benchmark history store is '$store', but the CI wiring restores and publishes '$wired'. Results written to the former would never be persisted, so every run would cold-start and report a false clean." + exit 1 + } + } + $reportDir = 'target/anvil/bench' + $findingsMd = Join-Path $reportDir 'findings.md' + $summaryMd = Join-Path $reportDir 'findings-summary.md' + $findingsJson = Join-Path $reportDir 'findings.json' + $blessingsFile = '.config/bench-blessings.toml' + + [System.IO.Directory]::CreateDirectory($store) | Out-Null + [System.IO.Directory]::CreateDirectory($reportDir) | Out-Null + + # The machine key partitions every series. cargo-bench-history derives + # it from the host's hardware fingerprint; an adopter whose runner pool + # is heterogeneous enough to fragment the series into unanalyzable + # partitions sets ANVIL_BENCH_MACHINE_KEY to a stable pool label + # instead. It has to be the same on collect, bless, list and analyze, + # so every invocation below splats the same argument list. + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + # --skip-existing makes a re-run at an already-recorded commit a + # success that writes nothing, so a re-queued scheduled build does not + # fail on the duplicate and does not overwrite the original sample. + Write-Host 'anvil-bench-history: collecting benchmark results' + & cargo bench-history collect --local="$store" --skip-existing --all-features @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Blessings accept an intentional change. They live in a reviewed, + # committed file and are applied into the store here, ahead of the + # analysis, so the store stays single-writer. + & "{{just_executable()}}" _anvil-bench-history-bless "$store" "$blessingsFile" + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + Write-Host 'anvil-bench-history: analyzing history' + & cargo bench-history analyze --local="$store" ` + --markdown $findingsMd --markdown-summary $summaryMd --json $findingsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + # Findings never affect cargo-bench-history's own exit code -- the + # machine-readable report is the signal. An *active* regression is the + # one thing that gates: an inactive finding has already recovered, and + # an improvement needs no action. + $report = Get-Content -LiteralPath $findingsJson -Raw | ConvertFrom-Json + $regressions = @($report.findings | Where-Object { $_.direction -eq 'regression' -and $_.active }) + if ($regressions.Count -eq 0) { + Write-Host 'anvil-bench-history: no active regressions' + exit 0 + } + + Write-Host '' + Write-Host "anvil-bench-history: $($regressions.Count) active benchmark regression(s)" -ForegroundColor Red + foreach ($r in $regressions) { + $id = ($r.segments -join '/') + $delta = '{0:P2}' -f $r.relative_delta + Write-Host " $id ($($r.kind)) $delta at $($r.commit)" + } + Write-Host '' + Write-Host "Findings: $findingsMd" + + # CI sets its own marker (GitHub: CI, ADO: TF_BUILD); locally the gate + # is opt-in via ANVIL_BENCH_GATE. Reporting above is unconditional -- + # only the exit code differs. + # + # Read as boolean-like rather than for truthiness: PowerShell treats any + # non-empty string as true, so a developer with CI=false exported would + # otherwise have local runs fail on a shared trend measured on their own + # hardware. Unrecognised non-empty values still gate, keeping the CI side + # fail-closed. + function Test-Flag([string]$value) { + if ([string]::IsNullOrWhiteSpace($value)) { return $false } + return $value.Trim().ToLowerInvariant() -notin @('0', 'false', 'no', 'off') + } + $gate = (Test-Flag $env:ANVIL_BENCH_GATE) -or (Test-Flag $env:CI) -or (Test-Flag $env:TF_BUILD) + if (-not $gate) { + Write-Host 'Reporting only: a local run does not gate on the shared trend.' -ForegroundColor Yellow + Write-Host 'These numbers come from this machine, whose noise a shared runner pool does not have.' + Write-Host 'Set ANVIL_BENCH_GATE=1 to make this fail locally too.' + exit 0 + } + + Write-Host "Fix the regression, or accept it by adding an entry to $blessingsFile." + exit 1 + +# Apply the committed blessings into the history store, idempotently. +# +# `bless` writes an append-only sidecar into the store, so an entry that +# is already in effect must not be re-applied on every scheduled run. +# The already-applied set comes from `list blessings`, widened past the +# default look-back so an old entry is not mistaken for a missing one. +# +# The file is a table array; unknown keys are ignored so the schema can +# grow without breaking older tool pins: +# +# [[blessing]] +# benchmark = "my_pkg/my_group/my_case" +# commit = "8392995a" +# reason = "switched to the arena allocator; the extra setup is intentional" +[private] +[script("pwsh", "-NoProfile")] +_anvil-bench-history-bless store blessings: + $ErrorActionPreference = 'Stop' + $store = '{{store}}' + $blessingsFile = '{{blessings}}' + + if (-not (Test-Path -LiteralPath $blessingsFile)) { + Write-Host "anvil-bench-history: no $blessingsFile; nothing to bless" + exit 0 + } + + # A deliberately small TOML subset: `[[blessing]]` headers plus + # `key = "value"` pairs with no escapes. Anything outside it is + # rejected rather than reinterpreted, so a value this cannot represent + # fails loudly instead of being silently rewritten. The same subset is + # documented in the emitted file's own header. + $entries = New-Object System.Collections.Generic.List[object] + $current = $null + foreach ($rawLine in (Get-Content -LiteralPath $blessingsFile)) { + $line = $rawLine.Trim() + # A `#` only starts a comment outside a value. Stripping to the + # first `#` unconditionally would silently truncate a reason + # citing an issue or PR number -- exactly what a rationale + # contains -- so comments are only recognised at line start. + if (-not $line -or $line.StartsWith('#')) { continue } + if ($line -eq '[[blessing]]') { + $current = @{} + $entries.Add($current) | Out-Null + continue + } + if ($line -match '^\[') { + Write-Error "anvil-bench-history: unexpected table '$line' in $blessingsFile (expected only [[blessing]])" + exit 1 + } + if ($line -match '^([A-Za-z_][A-Za-z0-9_-]*)\s*=\s*"([^"\\]*)"$') { + if ($null -eq $current) { + Write-Error "anvil-bench-history: key '$($Matches[1])' outside any [[blessing]] in $blessingsFile" + exit 1 + } + $current[$Matches[1]] = $Matches[2] + continue + } + Write-Error ('anvil-bench-history: cannot parse ''{0}'' in {1}. Expected a [[blessing]] header, a line-leading # comment, or key = "value" with a double-quoted single-line value containing no backslash escapes.' -f $line, $blessingsFile) + exit 1 + } + + if ($entries.Count -eq 0) { + Write-Host "anvil-bench-history: $blessingsFile declares no blessings" + exit 0 + } + + foreach ($e in $entries) { + foreach ($required in @('benchmark', 'commit', 'reason')) { + if (-not $e[$required]) { + Write-Error "anvil-bench-history: a [[blessing]] in $blessingsFile is missing '$required'" + exit 1 + } + } + # `benchmark` and `commit` are passed to cbh and git. A leading `-` + # would be read as an option there, so reject it here rather than let + # the file's documented subset be reinterpreted: `benchmark = "--all"` + # would otherwise accept every benchmark at the commit while the log + # claimed a benchmark of that name. + foreach ($guarded in @('benchmark', 'commit')) { + if ($e[$guarded].StartsWith('-')) { + Write-Error "anvil-bench-history: '$guarded' in $blessingsFile must not begin with '-' (got '$($e[$guarded])'); it is a benchmark id or commit, not an option" + exit 1 + } + } + } + + $key = @() + if ($env:ANVIL_BENCH_MACHINE_KEY) { $key = @('--machine-key', $env:ANVIL_BENCH_MACHINE_KEY) } + + $tmpDir = $env:RUNNER_TEMP + if (-not $tmpDir) { $tmpDir = $env:AGENT_TEMPDIRECTORY } + if (-not $tmpDir) { $tmpDir = [System.IO.Path]::GetTempPath() } + # Process-scoped: two jobs sharing a machine (matrix legs on a self-hosted + # agent, or concurrent local runs) would otherwise race on one filename and + # read each other's listing. + $listJson = Join-Path $tmpDir "anvil-bench-blessings-$PID.json" + + & cargo bench-history list blessings --all --local="$store" ` + --since 1970-01-01 --no-text --json $listJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $applied = @((Get-Content -LiteralPath $listJson -Raw | ConvertFrom-Json).blessings) + + # A blessing can only be applied at a commit the store still holds a clean + # run for: cbh rejects a context commit with no data point, and `collect` + # only ever records the current commit, so a commit the store has forgotten + # can never be re-established by a later run. Without this, any cold start + # or artifact eviction past a blessed commit would turn the group + # permanently red until a human edited the ledger -- a blessing whose + # commit has aged out is inapplicable, not invalid. + $runsJson = Join-Path $tmpDir "anvil-bench-runs-$PID.json" + & cargo bench-history list runs --local="$store" ` + --since 1970-01-01 --no-text --json $runsJson @key + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + $stored = [System.Collections.Generic.List[string]]::new() + foreach ($set in (Get-Content -LiteralPath $runsJson -Raw | ConvertFrom-Json).sets) { + foreach ($c in $set.commits) { + if ($c.clean -ge 1 -and $c.commit) { $stored.Add($c.commit) | Out-Null } + } + } + + foreach ($e in $entries) { + $commit = $e['commit'] + $benchmark = $e['benchmark'] + # Resolve to a full commit id up front: the file may carry an + # abbreviated id, and a bogus one should fail here with git's own + # message rather than silently bless nothing. + $resolved = (& git rev-parse --verify "$commit^{commit}" 2>$null) + if ($LASTEXITCODE -ne 0 -or -not $resolved) { + Write-Error "anvil-bench-history: commit '$commit' in $blessingsFile is not present in this clone" + exit 1 + } + $resolved = $resolved.Trim() + # Compare the persisted identity exactly. `list blessings --all` is the + # window view: it reports resolved, concrete benchmark ids and never + # populates `prefixes`. Since cbh's positional is a prefix, a ledger + # entry naming a family expands to several concrete rows and matches + # none of them, so such an entry is re-applied each run. That is the + # safe direction to be wrong in: re-blessing is idempotent in effect, + # whereas a loose prefix test would skip an entry that is only + # partially applied and leave the build red while claiming otherwise. + $already = $applied | Where-Object { + $resolved.StartsWith($_.commit) -and ($_.benchmark -eq $benchmark) + } + if ($already) { + Write-Host "anvil-bench-history: blessing already in effect: $benchmark at $commit" + continue + } + if (-not ($stored | Where-Object { $resolved.StartsWith($_) })) { + Write-Host "anvil-bench-history: skipping blessing of $benchmark at $commit -- the store holds no run at that commit, so there is nothing to accept. It applies again if that commit is measured again." + continue + } + Write-Host "anvil-bench-history: blessing $benchmark at $commit -- $($e['reason'])" + # `--` so a benchmark id is never parsed as a flag: bless's positional + # takes no hyphen values, and it has an `--all` sibling that would + # otherwise accept every benchmark at the commit. + & cargo bench-history bless --local="$store" --context $resolved @key -- $benchmark + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + } + +# Install prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-setup installer="install": (anvil-tool-cargo-bench-history-install installer) + +# Validate prerequisites for the `anvil-bench-history` recipe. +[group("anvil-setup")] +anvil-bench-history-validate-prereqs: anvil-tool-cargo-bench-history-validate-prereqs diff --git a/justfiles/anvil/groups/scheduled-benchmarks.just b/justfiles/anvil/groups/scheduled-benchmarks.just new file mode 100644 index 000000000..9a4e119aa --- /dev/null +++ b/justfiles/anvil/groups/scheduled-benchmarks.just @@ -0,0 +1,34 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# GENERATED BY cargo-anvil. DO NOT EDIT DIRECTLY. +# Update cargo-anvil and regenerate; repository-specific edits stop automatic updates. +# Update behaviour: https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/updates.md + +# See https://github.com/microsoft/ox-tools/blob/main/crates/cargo-anvil/docs/design/benchmarks.md + +# scheduled-benchmarks holds the one check whose verdict depends on state +# carried between runs. Keeping it in its own group isolates that history +# round-trip and its fail-on-regression semantics from the rest of the +# scheduled work, so a red build names the regression unambiguously. +# +# Routed through _anvil-unscoped like every scheduled group: the series only +# stays comparable if the same suite is measured at every commit, so scoping +# the run to a change set would silently break detection. + +# Run the scheduled benchmark regression detection. +[group("anvil")] +anvil-scheduled-benchmarks: (_anvil-unscoped "scheduled-benchmarks") + +[private] +_anvil-scheduled-benchmarks: anvil-scheduled-benchmarks-validate-prereqs \ + anvil-bench-history + +# Install prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-setup installer="install": \ + (anvil-bench-history-setup installer) + +# Validate prerequisites for the `anvil-scheduled-benchmarks` recipe. +[group("anvil-setup")] +anvil-scheduled-benchmarks-validate-prereqs: \ + anvil-bench-history-validate-prereqs diff --git a/justfiles/anvil/mod.just b/justfiles/anvil/mod.just index ee29054c9..8f937ae22 100644 --- a/justfiles/anvil/mod.just +++ b/justfiles/anvil/mod.just @@ -40,6 +40,7 @@ import 'impact.just' import 'checks/aprz.just' import 'checks/audit.just' import 'checks/bench.just' +import 'checks/bench-history.just' import 'checks/bolero.just' import 'checks/careful.just' import 'checks/cargo-hack.just' @@ -81,6 +82,7 @@ import 'groups/scheduled-test.just' import 'groups/scheduled-advisories.just' import 'groups/scheduled-runtime-analysis.just' import 'groups/scheduled-exhaustive.just' +import 'groups/scheduled-benchmarks.just' import 'tiers.just' import 'tools.just' import 'versions.just' diff --git a/justfiles/anvil/tiers.just b/justfiles/anvil/tiers.just index a06e87b4f..9e0822b6a 100644 --- a/justfiles/anvil/tiers.just +++ b/justfiles/anvil/tiers.just @@ -39,7 +39,8 @@ _anvil-scheduled: anvil-scheduled-validate-prereqs \ anvil-scheduled-test \ anvil-scheduled-advisories \ anvil-scheduled-runtime-analysis \ - anvil-scheduled-exhaustive + anvil-scheduled-exhaustive \ + anvil-scheduled-benchmarks # Full-workspace for the same reason as the scheduled tier. @@ -82,7 +83,8 @@ anvil-scheduled-setup installer="install": \ (anvil-scheduled-test-setup installer) \ (anvil-scheduled-advisories-setup installer) \ (anvil-scheduled-runtime-analysis-setup installer) \ - (anvil-scheduled-exhaustive-setup installer) + (anvil-scheduled-exhaustive-setup installer) \ + (anvil-scheduled-benchmarks-setup installer) # Validate prerequisites for the `anvil-scheduled` recipe. [group("anvil-setup")] @@ -90,7 +92,8 @@ anvil-scheduled-validate-prereqs: \ anvil-scheduled-test-validate-prereqs \ anvil-scheduled-advisories-validate-prereqs \ anvil-scheduled-runtime-analysis-validate-prereqs \ - anvil-scheduled-exhaustive-validate-prereqs + anvil-scheduled-exhaustive-validate-prereqs \ + anvil-scheduled-benchmarks-validate-prereqs # Install prerequisites for the `anvil-full` recipe. [group("anvil-setup")] diff --git a/justfiles/anvil/tools.just b/justfiles/anvil/tools.just index cb6aab145..9b0c751dc 100644 --- a/justfiles/anvil/tools.just +++ b/justfiles/anvil/tools.just @@ -550,6 +550,14 @@ anvil-tool-cargo-audit-install installer="install": (_install-tool "cargo-audit" [group("anvil-setup")] anvil-tool-cargo-audit-validate-prereqs: (_check-tool "cargo-audit" cargo_audit_version) +# Install the pinned `cargo-bench-history` tool. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-install installer="install": (_install-tool "cargo-bench-history" cargo_bench_history_version installer) + +# Validate that the pinned `cargo-bench-history` tool is available. +[group("anvil-setup")] +anvil-tool-cargo-bench-history-validate-prereqs: (_check-tool "cargo-bench-history" cargo_bench_history_version) + # cargo-bolero is Linux-only: its `bolero-afl` build dependency # compiles AFL's native C (afl-fuzz.c), which needs POSIX headers # (`unistd.h`) and uses preprocessor constructs MSVC rejects, so the diff --git a/justfiles/anvil/versions.just b/justfiles/anvil/versions.just index 0a4aeaa13..c9ad5d32c 100644 --- a/justfiles/anvil/versions.just +++ b/justfiles/anvil/versions.just @@ -51,6 +51,7 @@ rust_nightly_external_types := "nightly-2026-03-20" cargo_aprz_version := "1.1.0" cargo_audit_version := "0.22.2" +cargo_bench_history_version := "0.0.9" cargo_bolero_version := "0.13.4" cargo_careful_version := "0.4.10" cargo_check_external_types_version := "0.5.0"