diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000000..d5a09280d5 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,89 @@ +name: CodSpeed + +on: + push: + branches: + - master + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + +permissions: + contents: read + id-token: write + +env: + CARGO_INCREMENTAL: 0 + CARGO_NET_RETRY: 10 + RUSTUP_MAX_RETRIES: 10 + RUST_BACKTRACE: short + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 1 + CARGO_PROFILE_DEV_DEBUG: 0 + +jobs: + simulation: + name: CodSpeed simulation + if: vars.CODSPEED_ENABLED == 'true' + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v6 + + - name: Install build dependencies + uses: ./.github/actions/install-build-deps + + - name: Rust cache + uses: Swatinem/rust-cache@v2.9.1 + + - name: Corpus cache + uses: actions/cache@v5 + with: + path: ./bench-data/wireshark-rdp + key: ${{ runner.os }}-codspeed-corpus-${{ hashFiles('crates/ironrdp-bench/corpus.toml') }} + + - name: Fetch corpus + run: cargo xtask bench corpus-fetch + + - name: Install cargo-codspeed + run: cargo install cargo-codspeed --version 5.0.1 --locked + + - name: Build benchmarks + run: cargo codspeed build -p ironrdp-bench --bench bench --bench capture_replay --features codspeed --locked -m simulation + + - name: Measure benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: cargo codspeed run -p ironrdp-bench --bench bench --bench capture_replay -m simulation + + walltime: + name: CodSpeed walltime + if: github.event_name == 'workflow_dispatch' && vars.CODSPEED_ENABLED == 'true' && vars.CODSPEED_WALLTIME_RUNNER != '' + runs-on: ${{ vars.CODSPEED_WALLTIME_RUNNER }} + + steps: + - uses: actions/checkout@v6 + + - name: Install build dependencies + uses: ./.github/actions/install-build-deps + + - name: Rust cache + uses: Swatinem/rust-cache@v2.9.1 + + - name: Corpus cache + uses: actions/cache@v5 + with: + path: ./bench-data/wireshark-rdp + key: ${{ runner.os }}-codspeed-corpus-${{ hashFiles('crates/ironrdp-bench/corpus.toml') }} + + - name: Fetch corpus + run: cargo xtask bench corpus-fetch + + - name: Build replay command + run: cargo build --release -p ironrdp-bench --bin capture-replay-bench --locked + + - name: Measure replay commands + uses: CodSpeedHQ/action@v5 + with: + mode: walltime diff --git a/Cargo.lock b/Cargo.lock index 0920382be9..ec2288ba09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -264,6 +264,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arbitrary" version = "1.4.2" @@ -798,12 +807,79 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "codspeed" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7083f253260bcb4aaa3b4aa4c52973703dabc1a85c2f193997e2689aafa8a919" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.4.3", + "glob", + "libc", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-criterion-compat" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9f24251445188c69d50f10179d795424bd71b533b7e3f84fc03f26893b01af0" +dependencies = [ + "clap", + "codspeed", + "codspeed-criterion-compat-walltime", + "colored", + "regex", +] + +[[package]] +name = "codspeed-criterion-compat-walltime" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c38205d56e2cb4fe04b708de7f9653a3f1b89edbe3a20b28f21e9e525e9e061" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "codspeed", + "criterion-plot 0.5.0", + "is-terminal", + "itertools 0.10.5", + "num-traits", + "once_cell", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_derive", + "serde_json", + "tinytemplate", + "walkdir", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "combine" version = "4.6.7" @@ -992,8 +1068,8 @@ dependencies = [ "cast", "ciborium", "clap", - "criterion-plot", - "itertools", + "criterion-plot 0.8.2", + "itertools 0.13.0", "num-traits", "oorandom", "page_size", @@ -1006,6 +1082,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "criterion-plot" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +dependencies = [ + "cast", + "itertools 0.10.5", +] + [[package]] name = "criterion-plot" version = "0.8.2" @@ -1013,7 +1099,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" dependencies = [ "cast", - "itertools", + "itertools 0.13.0", ] [[package]] @@ -2670,6 +2756,7 @@ version = "0.0.0" dependencies = [ "anyhow", "async-trait", + "codspeed-criterion-compat", "criterion", "ironrdp", "ironrdp-capture-replay", @@ -3567,6 +3654,17 @@ dependencies = [ "x509-cert", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -3591,6 +3689,15 @@ dependencies = [ "untrusted", ] +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -6323,6 +6430,16 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strck" version = "0.1.2" diff --git a/codspeed.yml b/codspeed.yml new file mode 100644 index 0000000000..c7a6dc4da7 --- /dev/null +++ b/codspeed.yml @@ -0,0 +1,9 @@ +benchmarks: + - name: partial-replay/no-nla-accepted/strict-cli + exec: ./target/release/capture-replay-bench --capture no-nla-accepted + + - name: partial-replay/no-nla-smartcard/strict-cli + exec: ./target/release/capture-replay-bench --capture no-nla-smartcard + + - name: connector-replay/no-nla-accepted/strict-cli + exec: ./target/release/capture-replay-bench --connector no-nla-accepted diff --git a/crates/ironrdp-bench/Cargo.toml b/crates/ironrdp-bench/Cargo.toml index 31994e2ddd..0e04688061 100644 --- a/crates/ironrdp-bench/Cargo.toml +++ b/crates/ironrdp-bench/Cargo.toml @@ -9,6 +9,7 @@ publish = false default = ["qoi", "qoiz"] qoi = ["ironrdp/qoi"] qoiz = ["ironrdp/qoiz"] +codspeed = ["dep:codspeed-criterion-compat"] [dependencies] anyhow = "1" @@ -21,6 +22,7 @@ tokio = { version = "1", features = ["sync", "fs", "time", "io-util", "macros", toml = "1.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing = { version = "0.1", features = ["log"] } +codspeed-criterion-compat = { version = "5.0.1", optional = true } [dev-dependencies] criterion = "0.8" diff --git a/crates/ironrdp-bench/benches/bench.rs b/crates/ironrdp-bench/benches/bench.rs index fdf36bbdc3..d7c546952e 100644 --- a/crates/ironrdp-bench/benches/bench.rs +++ b/crates/ironrdp-bench/benches/bench.rs @@ -4,6 +4,8 @@ use core::hint::black_box; use core::num::{NonZeroU16, NonZeroUsize}; +#[cfg(feature = "codspeed")] +use codspeed_criterion_compat as criterion; use criterion::{Criterion, criterion_group, criterion_main}; use ironrdp_graphics::color_conversion::to_64x64_ycbcr_tile; use ironrdp_pdu::codecs::rfx; diff --git a/crates/ironrdp-bench/benches/capture_replay.rs b/crates/ironrdp-bench/benches/capture_replay.rs index 85de160ce7..1e8790fdc3 100644 --- a/crates/ironrdp-bench/benches/capture_replay.rs +++ b/crates/ironrdp-bench/benches/capture_replay.rs @@ -2,6 +2,8 @@ use core::hint::black_box; +#[cfg(feature = "codspeed")] +use codspeed_criterion_compat as criterion; use criterion::{Criterion, criterion_group, criterion_main}; use ironrdp_bench::connector_replay::{ConnectorReplayId, ConnectorReplayWorkload}; use ironrdp_bench::replay::{PartialReplayId, PartialReplayWorkload}; diff --git a/xtask/README.md b/xtask/README.md index d076ca1f8b..c1dbbeb93e 100644 --- a/xtask/README.md +++ b/xtask/README.md @@ -98,5 +98,38 @@ hyperfine --warmup 1 '.\target\release\capture-replay-bench.exe --connector no-n ``` Use `cargo xtask bench replay` to regression-test the complete pinned corpus, not to produce a single-capture timing score. +## CodSpeed reporting + +CodSpeed simulation measures the three individually named encoder workloads (`rfx_enc_tile`, `rfx_enc`, and `to_ycbcr`) plus the two passive partial replays and one connector replay. +The simulation job fetches and verifies the corpus before benchmark setup, then each replay target prepares and strictly preflights its selected capture outside Criterion's timed iterations. +Every timed iteration starts with fresh replay, connector, and session state. +CPU simulation models deterministic user-space work and is useful for focused processing regressions, not wall-clock completion, hardware-specific SIMD behavior, or native H.264 performance. + +The `codspeed` feature selects `codspeed-criterion-compat` only for the two benchmark targets. +Regular `cargo bench` continues to use Criterion 0.8. +Build the CodSpeed targets locally without uploading results with: + +```PowerShell +cargo xtask bench corpus-fetch +cargo install cargo-codspeed --version 5.0.1 --locked +cargo codspeed build -p ironrdp-bench --bench bench --bench capture_replay --features codspeed --locked -m simulation +``` + +Use the local Criterion and Hyperfine commands above; replace `no-nla-accepted` with `no-nla-smartcard` in the Criterion filter to benchmark that passive replay. + +Hyperfine remains a separate local tool and does not import results into CodSpeed. + +`codspeed.yml` defines the independent wall-time command identities `exec_harness::partial-replay/no-nla-accepted/strict-cli`, `exec_harness::partial-replay/no-nla-smartcard/strict-cli`, and `exec_harness::connector-replay/no-nla-accepted/strict-cli`. +Each command starts one release binary process, verifies and loads the selected cached capture, decrypts and prepares its replay state, and executes one strict replay. +Compilation and corpus fetching occur before the CodSpeed action and are excluded from this measurement. + +The `CodSpeed` workflow runs simulation for pushes, pull requests, and manual dispatches after maintainers set the `CODSPEED_ENABLED` repository variable to `true` and enable the repository in CodSpeed. +It authenticates through GitHub OIDC and requires no repository secret for this public repository. +The wall-time job is manual only and runs only when `CODSPEED_ENABLED` is `true` and maintainers set the repository-controlled `CODSPEED_WALLTIME_RUNNER` variable to a known, approved Linux runner label. +Dispatchers cannot select or override the wall-time runner. +Use a dedicated stable runner for wall time; shared hosted runners are too noisy for performance decisions. +The public capture cache uses the existing manifest-hash cache key and is digest-verified before use. +Do not upload captures, decrypted payloads, TLS key material, screenshots, or generated replay output; CodSpeed receives measurements and may retain symbol-bearing profiles, not raw replay payloads. + To update the corpus, inspect the upstream capture inventory, revise the manifest revision, inventory, scenario intent metadata, replay expectations, and SHA-256 digests together, then run `cargo xtask bench corpus-fetch` followed by `cargo xtask bench replay`. Do not commit captures, TLS key material, decrypted payloads, screenshots, or generated output.