From 65ba4279688393fd14af50f1fcefcdaf891df6f4 Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:24:41 +0000 Subject: [PATCH 1/7] ci: remove self-hosted runner privilege requirements --- .github/SELF_HOSTED_RUNNER.md | 9 ++++++ .github/workflows/kotlin-sdk-build.yml | 36 ++++++++++++--------- .github/workflows/tests-rs-workspace.yml | 41 +++++++++--------------- 3 files changed, 46 insertions(+), 40 deletions(-) create mode 100644 .github/SELF_HOSTED_RUNNER.md diff --git a/.github/SELF_HOSTED_RUNNER.md b/.github/SELF_HOSTED_RUNNER.md new file mode 100644 index 00000000000..9ea97cf2e65 --- /dev/null +++ b/.github/SELF_HOSTED_RUNNER.md @@ -0,0 +1,9 @@ +# Linux self-hosted runner contract + +The persistent Linux runners used by `kotlin-ci` and `rust-ci` are containerized for reproducible toolchains. The runner image is the versioned environment; jobs must not install host packages or modify host device permissions. + +The runner container runs as the non-root `runner` user with default seccomp/AppArmor confinement, no added capabilities, and no `/var/run/docker.sock` mount. The Kotlin runner receives only `/dev/kvm` through the `kvm` supplementary group for the Android emulator. The host should expose that device as `root:kvm` with mode `0660`. + +The image must include the build-essential, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, and the Cargo tools required by the workflows. Rebuild and repin the image when those dependencies change. + +Jobs that need to publish Docker images run on GitHub-hosted runners. A future self-hosted job that needs Docker must use a separately isolated runner; the `kotlin-ci` and `rust-ci` containers must not regain a host Docker socket. diff --git a/.github/workflows/kotlin-sdk-build.yml b/.github/workflows/kotlin-sdk-build.yml index 496d5ae5fa6..e24021decbb 100644 --- a/.github/workflows/kotlin-sdk-build.yml +++ b/.github/workflows/kotlin-sdk-build.yml @@ -50,18 +50,25 @@ jobs: # Preserve target/ and other untracked build outputs between runs. clean: false - - name: Ensure runner dependencies + # The persistent runner image is the versioned CI environment. Keep + # dependency installation out of the job: it required sudo and made a + # CI job capable of mutating its container. Rebuild the runner image when + # one of these requirements changes instead. + - name: Verify runner image dependencies run: | set -euo pipefail - MISSING=() - for pkg in build-essential cmake curl jq libgmp-dev libpulse0 libssl-dev libx11-xcb1 pkg-config python3 unzip zip; do - dpkg -s "$pkg" >/dev/null 2>&1 || MISSING+=("$pkg") + missing=() + for tool in cmake curl jq python3 unzip zip; do + command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") done - if [ ${#MISSING[@]} -gt 0 ]; then - echo "Installing: ${MISSING[*]}" - sudo apt-get update -qq - sudo apt-get install -qq --yes "${MISSING[@]}" + for pkg in build-essential libgmp-dev libpulse0 libssl-dev libx11-xcb1 pkg-config; do + dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::Self-hosted runner image is missing: ${missing[*]}" + echo "::error::Provision these in the pinned runner image; CI jobs must not install host packages." + exit 1 fi if [ ! -x "$HOME/.cargo/bin/rustup" ]; then @@ -111,16 +118,12 @@ jobs: cargo ndk --version cargo ndk --version | grep -qx 'cargo-ndk 4.1.2' - - name: Ensure protoc v32.0 is installed (repo-standard; apt's 3.21 breaks tenderdash-proto) + - name: Verify protoc v32.0 (repo-standard; apt's 3.21 breaks tenderdash-proto) run: | set -euo pipefail - if ! protoc --version 2>/dev/null | grep -qx 'libprotoc 32.0'; then - curl -fsSL -o /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip - sudo unzip -o /tmp/protoc.zip -d /usr/local 'bin/protoc' 'include/*' - fi protoc --version - # A stale protoc earlier on PATH would shadow the one just unpacked - # into /usr/local; catch that here rather than in a codegen failure. + # protoc is part of the runner image. Installing it into /usr/local + # from a job required sudo and made the CI container mutable. protoc --version | grep -qx 'libprotoc 32.0' - name: Build native library (x86_64, dev profile) @@ -144,6 +147,9 @@ jobs: working-directory: packages/kotlin-sdk run: ./gradlew :sdk:compileDebugAndroidTestKotlin --stacktrace + # KVM is the only host device this job needs. The container receives it + # through the kvm group; no Docker socket or host package-management + # access is required. - name: Verify KVM access run: | test -c /dev/kvm diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index 622cd5ab76d..736321b0ed4 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -115,38 +115,29 @@ jobs: done exit "$missing" - # clang, llvm and libsnappy are installed by ./.github/actions/rust on - # Linux; this covers what the rest of the job needs and what a bare - # self-hosted image doesn't ship. Every branch is a no-op once the - # persistent runner has been provisioned by the first run. - - name: Install build dependencies (Linux) + # The persistent runner image is the versioned CI environment. Runtime + # apt/sudo would let a job mutate the runner and is unnecessary once the + # image contract is provisioned. + - name: Verify build dependencies (Linux) if: runner.os == 'Linux' run: | set -euo pipefail - MISSING=() - for pkg in build-essential cmake libgmp-dev libssl-dev pkg-config jq zip; do - dpkg -s "$pkg" >/dev/null 2>&1 || MISSING+=("$pkg") + missing=() + for tool in cmake gh jq zip; do + command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") done - if [ ${#MISSING[@]} -gt 0 ]; then - echo "Installing: ${MISSING[*]}" - sudo apt-get update -qq - sudo apt-get install -qq --yes "${MISSING[@]}" - fi - - # Needed by the immutable-structure check below. - if ! command -v gh >/dev/null 2>&1; then - sudo apt-get install -qq --yes gh || { - curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ - | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ - | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null - sudo apt-get update -qq - sudo apt-get install -qq --yes gh - } + for pkg in build-essential libgmp-dev libssl-dev pkg-config; do + dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::Self-hosted runner image is missing: ${missing[*]}" + echo "::error::Provision these in the pinned runner image; CI jobs must not install host packages." + exit 1 fi - # dtolnay/rust-toolchain drives rustup; it must already exist. + # dtolnay/rust-toolchain drives rustup; it may install the toolchain + # under the runner user's home, but never needs root. if ! command -v rustup >/dev/null 2>&1; then curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ | sh -s -- -y --no-modify-path --default-toolchain none From 41f93a13adda860fcd29ec5849ecdd648ee73841 Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:36:54 +0000 Subject: [PATCH 2/7] ci: remove Rust runner package installation --- .github/SELF_HOSTED_RUNNER.md | 2 +- .github/actions/rust/action.yaml | 26 +++++++++++++++++++----- .github/workflows/tests-rs-workspace.yml | 4 ++-- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/SELF_HOSTED_RUNNER.md b/.github/SELF_HOSTED_RUNNER.md index 9ea97cf2e65..8ac074855c2 100644 --- a/.github/SELF_HOSTED_RUNNER.md +++ b/.github/SELF_HOSTED_RUNNER.md @@ -4,6 +4,6 @@ The persistent Linux runners used by `kotlin-ci` and `rust-ci` are containerized The runner container runs as the non-root `runner` user with default seccomp/AppArmor confinement, no added capabilities, and no `/var/run/docker.sock` mount. The Kotlin runner receives only `/dev/kvm` through the `kvm` supplementary group for the Android emulator. The host should expose that device as `root:kvm` with mode `0660`. -The image must include the build-essential, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, and the Cargo tools required by the workflows. Rebuild and repin the image when those dependencies change. +The image must include build-essential, clang, llvm, libsnappy-dev, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, and the Cargo tools required by the workflows. The shared Rust action verifies clang, llvm, and libsnappy-dev rather than installing them. Rebuild and repin the image when those dependencies change. Jobs that need to publish Docker images run on GitHub-hosted runners. A future self-hosted job that needs Docker must use a separately isolated runner; the `kotlin-ci` and `rust-ci` containers must not regain a host Docker socket. diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 808f86cbe32..22272d62be8 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -118,12 +118,28 @@ runs: ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }} ${{ runner.os }}/cargo/registry/ - - name: Install clang + # Linux self-hosted runners provide the compiler and native libraries in + # the pinned image. Do not install packages or mutate system alternatives + # from a job: the runner is intentionally non-root. + - name: Verify clang and native dependencies id: deps-clang shell: bash if: runner.os == 'Linux' run: | - sudo apt update - # snappy is required by rust rocksdb - sudo apt install -qq --yes clang llvm libsnappy-dev - sudo update-alternatives --set cc /usr/bin/clang + set -euo pipefail + missing=() + for tool in clang llvm-config; do + command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") + done + for pkg in clang llvm libsnappy-dev; do + dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") + done + if [ ${#missing[@]} -gt 0 ]; then + echo "::error::Self-hosted runner image is missing: ${missing[*]}" + echo "::error::Provision these dependencies in the pinned image; Rust CI must not use sudo or apt." + exit 1 + fi + # Use clang for native C/C++ build scripts without changing the + # system-wide cc alternative. + echo "CC=/usr/bin/clang" >> "$GITHUB_ENV" + echo "CXX=/usr/bin/clang++" >> "$GITHUB_ENV" diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index 736321b0ed4..e1c2af37a5c 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -124,10 +124,10 @@ jobs: set -euo pipefail missing=() - for tool in cmake gh jq zip; do + for tool in clang cmake gh jq llvm-config zip; do command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") done - for pkg in build-essential libgmp-dev libssl-dev pkg-config; do + for pkg in build-essential clang libgmp-dev libsnappy-dev libssl-dev llvm pkg-config; do dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") done if [ ${#missing[@]} -gt 0 ]; then From 961e45e26488b72d208e32cb88778f028e0f73fa Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Fri, 11 Sep 2026 22:46:05 +0000 Subject: [PATCH 3/7] ci: pin self-hosted cargo tools --- .github/SELF_HOSTED_RUNNER.md | 4 ++-- .github/workflows/kotlin-sdk-build.yml | 11 +++------- .github/workflows/kotlin-sdk-nightly.yml | 4 ++-- .github/workflows/release-kotlin-sdk.yml | 4 ++-- .github/workflows/tests-rs-wallet.yml | 2 +- .github/workflows/tests-rs-workspace.yml | 27 +++++++----------------- 6 files changed, 18 insertions(+), 34 deletions(-) diff --git a/.github/SELF_HOSTED_RUNNER.md b/.github/SELF_HOSTED_RUNNER.md index 8ac074855c2..555d4bb5f1d 100644 --- a/.github/SELF_HOSTED_RUNNER.md +++ b/.github/SELF_HOSTED_RUNNER.md @@ -4,6 +4,6 @@ The persistent Linux runners used by `kotlin-ci` and `rust-ci` are containerized The runner container runs as the non-root `runner` user with default seccomp/AppArmor confinement, no added capabilities, and no `/var/run/docker.sock` mount. The Kotlin runner receives only `/dev/kvm` through the `kvm` supplementary group for the Android emulator. The host should expose that device as `root:kvm` with mode `0660`. -The image must include build-essential, clang, llvm, libsnappy-dev, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, and the Cargo tools required by the workflows. The shared Rust action verifies clang, llvm, and libsnappy-dev rather than installing them. Rebuild and repin the image when those dependencies change. +The image must include build-essential, clang, llvm, libsnappy-dev, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, cargo-llvm-cov 0.9.1, cargo-nextest 0.9.144, cargo-machete 0.9.2, cargo-ndk 4.1.2, and the other Cargo tools required by the workflows. The shared Rust action verifies clang, llvm, and libsnappy-dev rather than installing them. Rebuild and repin the image when those dependencies change. -Jobs that need to publish Docker images run on GitHub-hosted runners. A future self-hosted job that needs Docker must use a separately isolated runner; the `kotlin-ci` and `rust-ci` containers must not regain a host Docker socket. +The persistent macOS wallet runner follows the same no-suppressed-install rule and must provision cargo-machete 0.9.2 in its image. Jobs that need to publish Docker images, or the nightly hosted Kotlin emulator job, run on ephemeral GitHub-hosted runners; those workflows are an explicit exception and pin their cargo-ndk installation to 4.1.2. A future self-hosted job that needs Docker must use a separately isolated runner; the `kotlin-ci` and `rust-ci` containers must not regain a host Docker socket. diff --git a/.github/workflows/kotlin-sdk-build.yml b/.github/workflows/kotlin-sdk-build.yml index e24021decbb..c0a039683a0 100644 --- a/.github/workflows/kotlin-sdk-build.yml +++ b/.github/workflows/kotlin-sdk-build.yml @@ -105,16 +105,11 @@ jobs: with: targets: x86_64-linux-android - # Pinned: this runner is persistent, so an unpinned `cargo install` - # leaves whatever version happened to be current on the day it first ran, - # and every later job silently builds with it. Assert after installing so - # a drifted host fails here instead of somewhere in the NDK build. - - name: Ensure cargo-ndk v4.1.2 is installed + # Pinned: this runner is persistent, so cargo-ndk is provisioned in the + # image and checked here before the NDK build can start. + - name: Verify cargo-ndk v4.1.2 run: | set -euo pipefail - if ! cargo ndk --version 2>/dev/null | grep -qx 'cargo-ndk 4.1.2'; then - cargo install cargo-ndk --version 4.1.2 --locked --force - fi cargo ndk --version cargo ndk --version | grep -qx 'cargo-ndk 4.1.2' diff --git a/.github/workflows/kotlin-sdk-nightly.yml b/.github/workflows/kotlin-sdk-nightly.yml index c53510ddc11..14fab4520a1 100644 --- a/.github/workflows/kotlin-sdk-nightly.yml +++ b/.github/workflows/kotlin-sdk-nightly.yml @@ -54,8 +54,8 @@ jobs: restore-keys: | kotlin-sdk-cargo- - - name: Install cargo-ndk - run: cargo install cargo-ndk --locked + - name: Install pinned cargo-ndk v4.1.2 (ephemeral runner) + run: cargo install cargo-ndk --version 4.1.2 --locked - name: Install protoc v32.0 (repo-standard; apt's 3.21 breaks tenderdash-proto) run: | diff --git a/.github/workflows/release-kotlin-sdk.yml b/.github/workflows/release-kotlin-sdk.yml index 8286d0b371e..585f0706e62 100644 --- a/.github/workflows/release-kotlin-sdk.yml +++ b/.github/workflows/release-kotlin-sdk.yml @@ -165,8 +165,8 @@ jobs: kotlin-sdk-release-cargo- kotlin-sdk-cargo- - - name: Install cargo-ndk - run: cargo install cargo-ndk --locked + - name: Install pinned cargo-ndk v4.1.2 (ephemeral runner) + run: cargo install cargo-ndk --version 4.1.2 --locked - name: Install protoc v32.0 (repo-standard; apt's 3.21 breaks tenderdash-proto) run: | diff --git a/.github/workflows/tests-rs-wallet.yml b/.github/workflows/tests-rs-wallet.yml index f8e95ea20ac..e52ca7f8037 100644 --- a/.github/workflows/tests-rs-wallet.yml +++ b/.github/workflows/tests-rs-wallet.yml @@ -130,7 +130,7 @@ jobs: - name: Find unused dependencies run: | - cargo install cargo-machete 2>/dev/null || true + cargo machete --version | grep -qx 'cargo-machete 0.9.2' cargo machete - name: Detect immutable structure changes diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index e1c2af37a5c..7683834e3c9 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -150,25 +150,14 @@ jobs: cache: false components: llvm-tools, rustfmt, clippy - # Both tools are required by the test steps below. Install only when - # missing, and fail HERE, loudly, if the tool still doesn't work - # afterwards — the previous `cargo install ... 2>/dev/null || true` - # silently swallowed a failed nextest install on a freshly provisioned - # runner, and the job then died 10 minutes later in the test step with - # "no such command: nextest". - - name: Install cargo-llvm-cov - run: | - if ! cargo llvm-cov --version >/dev/null 2>&1; then - cargo install cargo-llvm-cov --locked - fi - cargo llvm-cov --version + # These helpers are part of the pinned runner image. Version checks make + # image drift fail before the test phase instead of installing tools from + # a job or silently swallowing a failed installation. + - name: Verify cargo-llvm-cov v0.9.1 + run: cargo llvm-cov --version | grep -qx 'cargo-llvm-cov 0.9.1' - - name: Install cargo-nextest - run: | - if ! cargo nextest --version >/dev/null 2>&1; then - cargo install cargo-nextest --locked - fi - cargo nextest --version + - name: Verify cargo-nextest v0.9.144 + run: cargo nextest --version | grep -qx 'cargo-nextest 0.9.144' - name: Check formatting run: cargo fmt --check --all @@ -194,7 +183,7 @@ jobs: - name: Find unused dependencies run: | - cargo install cargo-machete 2>/dev/null || true + cargo machete --version | grep -qx 'cargo-machete 0.9.2' cargo machete # The transport-free cuts are how embedders with their own networking From 47317bfbe89a402644ce3f09f69b49eb480834d9 Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:36:14 +0000 Subject: [PATCH 4/7] ci: consume the reproducible runner image and preserve hosted setup --- .github/SELF_HOSTED_RUNNER.md | 82 +++++++++++- .github/actions/rust/action.yaml | 39 +++++- .github/scripts/kotlin-instrumented-tests.sh | 34 +++++ .github/workflows/kotlin-sdk-build.yml | 129 +++++-------------- .github/workflows/tests-rs-workspace.yml | 11 +- 5 files changed, 179 insertions(+), 116 deletions(-) create mode 100644 .github/scripts/kotlin-instrumented-tests.sh diff --git a/.github/SELF_HOSTED_RUNNER.md b/.github/SELF_HOSTED_RUNNER.md index 555d4bb5f1d..e5dc52e79e9 100644 --- a/.github/SELF_HOSTED_RUNNER.md +++ b/.github/SELF_HOSTED_RUNNER.md @@ -1,9 +1,81 @@ -# Linux self-hosted runner contract +# Self-hosted runner contract -The persistent Linux runners used by `kotlin-ci` and `rust-ci` are containerized for reproducible toolchains. The runner image is the versioned environment; jobs must not install host packages or modify host device permissions. +## Reproducible Linux image -The runner container runs as the non-root `runner` user with default seccomp/AppArmor confinement, no added capabilities, and no `/var/run/docker.sock` mount. The Kotlin runner receives only `/dev/kvm` through the `kvm` supplementary group for the Android emulator. The host should expose that device as `root:kvm` with mode `0660`. +Source, locked dependencies, deployment examples and publishing workflow: +**[dashpay/dash-selfhosted-image](https://github.com/dashpay/dash-selfhosted-image)**. +Use its `linux/amd64` contract-1 image for persistent Linux `kotlin-ci` / `rust-ci` +runners. The shared Rust action requires `/opt/ci/contract-version` to be `1` on +self-hosted Linux and fails early if an old/native runner picks up the job. -The image must include build-essential, clang, llvm, libsnappy-dev, CMake, GMP/OpenSSL/PkgConfig libraries, GitHub CLI, protoc 32.0, JDK 17, Android SDK/NDK, Rust, cargo-llvm-cov 0.9.1, cargo-nextest 0.9.144, cargo-machete 0.9.2, cargo-ndk 4.1.2, and the other Cargo tools required by the workflows. The shared Rust action verifies clang, llvm, and libsnappy-dev rather than installing them. Rebuild and repin the image when those dependencies change. +The image locks Ubuntu 24.04 by digest, apt to a signed archive snapshot, and +downloaded toolchains to exact URLs and SHA-256 hashes. It includes: -The persistent macOS wallet runner follows the same no-suppressed-install rule and must provision cargo-machete 0.9.2 in its image. Jobs that need to publish Docker images, or the nightly hosted Kotlin emulator job, run on ephemeral GitHub-hosted runners; those workflows are an explicit exception and pin their cargo-ndk installation to 4.1.2. A future self-hosted job that needs Docker must use a separately isolated runner; the `kotlin-ci` and `rust-ci` containers must not regain a host Docker socket. +| Toolchain | Contract | +| --- | --- | +| Native build | build-essential, clang/LLVM, Snappy, CMake, GMP, OpenSSL, pkg-config | +| Rust | rustup plus the repository baseline; exact repo-selected toolchains may be installed user-locally | +| Cargo helpers | llvm-cov **0.9.1**, nextest **0.9.144**, machete **0.9.2**, ndk **4.1.2** | +| Protobuf / Java | protoc **32.0**, JDK **17** | +| Android | API **35**, build-tools **35.0.0**, NDK **28.1.13356709**, pinned emulator/system image | +| Other job tools | Git, GitHub CLI, jq, Python 3, gpg, zip/unzip | + +The SDK is image-owned. The persistent Kotlin workflow uses the image's +`ci-android-emulator` wrapper to create user-writable AVDs, boot with KVM, and stop +the emulator after testing. It does **not** run setup-android, sdkmanager, or an +emulator action that upgrades image packages at job runtime. Lockscreen/PIN and +unlocked-device checks remain in `.github/scripts/kotlin-instrumented-tests.sh`. + +## Runtime privileges + +- Non-root uid/gid **1001:1001**, no sudo, no Docker CLI or host Docker socket. +- Drop **all** capabilities; enable **no-new-privileges**. Keep default Docker + seccomp/AppArmor policies, with no privileged mode or host namespaces. +- Dedicated registration/work volumes only. Kotlin adds **`/dev/kvm:rw`** and its + numeric host group; Rust-only runners do not need that device. +- The operator configures `/dev/kvm` as `root:kvm`, mode **0660**. Jobs only check + access; they never modify host udev rules, permissions or system packages. +- Preserve runner-group selected-repository access and the existing fork guards. + Persistent job data is not isolation between mutually untrusted repositories. + +Building/publishing the image uses Docker on an ephemeral GitHub-hosted builder; +that privilege is not passed into the resulting persistent runner. + +## Publish, prove, then roll out + +1. Use a successful [image publishing run](https://github.com/dashpay/dash-selfhosted-image/actions/workflows/image.yml). + Publication requires non-root compiler/confinement checks, `KVM_CREATE_VM`, and + a real API 35 emulator boot. Retrieve `image-reference.txt` from the run. +2. Set `RUNNER_IMAGE=dashpay/dash-selfhosted-image@sha256:` in + the operator's deployment. Do not use a floating image or a locally inherited + `github-runner-runner:latest` parent. The image repo's Compose files enforce the + runtime boundary above; the KVM overlay is optional. +3. Drain the old runner before migration. Register a new name in the **existing + group**, retaining its selected repositories, with only the required labels. + Use a short-lived registration-token file, not a PAT stored in Compose. +4. Prove a real Rust job and Kotlin job on that exact runner/digest before retiring + the old instance. Keep the previous registration/image for rollback. Rebuilding + or pushing source does not replace any live runner automatically. + +Deploy and prove the contract-1 image **before merging the consuming workflows**. +Record the selected digest and real-job evidence with the deployment; do not infer +runtime health from YAML validation or the image tag alone. Rebuild/repin when +dependencies change, including runner updates required by GitHub's update policy. + +## Hosted Linux and native macOS remain distinct + +The shared Rust action branches on `runner.environment`: persistent Linux verifies +the image's native libraries and protoc, while GitHub-hosted release/nightly/book/ +JavaScript consumers retain apt provisioning and the user-local protoc cache. +Both select clang through `CC`/`CXX`, without mutating system alternatives. + +The Linux image does not provision macOS. Native macOS `rust-ci` runners still +need the existing Homebrew dependencies plus llvm-cov 0.9.1, nextest 0.9.144 and +machete 0.9.2; the wallet fast path needs machete 0.9.2. Provision and verify these +separately before rollout. Do not silently install tools or swallow failures in +persistent jobs. + +Docker publication and hosted Kotlin release/nightly jobs remain on ephemeral +GitHub-hosted runners. The latter explicitly install cargo-ndk 4.1.2. Any future +self-hosted job that genuinely needs Docker must use separately isolated capacity; +the persistent Rust/Kotlin runner must not regain the host Docker socket. diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 22272d62be8..47c07792f92 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -21,6 +21,16 @@ inputs: runs: using: composite steps: + - name: Verify persistent Linux runner image contract + if: runner.os == 'Linux' && runner.environment == 'self-hosted' + shell: bash + run: | + if [ "$(cat /opt/ci/contract-version 2>/dev/null)" != 1 ]; then + echo '::error::This runner needs the versioned dashpay/dash-selfhosted-image (contract 1); see .github/SELF_HOSTED_RUNNER.md.' + exit 1 + fi + command -v rustup + - name: Resolve HOME path for caching id: resolved_home shell: bash @@ -47,7 +57,7 @@ runs: components: ${{ inputs.components }} - name: Get protoc arch - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.environment == 'github-hosted' shell: bash id: protoc_arch run: | @@ -67,7 +77,7 @@ runs: esac - name: Restore cached protoc (v32.0) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.environment == 'github-hosted' id: cache-protoc uses: actions/cache@v5 with: @@ -77,7 +87,7 @@ runs: key: protoc/32.0/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} - name: Install protoc (cached v32.0) - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.environment == 'github-hosted' id: deps-protoc shell: bash run: | @@ -93,7 +103,7 @@ runs: echo "PROTOC=${PROTOC_DIR}/bin/protoc" >> "$GITHUB_ENV" - name: Save cached protoc (v32.0) - if: runner.os == 'Linux' && steps.cache-protoc.outputs.cache-hit != 'true' + if: runner.os == 'Linux' && runner.environment == 'github-hosted' && steps.cache-protoc.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: path: | @@ -101,6 +111,14 @@ runs: ${{ steps.resolved_home.outputs.home }}/.local/protoc-32.0/include key: protoc/32.0/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} + - name: Verify prebaked protoc (v32.0) + if: runner.os == 'Linux' && runner.environment == 'self-hosted' + shell: bash + run: | + set -euo pipefail + protoc --version | grep -qx 'libprotoc 32.0' + echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" + - name: Set HOME variable to github context shell: bash run: echo "HOME=$HOME" >> $GITHUB_ENV @@ -118,6 +136,15 @@ runs: ${{ runner.os }}/cargo/registry/${{ hashFiles('**/Cargo.lock') }} ${{ runner.os }}/cargo/registry/ + # This composite is also used by hosted release, nightly and book jobs. + # Keep their bootstrap path; only persistent runners require a prebaked image. + - name: Install native dependencies on ephemeral GitHub-hosted Linux + if: runner.os == 'Linux' && runner.environment == 'github-hosted' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends clang llvm libsnappy-dev + # Linux self-hosted runners provide the compiler and native libraries in # the pinned image. Do not install packages or mutate system alternatives # from a job: the runner is intentionally non-root. @@ -135,8 +162,8 @@ runs: dpkg -s "$pkg" >/dev/null 2>&1 || missing+=("$pkg") done if [ ${#missing[@]} -gt 0 ]; then - echo "::error::Self-hosted runner image is missing: ${missing[*]}" - echo "::error::Provision these dependencies in the pinned image; Rust CI must not use sudo or apt." + echo "::error::Linux runner is missing: ${missing[*]}" + echo "::error::Persistent runners must provision these dependencies in the pinned image; hosted runners use the install step above." exit 1 fi # Use clang for native C/C++ build scripts without changing the diff --git a/.github/scripts/kotlin-instrumented-tests.sh b/.github/scripts/kotlin-instrumented-tests.sh new file mode 100644 index 00000000000..586d43ab8c5 --- /dev/null +++ b/.github/scripts/kotlin-instrumented-tests.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Called from packages/kotlin-sdk by the image's ci-android-emulator wrapper. +set -euo pipefail + +# Keystore's unlocked-device-required keys fail if the screen re-locks mid-test. +adb shell settings put system screen_off_timeout 2147483647 +adb shell svc power stayon true + +# Auth-required identity keys need an enrolled secure lockscreen on the test AVD. +adb shell locksettings set-pin 1234 + +# Cold boot can race credential acceptance and keyguard dismissal. Preserve the +# upstream retry and trust-state gate, now in one shell (not line-by-line sh -c). +unlocked=false +for attempt in 1 2 3; do + adb shell input keyevent KEYCODE_WAKEUP + adb shell wm dismiss-keyguard + adb shell input text 1234 + adb shell input keyevent KEYCODE_ENTER + sleep 2 + adb shell wm dismiss-keyguard + sleep 1 + if adb shell dumpsys trust | grep -q 'deviceLocked=0'; then + unlocked=true + break + fi +done +if [ "$unlocked" != true ]; then + echo '::error::Emulator is still locked; Keystore-backed tests would fail spuriously.' + adb shell dumpsys trust + exit 1 +fi + +./gradlew :sdk:connectedDebugAndroidTest --stacktrace diff --git a/.github/workflows/kotlin-sdk-build.yml b/.github/workflows/kotlin-sdk-build.yml index 64fe7e36ed9..cf2034ba0b7 100644 --- a/.github/workflows/kotlin-sdk-build.yml +++ b/.github/workflows/kotlin-sdk-build.yml @@ -23,6 +23,8 @@ on: - 'scripts/check_sdk_parity_manifest.py' - 'scripts/tests/**' - '.github/workflows/kotlin-sdk-build.yml' + - '.github/scripts/kotlin-instrumented-tests.sh' + - '.github/actions/rust/**' permissions: contents: read @@ -59,7 +61,7 @@ jobs: set -euo pipefail missing=() - for tool in cmake curl jq python3 unzip zip; do + for tool in cmake curl jq python3 rustup unzip zip; do command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") done for pkg in build-essential libgmp-dev libpulse0 libssl-dev libx11-xcb1 pkg-config; do @@ -71,12 +73,6 @@ jobs: exit 1 fi - if [ ! -x "$HOME/.cargo/bin/rustup" ]; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --default-toolchain none - fi - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - - name: Validate executable SDK parity manifest run: | python3 scripts/check_sdk_parity_manifest.py @@ -91,19 +87,23 @@ jobs: echo "JAVA_HOME=$JAVA_HOME_RESOLVED" >> "$GITHUB_ENV" echo "$JAVA_HOME_RESOLVED/bin" >> "$GITHUB_PATH" - - name: Set up Android SDK - uses: android-actions/setup-android@v3 - with: - packages: >- - platforms;android-35 - build-tools;35.0.0 - ndk;28.1.13356709 - platform-tools - - - name: Set up Rust toolchain - uses: dtolnay/rust-toolchain@stable + - name: Verify prebaked Android SDK + run: | + set -euo pipefail + test "${ANDROID_HOME:-}" = /opt/android-sdk + test -x "$ANDROID_HOME/ndk/28.1.13356709/ndk-build" + test -x "$ANDROID_HOME/build-tools/35.0.0/aapt2" + test -f "$ANDROID_HOME/platforms/android-35/android.jar" + test -f "$ANDROID_HOME/system-images/android-35/default/x86_64/system.img" + command -v ci-android-emulator + command -v adb + # Do not run setup-android/sdkmanager: the SDK is pinned and root-owned. + + - name: Set up repository-pinned Rust toolchain + uses: ./.github/actions/rust with: - targets: x86_64-linux-android + target: x86_64-linux-android + cache: false # Pinned: this runner is persistent, so cargo-ndk is provisioned in the # image and checked here before the NDK build can start. @@ -121,10 +121,18 @@ jobs: # from a job required sudo and made the CI container mutable. protoc --version | grep -qx 'libprotoc 32.0' + # KVM is the only host device this job needs. The container receives it + # through the kvm group; no Docker socket or host package-management + # access is required. + - name: Verify KVM access + run: | + test -c /dev/kvm + test -r /dev/kvm + test -w /dev/kvm + ls -l /dev/kvm + - name: Build native library (x86_64, dev profile) working-directory: packages/kotlin-sdk - env: - ANDROID_NDK_HOME: ${{ env.ANDROID_SDK_ROOT }}/ndk/28.1.13356709 run: ./build_android.sh --abi x86_64 --profile dev --verify - name: Setup Gradle @@ -142,82 +150,9 @@ jobs: working-directory: packages/kotlin-sdk run: ./gradlew :sdk:compileDebugAndroidTestKotlin --stacktrace - # KVM is the only host device this job needs. The container receives it - # through the kvm group; no Docker socket or host package-management - # access is required. - - name: Verify KVM access - run: | - test -c /dev/kvm - test -r /dev/kvm - test -w /dev/kvm - ls -l /dev/kvm - - - name: Align Android emulator configuration paths - run: | - # android-emulator-runner puts AVD data in $HOME/.android/avd. - # An inherited user/emulator-home override can make avdmanager - # write the .ini elsewhere, leaving the emulator unable to find it. - mkdir -p "$HOME/.android/avd" - { - printf 'ANDROID_USER_HOME=%s/.android\n' "$HOME" - printf 'ANDROID_EMULATOR_HOME=%s/.android\n' "$HOME" - printf 'ANDROID_SDK_HOME=%s\n' "$HOME" - } >> "$GITHUB_ENV" - - - name: Run instrumented FFI smoke test (API 35 emulator) - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 35 - arch: x86_64 - profile: pixel_6 - # Avoid sharing the default "test" AVD with other jobs on this host. - avd-name: platform-${{ github.run_id }}-${{ github.run_attempt }} - disable-animations: true - emulator-options: -no-snapshot -no-window -no-audio -no-boot-anim -camera-back none -dns-server 8.8.8.8,1.1.1.1 - working-directory: packages/kotlin-sdk - script: | - # NOTE: android-emulator-runner executes this script line by line — - # each line is its own `sh -c` with no shared state — so every - # command must be self-contained on a single line (multi-line loops - # or cross-line variables silently fail). - - # Keep the screen on and never time out. THIS is the fix: the flake - # was the emulator screen turning off mid-run and RE-LOCKING the - # device. The mnemonic MASTER_ALIAS AES key is - # setUnlockedDeviceRequired(true), so once the device re-locks even - # an ENCRYPT throws `InvalidKeyException: Keystore operation failed` - # (see WalletStorage.storeMnemonic). Unlocking alone is not enough — - # a slower suite (more instrumented tests) crosses the screen-off - # deadline before its wallet tests run, which is why this branch - # failed where lighter ones passed. Prevent the re-lock outright. - adb shell settings put system screen_off_timeout 2147483647 - adb shell svc power stayon true - - # Enroll a device-wide secure lock screen (PIN) — Android Keystore - # refuses to generate an auth-required key - # (setUserAuthenticationRequired(true), used for the identity-key - # KEYS_ALIAS RSA pair) without one enrolled, even though nothing - # here ever prompts for it (private-key ENCRYPT is never auth-gated; - # only DECRYPT is). - adb shell locksettings set-pin 1234 - - # Unlock the keyguard. With the screen kept on above, the device now - # stays unlocked for the whole run instead of re-locking. - # Credential acceptance and keyguard dismissal can race during a - # cold emulator boot. Retry the complete sequence atomically, then - # fail loudly before tests if the device never reaches unlocked. - for attempt in 1 2 3; do adb shell input keyevent KEYCODE_WAKEUP; adb shell wm dismiss-keyguard; adb shell input text 1234; adb shell input keyevent KEYCODE_ENTER; sleep 2; adb shell wm dismiss-keyguard; sleep 1; adb shell dumpsys trust | grep -q 'deviceLocked=0' && exit 0; done; echo "::error::Emulator is still locked (deviceLocked=1); Keystore-backed tests would fail spuriously."; adb shell dumpsys trust; exit 1 - - ./gradlew :sdk:connectedDebugAndroidTest --stacktrace - - - name: Remove this run's emulator device - if: always() - env: - CI_AVD_NAME: platform-${{ github.run_id }}-${{ github.run_attempt }} - run: | - if [ -f "$HOME/.android/avd/$CI_AVD_NAME.ini" ]; then - avdmanager delete avd --name "$CI_AVD_NAME" - fi + - name: Run instrumented FFI smoke test (prebaked API 35 emulator) + working-directory: packages/kotlin-sdk + run: ci-android-emulator bash ../../.github/scripts/kotlin-instrumented-tests.sh - name: Upload test reports on failure if: failure() diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index 568dbb12ef8..a7599fddca8 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -124,7 +124,7 @@ jobs: set -euo pipefail missing=() - for tool in clang cmake gh jq llvm-config zip; do + for tool in clang cmake gh jq llvm-config rustup zip; do command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") done for pkg in build-essential clang libgmp-dev libsnappy-dev libssl-dev llvm pkg-config; do @@ -136,13 +136,8 @@ jobs: exit 1 fi - # dtolnay/rust-toolchain drives rustup; it may install the toolchain - # under the runner user's home, but never needs root. - if ! command -v rustup >/dev/null 2>&1; then - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs \ - | sh -s -- -y --no-modify-path --default-toolchain none - echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" - fi + # rustup is prebaked; Setup Rust may install the repository-selected + # toolchain under the runner user's home, without root. - name: Setup Rust uses: ./.github/actions/rust From d5a1d30aa82c440963a2f034aac1fbcc113387bf Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:41:18 +0000 Subject: [PATCH 5/7] fix: check pinned cargo versions using their actual CLI output --- .github/workflows/tests-rs-wallet.yml | 2 +- .github/workflows/tests-rs-workspace.yml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests-rs-wallet.yml b/.github/workflows/tests-rs-wallet.yml index e52ca7f8037..ec9d2e5a62d 100644 --- a/.github/workflows/tests-rs-wallet.yml +++ b/.github/workflows/tests-rs-wallet.yml @@ -130,7 +130,7 @@ jobs: - name: Find unused dependencies run: | - cargo machete --version | grep -qx 'cargo-machete 0.9.2' + cargo machete --version | grep -Eq '^(cargo-machete )?0[.]9[.]2$' cargo machete - name: Detect immutable structure changes diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index a7599fddca8..61f87fd7d6f 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -152,7 +152,8 @@ jobs: run: cargo llvm-cov --version | grep -qx 'cargo-llvm-cov 0.9.1' - name: Verify cargo-nextest v0.9.144 - run: cargo nextest --version | grep -qx 'cargo-nextest 0.9.144' + # Release binaries include commit/host metadata after the version. + run: cargo nextest --version | sed -n '1p' | grep -Eq '^cargo-nextest 0[.]9[.]144( |$)' - name: Check formatting run: cargo fmt --check --all @@ -178,7 +179,7 @@ jobs: - name: Find unused dependencies run: | - cargo machete --version | grep -qx 'cargo-machete 0.9.2' + cargo machete --version | grep -Eq '^(cargo-machete )?0[.]9[.]2$' cargo machete # The transport-free cuts are how embedders with their own networking From c39431b01569b2b4eb999e7373e2cad63d05ca7c Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Tue, 22 Sep 2026 14:50:02 +0000 Subject: [PATCH 6/7] ci: test requirement changes on PR-specific runner images --- .github/SELF_HOSTED_RUNNER.md | 46 +++++ .github/actions/rust/action.yaml | 31 ++-- .github/runner-requirements.json | 206 +++++++++++++++++++++ .github/scripts/runner-image.py | 152 +++++++++++++++ .github/scripts/tests/test_runner_image.py | 97 ++++++++++ .github/workflows/kotlin-sdk-build.yml | 55 ++++-- .github/workflows/tests-rs-wallet.yml | 2 +- .github/workflows/tests-rs-workspace.yml | 38 +++- .github/workflows/tests.yml | 10 +- 9 files changed, 604 insertions(+), 33 deletions(-) create mode 100644 .github/runner-requirements.json create mode 100644 .github/scripts/runner-image.py create mode 100644 .github/scripts/tests/test_runner_image.py diff --git a/.github/SELF_HOSTED_RUNNER.md b/.github/SELF_HOSTED_RUNNER.md index e5dc52e79e9..5367bc87ebe 100644 --- a/.github/SELF_HOSTED_RUNNER.md +++ b/.github/SELF_HOSTED_RUNNER.md @@ -8,6 +8,13 @@ Use its `linux/amd64` contract-1 image for persistent Linux `kotlin-ci` / `rust- runners. The shared Rust action requires `/opt/ci/contract-version` to be `1` on self-hosted Linux and fails early if an old/native runner picks up the job. +Platform's desired versions and checksums now live in +[.github/runner-requirements.json](runner-requirements.json). This includes an immutable image-recipe +commit. Persistent Linux jobs verify **both** the installed lock and recipe +revision; a contract-1 marker by itself is not sufficient. The earlier published +bootstrap image must be replaced by a matching candidate before these changes +can be merged. + The image locks Ubuntu 24.04 by digest, apt to a signed archive snapshot, and downloaded toolchains to exact URLs and SHA-256 hashes. It includes: @@ -43,6 +50,10 @@ that privilege is not passed into the resulting persistent runner. ## Publish, prove, then roll out +For Platform requirements changes, use the PR-first lifecycle below. The +standalone image publishing workflow remains useful for recipe development, +but its default lock is not a separate source of Platform requirements. + 1. Use a successful [image publishing run](https://github.com/dashpay/dash-selfhosted-image/actions/workflows/image.yml). Publication requires non-root compiler/confinement checks, `KVM_CREATE_VM`, and a real API 35 emulator boot. Retrieve `image-reference.txt` from the run. @@ -62,6 +73,41 @@ Record the selected digest and real-job evidence with the deployment; do not inf runtime health from YAML validation or the image tag alone. Rebuild/repin when dependencies change, including runner updates required by GitHub's update policy. +## Requirements changes: candidate before merge, promotion after + +1. Change .github/runner-requirements.json in the Platform PR, including exact download URLs, + checksums and package metadata. A change to this file is the automatic build + flag; no separate label is required. Change the pinned recipe commit only + when recipe/image code changes. Ordinary user-local Rust toolchain updates + still follow rust-toolchain.toml. +2. The trusted base-branch publisher builds and smoke-tests a candidate on a + disposable VM. A separate VM publishes it without executing PR image/code + with Docker Hub credentials. +3. The normal Rust and Kotlin workflows wait for the candidate, then request + temporary runners labelled for **this PR head, exact digest and job kind**. + A host-side controller creates one-job non-root containers, with KVM only + for Kotlin. Real application jobs must pass; skipped fork jobs do not count. +4. After merge, the publisher verifies the merged/current requirements and both + real candidate jobs, then promotes **the same tested digest**, without a + rebuild. Each base branch gets a platform- channel; main advances only + for Platform's actual GitHub default branch. +5. Promotion does not restart production runners. The operator drains and + switches ordinary runner capacity to the reviewed digest using the rollout + procedure above. Requirements checks fail explicitly until capacity matches. + +The trusted caller must land separately before a PR can use this flow. +See the image repository's +[bootstrap, GitHub App and candidate-controller setup](https://github.com/dashpay/dash-selfhosted-image/blob/feat/platform-pr-images/docs/platform-pr-images.md). +No App key, Docker socket, registry credential or host workspace enters a job. +The existing trusted-fork restrictions are unchanged; the controller's +exact-head approvals do not override workflow-side guards. + +Run the routing checks with: + +~~~sh +python3 -m unittest discover -s .github/scripts/tests -v +~~~ + ## Hosted Linux and native macOS remain distinct The shared Rust action branches on `runner.environment`: persistent Linux verifies diff --git a/.github/actions/rust/action.yaml b/.github/actions/rust/action.yaml index 47c07792f92..b5b0f696360 100644 --- a/.github/actions/rust/action.yaml +++ b/.github/actions/rust/action.yaml @@ -21,6 +21,10 @@ inputs: runs: using: composite steps: + - name: Read shared runner tool requirements + shell: bash + run: python3 .github/scripts/runner-image.py env + - name: Verify persistent Linux runner image contract if: runner.os == 'Linux' && runner.environment == 'self-hosted' shell: bash @@ -29,6 +33,7 @@ runs: echo '::error::This runner needs the versioned dashpay/dash-selfhosted-image (contract 1); see .github/SELF_HOSTED_RUNNER.md.' exit 1 fi + python3 .github/scripts/runner-image.py verify command -v rustup - name: Resolve HOME path for caching @@ -76,47 +81,47 @@ runs: ;; esac - - name: Restore cached protoc (v32.0) + - name: Restore cached repository-pinned protoc if: runner.os == 'Linux' && runner.environment == 'github-hosted' id: cache-protoc uses: actions/cache@v5 with: path: | - ${{ steps.resolved_home.outputs.home }}/.local/protoc-32.0/bin - ${{ steps.resolved_home.outputs.home }}/.local/protoc-32.0/include - key: protoc/32.0/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} + ${{ steps.resolved_home.outputs.home }}/.local/protoc-${{ env.CI_PROTOC_VERSION }}/bin + ${{ steps.resolved_home.outputs.home }}/.local/protoc-${{ env.CI_PROTOC_VERSION }}/include + key: protoc/${{ env.CI_PROTOC_VERSION }}/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} - - name: Install protoc (cached v32.0) + - name: Install repository-pinned protoc if: runner.os == 'Linux' && runner.environment == 'github-hosted' id: deps-protoc shell: bash run: | set -euxo pipefail - PROTOC_DIR="${HOME}/.local/protoc-32.0" + PROTOC_DIR="${HOME}/.local/protoc-${{ env.CI_PROTOC_VERSION }}" if [ ! -x "${PROTOC_DIR}/bin/protoc" ]; then mkdir -p "${PROTOC_DIR}" curl -fsSL -o /tmp/protoc.zip \ - "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-${{ steps.protoc_arch.outputs.arch }}.zip" + "https://github.com/protocolbuffers/protobuf/releases/download/v${CI_PROTOC_VERSION}/protoc-${{ env.CI_PROTOC_VERSION }}-linux-${{ steps.protoc_arch.outputs.arch }}.zip" unzip -o /tmp/protoc.zip -d "${PROTOC_DIR}" fi echo "${PROTOC_DIR}/bin" >> "$GITHUB_PATH" echo "PROTOC=${PROTOC_DIR}/bin/protoc" >> "$GITHUB_ENV" - - name: Save cached protoc (v32.0) + - name: Save cached repository-pinned protoc if: runner.os == 'Linux' && runner.environment == 'github-hosted' && steps.cache-protoc.outputs.cache-hit != 'true' uses: actions/cache/save@v5 with: path: | - ${{ steps.resolved_home.outputs.home }}/.local/protoc-32.0/bin - ${{ steps.resolved_home.outputs.home }}/.local/protoc-32.0/include - key: protoc/32.0/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} + ${{ steps.resolved_home.outputs.home }}/.local/protoc-${{ env.CI_PROTOC_VERSION }}/bin + ${{ steps.resolved_home.outputs.home }}/.local/protoc-${{ env.CI_PROTOC_VERSION }}/include + key: protoc/${{ env.CI_PROTOC_VERSION }}/${{ runner.os }}/${{ steps.protoc_arch.outputs.arch }} - - name: Verify prebaked protoc (v32.0) + - name: Verify prebaked repository-pinned protoc if: runner.os == 'Linux' && runner.environment == 'self-hosted' shell: bash run: | set -euo pipefail - protoc --version | grep -qx 'libprotoc 32.0' + protoc --version | grep -Fx "libprotoc $CI_PROTOC_VERSION" echo "PROTOC=$(command -v protoc)" >> "$GITHUB_ENV" - name: Set HOME variable to github context diff --git a/.github/runner-requirements.json b/.github/runner-requirements.json new file mode 100644 index 00000000000..352fb808d33 --- /dev/null +++ b/.github/runner-requirements.json @@ -0,0 +1,206 @@ +{ + "schema": 1, + "recipe_revision": "baf8849b900555d66714e0e1fcffdff669b9e404", + "requirements": { + "schema": 2, + "contract_version": "1", + "platform": "linux/amd64", + "ubuntu_image": "ubuntu:24.04@sha256:496754492fb28b4d3049432f2ca787449331e23fb14f0dd3fffea86bf5a93eb4", + "apt_snapshot": "20260920T000000Z", + "rust_version": "1.98.1", + "rust_manifest_sha256": "a7c8774a5fd8441c997d94c029776cbc5eb111e9d72ab5d256fa69866644347e", + "artifacts": [ + { + "name": "runner", + "url": "https://github.com/actions/runner/releases/download/v2.337.0/actions-runner-linux-x64-2.337.0.tar.gz", + "sha256": "70920811a4f8ad4328818682bca5c6469c1c942fab52448868071d0063816613", + "format": "tar", + "destination": "/opt/actions-runner" + }, + { + "name": "cargo-llvm-cov", + "url": "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.9.1/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "b3f68e625481fed9b16444174f3fa5ebcdbde4a1878803a35eabe2dcefcdc41a", + "format": "tar", + "destination": "/opt/ci/bin/cargo-llvm-cov", + "binary": "cargo-llvm-cov" + }, + { + "name": "cargo-nextest", + "url": "https://github.com/nextest-rs/nextest/releases/download/cargo-nextest-0.9.144/cargo-nextest-0.9.144-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "8a4f726272b0a1c499bd87ca3978bfbb1a8c20bb08ccf075b9996e2081bd1e1e", + "format": "tar", + "destination": "/opt/ci/bin/cargo-nextest", + "binary": "cargo-nextest" + }, + { + "name": "cargo-machete", + "url": "https://github.com/bnjbvr/cargo-machete/releases/download/v0.9.2/cargo-machete-v0.9.2-x86_64-unknown-linux-musl.tar.gz", + "sha256": "48200087f54c55aabcd4db4af1e25742b49846c02a1b1bfa134711945b35b2e9", + "format": "tar", + "destination": "/opt/ci/bin/cargo-machete", + "binary": "cargo-machete" + }, + { + "name": "cargo-ndk", + "url": "https://github.com/bbqsrc/cargo-ndk/releases/download/v4.1.2/cargo-ndk-x86_64-unknown-linux-gnu-v4.1.2.tgz", + "sha256": "9451622c4567e8abb2c8005001855e32901c0b716ccdd3a173a4375fd03426e1", + "format": "tar", + "destination": "/opt/ci/bin/cargo-ndk", + "binary": "cargo-ndk" + }, + { + "name": "protoc", + "url": "https://github.com/protocolbuffers/protobuf/releases/download/v32.0/protoc-32.0-linux-x86_64.zip", + "sha256": "7ca037bfe5e5cabd4255ccd21dd265f79eb82d3c010117994f5dc81d2140ee88", + "format": "zip", + "destination": "/opt/protoc" + }, + { + "name": "rustup-init", + "url": "https://static.rust-lang.org/rustup/archive/1.28.2/x86_64-unknown-linux-gnu/rustup-init", + "sha256": "20a06e644b0d9bd2fbdbfd52d42540bdde820ea7df86e92e533c073da0cdd43c", + "format": "file", + "destination": "/opt/ci/rustup-init", + "build_only": true + }, + { + "name": "platforms;android-35", + "url": "https://dl.google.com/android/repository/platform-35_r02.zip", + "upstream_sha1": "0bb560a90a7a2cbd0dd8348224d518b638fe7949", + "format": "zip", + "destination": "/opt/android-sdk/platforms/android-35", + "archive_root": "android-35", + "package_xml": "\n \n 35\n 13\n true\n \n \n \n 2\n \n Android SDK Platform 35\n \n ", + "sha256": "0988cacad01b38a18a47bac14a0695f246bc76c1b06c0eeb8eb0dc825ab0c8e0" + }, + { + "name": "ndk;28.1.13356709", + "url": "https://dl.google.com/android/repository/android-ndk-r28b-linux.zip", + "upstream_sha1": "f574d3165405bd59ffc5edaadac02689075a729f", + "format": "zip", + "destination": "/opt/android-sdk/ndk/28.1.13356709", + "archive_root": "android-ndk-r28b", + "package_xml": "\n \n \n 28\n 1\n 13356709\n \n NDK (Side by side) 28.1.13356709\n \n ", + "sha256": "e9f2759862cecfd48c20bbb7d8cfedbb020f4d91b5f78d9a2fc106f7db3c27ed" + }, + { + "name": "build-tools;35.0.0", + "url": "https://dl.google.com/android/repository/build-tools_r35_linux.zip", + "upstream_sha1": "2cfaa0bbb2336e9ec18ed3ecea84fa2e2af607bc", + "format": "zip", + "destination": "/opt/android-sdk/build-tools/35.0.0", + "archive_root": "android-15", + "package_xml": "\n \n \n 35\n 0\n 0\n \n Android SDK Build-Tools 35\n \n ", + "sha256": "bd3a4966912eb8b30ed0d00b0cda6b6543b949d5ffe00bea54c04c81e1561d88" + }, + { + "name": "cmdline-tools;19.0", + "url": "https://dl.google.com/android/repository/commandlinetools-linux-13114758_latest.zip", + "upstream_sha1": "5fdcc763663eefb86a5b8879697aa6088b041e70", + "format": "zip", + "destination": "/opt/android-sdk/cmdline-tools/19.0", + "archive_root": "cmdline-tools", + "package_xml": "\n \n \n 19\n 0\n \n Android SDK Command-line Tools\n \n ", + "sha256": "7ec965280a073311c339e571cd5de778b9975026cfcbe79f2b1cdcb1e15317ee" + }, + { + "name": "platform-tools", + "url": "https://dl.google.com/android/repository/platform-tools_r37.0.1-linux.zip", + "upstream_sha1": "477254aa5f903c15cf51001717bdf347fb6b53e0", + "format": "zip", + "destination": "/opt/android-sdk/platform-tools", + "archive_root": "platform-tools", + "package_xml": "\n \n \n 37\n 0\n 1\n \n Android SDK Platform-Tools\n \n ", + "sha256": "d230f13842f60f782a8645f9c813f8f845bf36089ea7289f28c48f17979313f1" + }, + { + "name": "emulator", + "url": "https://dl.google.com/android/repository/emulator-linux_x64-15917651.zip", + "upstream_sha1": "1b1f78891abf8ec268264356e1365c25519e8379", + "format": "zip", + "destination": "/opt/android-sdk/emulator", + "archive_root": "emulator", + "package_xml": "\n \n \n 37\n 1\n 11\n \n Android Emulator\n \n ", + "sha256": "95771e0ae431897b2a4bd2d97fa095f29a8b0624a7b216baf529f9306161c266" + }, + { + "name": "system-images;android-35;default;x86_64", + "url": "https://dl.google.com/android/repository/sys-img/android/x86_64-35_r02.zip", + "upstream_sha1": "2d857d170c0d1b827149565da34b3383e5306f7f", + "format": "zip", + "destination": "/opt/android-sdk/system-images/android-35/default/x86_64", + "archive_root": "x86_64", + "package_xml": "\n \n 35\n 13\n true\n \n default\n Default Android System Image\n \n x86_64\n \n \n 2\n \n Intel x86_64 Atom System Image\n \n \n \n 29\n 1\n 11\n \n \n \n \n ", + "sha256": "6dd7de33e63ef105cf2fabea6badda1dbe7665c96d8908e5f6e1407e63ff4556" + } + ], + "bootstrap_ca": { + "url": "https://snapshot.ubuntu.com/ubuntu/20260920T000000Z/pool/main/c/ca-certificates/ca-certificates_20240203_all.deb", + "sha256": "641de77d8f142cfd62a1a6f964ba67b20754d3337c480efb529d086075a06c9a", + "version": "20240203" + }, + "apt_packages": [ + "ca-certificates", + "curl", + "git", + "gh", + "jq", + "python3", + "unzip", + "zip", + "xz-utils", + "bzip2", + "gnupg", + "build-essential", + "clang", + "llvm", + "libsnappy-dev", + "cmake", + "libgmp-dev", + "libssl-dev", + "pkg-config", + "openjdk-17-jdk-headless", + "libicu74", + "libkrb5-3", + "zlib1g", + "libgcc-s1", + "libstdc++6", + "libcurl4t64", + "liblttng-ust1t64", + "libunwind8", + "libpulse0", + "libx11-xcb1", + "libnss3", + "libxcomposite1", + "libxcursor1", + "libxi6", + "libxrandr2", + "libxtst6", + "libasound2t64", + "libgl1", + "libegl1", + "libdbus-1-3", + "libxdamage1", + "libxfixes3" + ], + "java_major": 17, + "versions": { + "runner": "2.337.0", + "llvm_cov": "0.9.1", + "nextest": "0.9.144", + "machete": "0.9.2", + "cargo_ndk": "4.1.2", + "protoc": "32.0", + "rustup": "1.28.2" + }, + "android": { + "api": 35, + "build_tools": "35.0.0", + "ndk": "28.1.13356709", + "cmdline_tools": "19.0", + "system_image": "system-images;android-35;default;x86_64", + "abi": "x86_64" + } + } +} diff --git a/.github/scripts/runner-image.py b/.github/scripts/runner-image.py new file mode 100644 index 00000000000..9ee7dd1b2bd --- /dev/null +++ b/.github/scripts/runner-image.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Select an exact PR image and export the shared runner tool requirements.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import time +import urllib.parse +import urllib.request + +MANIFEST = ".github/runner-requirements.json" +REPO = "dashpay/platform" + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def read_manifest(path): + data = Path(path).read_bytes() + require(len(data) <= 256 * 1024, "Requirements manifest is too large") + manifest = json.loads(data) + require(set(manifest) == {"schema", "recipe_revision", "requirements"} and manifest["schema"] == 1, + "Unsupported requirements manifest") + require(re.fullmatch(r"[0-9a-f]{40}", manifest["recipe_revision"]), "Pin the image recipe to a full SHA") + require(manifest["requirements"]["platform"] == "linux/amd64", "Unsupported image platform") + return manifest + + +def fingerprint(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def api(path): + token = os.environ.get("GH_TOKEN", "") + headers = {"Accept": "application/vnd.github+json", "User-Agent": "platform-runner-image"} + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request("https://api.github.com/repos/" + REPO + "/" + path, headers=headers) + with urllib.request.urlopen(request, timeout=30) as response: + return json.load(response) + + +def changed_requirements(pr): + require(pr.get("changed_files", 0) <= 3000, "PR exceeds GitHub's file-list limit; requirements need explicit review") + for page in range(1, 31): + files = api(f"pulls/{pr['number']}/files?per_page=100&page={page}") + if any(f["filename"] == MANIFEST or f.get("previous_filename") == MANIFEST for f in files): + return True + if len(files) < 100: + return False + return False + + +def export_environment(manifest, output): + lock = manifest["requirements"] + versions, android = lock["versions"], lock["android"] + values = { + "CI_CARGO_LLVM_COV_VERSION": versions["llvm_cov"], + "CI_CARGO_NEXTEST_VERSION": versions["nextest"], + "CI_CARGO_MACHETE_VERSION": versions["machete"], + "CI_CARGO_NDK_VERSION": versions["cargo_ndk"], + "CI_PROTOC_VERSION": versions["protoc"], "CI_JAVA_MAJOR": str(lock["java_major"]), + "CI_ANDROID_API": str(android["api"]), "CI_ANDROID_NDK": android["ndk"], + "CI_ANDROID_BUILD_TOOLS": android["build_tools"], + } + require(all(isinstance(value, str) and re.fullmatch(r"[0-9]+(?:[.][0-9]+){0,3}(?:[-+][A-Za-z0-9.-]+)?", value) + for value in values.values()), "Versions must be version-pinned, newline-free values") + with open(output, "a") as handle: + for key, value in values.items(): + handle.write(f"{key}={value}\n") + + +def select(manifest, kind, output, wait_seconds): + fallback = ["self-hosted", "rust-ci" if kind == "rust" else "kotlin-ci"] + event = json.loads(Path(os.environ["GITHUB_EVENT_PATH"]).read_text()) + requested = event.get("pull_request") + labels, changed = fallback, False + if requested: + pr = api(f"pulls/{requested['number']}") + head = requested["head"]["sha"] + require(pr["state"] == "open" and pr["head"]["sha"] == head, "This PR run has been superseded") + changed = changed_requirements(pr) + if changed: + # Use the exact PR requirement, not an accidental merge-tree mix + # after both branches edited this file. Rebase such a PR first. + import base64 + remote = api("contents/" + MANIFEST + "?" + urllib.parse.urlencode({"ref": head})) + expected = json.loads(base64.b64decode(remote["content"])) + require(fingerprint(expected) == fingerprint(manifest), + "Merge-tree requirements differ from PR head; rebase before building a candidate") + deadline = time.monotonic() + wait_seconds + while True: + statuses = api(f"commits/{head}/status")["statuses"] + candidate = next((s for s in statuses if s["context"] == f"Runner image candidate / PR {pr['number']}"), None) + if candidate and candidate["state"] == "success": + require(candidate.get("creator", {}).get("login") == "github-actions[bot]", + "Candidate status must come from the trusted publisher") + require(re.fullmatch(r"sha256:[0-9a-f]{64}", candidate.get("description", "")), + "Publisher did not record an immutable digest") + match = re.fullmatch(r"https://github[.]com/dashpay/platform/actions/runs/([0-9]+)", + candidate.get("target_url", "")) + require(match, "Candidate status is not linked to its publishing workflow") + run = api(f"actions/runs/{match.group(1)}") + require(run["path"] == ".github/workflows/runner-image-candidate.yml" + and run["event"] == "pull_request_target", "Unexpected candidate publisher") + if run["conclusion"] == "success": + labels = ["self-hosted", "Linux", "X64", + f"platform-image-pr-{pr['number']}-{head}-{candidate['description'][7:]}-{kind}"] + break + require(time.monotonic() < deadline, + "Candidate image was not published in time. Check Runner image candidate CI and bootstrap setup.") + current = api(f"pulls/{pr['number']}") + require(current["state"] == "open" and current["head"]["sha"] == head, "PR changed while waiting") + time.sleep(20) + with open(output, "a") as handle: + handle.write("labels=" + json.dumps(labels, separators=(",", ":")) + "\n") + handle.write("image_changed=" + str(changed).lower() + "\n") + print("Candidate runner required" if changed else "Using the ordinary provisioned runner pool") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["env", "verify", "select"]) + parser.add_argument("--manifest", default=MANIFEST) + parser.add_argument("--env", default=os.environ.get("GITHUB_ENV")) + parser.add_argument("--output", default=os.environ.get("GITHUB_OUTPUT")) + parser.add_argument("--kind", choices=["rust", "kotlin"]) + parser.add_argument("--wait-seconds", type=int, default=2400) + args = parser.parse_args() + manifest = read_manifest(args.manifest) + if args.command == "select": + require(args.kind and args.output, "Runner selection needs kind and output") + select(manifest, args.kind, args.output, args.wait_seconds) + return + if args.command == "verify": + subprocess.run(["ci-image-contract", "verify", args.manifest], check=True) + require(args.env, "Environment output file is required") + export_environment(manifest, args.env) + + +if __name__ == "__main__": + try: + main() + except (ValueError, KeyError, TypeError, OSError) as error: + print(f"::error::{error}", file=sys.stderr) + sys.exit(1) diff --git a/.github/scripts/tests/test_runner_image.py b/.github/scripts/tests/test_runner_image.py new file mode 100644 index 00000000000..c670105855c --- /dev/null +++ b/.github/scripts/tests/test_runner_image.py @@ -0,0 +1,97 @@ +"""Exercise runner routing, stale candidates and safe environment exports.""" +import base64 +import copy +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[3] +spec = importlib.util.spec_from_file_location("runner_image", ROOT / ".github/scripts/runner-image.py") +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) +HEAD = "a" * 40 +DIGEST = "sha256:" + "d" * 64 + + +class SelectorTests(unittest.TestCase): + def setUp(self): + self.manifest = runner.read_manifest(ROOT / runner.MANIFEST) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.event = Path(self.temp.name) / "event.json" + self.output = Path(self.temp.name) / "output" + self.pr = {"number": 4702, "state": "open", "changed_files": 1, + "head": {"sha": HEAD}} + self.responses = { + "pulls/4702": self.pr, + "pulls/4702/files?per_page=100&page=1": [{"filename": runner.MANIFEST}], + f"contents/{runner.MANIFEST}?ref={HEAD}": { + "content": base64.b64encode(json.dumps(self.manifest).encode()).decode()}, + f"commits/{HEAD}/status": {"statuses": [{ + "context": "Runner image candidate / PR 4702", "state": "success", + "creator": {"login": "github-actions[bot]"}, "description": DIGEST, + "target_url": "https://github.com/dashpay/platform/actions/runs/7", + }]}, + "actions/runs/7": {"path": ".github/workflows/runner-image-candidate.yml", + "event": "pull_request_target", "conclusion": "success"}, + } + + def select(self, event=None): + self.event.write_text(json.dumps(event if event is not None else {"pull_request": self.pr})) + with patch.dict(os.environ, {"GITHUB_EVENT_PATH": str(self.event)}), \ + patch.object(runner, "api", side_effect=lambda path: self.responses[path]): + runner.select(self.manifest, "rust", self.output, 0) + return dict(line.split("=", 1) for line in self.output.read_text().splitlines()) + + def test_non_pr_and_unchanged_pr_use_existing_pool(self): + self.assertEqual(json.loads(self.select({})["labels"]), ["self-hosted", "rust-ci"]) + self.responses["pulls/4702/files?per_page=100&page=1"] = [{"filename": "Cargo.lock"}] + self.assertEqual(self.select()["image_changed"], "false") + + def test_exact_candidate_includes_head_digest_and_kind(self): + output = self.select() + labels = json.loads(output["labels"]) + self.assertEqual(labels[-1], f"platform-image-pr-4702-{HEAD}-{DIGEST[7:]}-rust") + self.assertEqual(output["image_changed"], "true") + + def test_new_head_or_closed_pr_rejects_stale_run(self): + event = {"pull_request": copy.deepcopy(self.pr)} + self.pr["head"]["sha"] = "e" * 40 + with self.assertRaisesRegex(ValueError, "superseded"): + self.select(event) + self.pr["head"]["sha"] = HEAD + self.pr["state"] = "closed" + with self.assertRaisesRegex(ValueError, "superseded"): + self.select(event) + + def test_merge_tree_cannot_mix_requirements_from_both_branches(self): + self.manifest["recipe_revision"] = "e" * 40 + with self.assertRaisesRegex(ValueError, "rebase"): + self.select() + + def test_missing_or_incomplete_publisher_cannot_select_image(self): + self.responses["actions/runs/7"]["conclusion"] = None + with self.assertRaisesRegex(ValueError, "not published"): + self.select() + self.responses[f"commits/{HEAD}/status"]["statuses"] = [] + with self.assertRaisesRegex(ValueError, "not published"): + self.select() + + def test_other_workflow_cannot_supply_candidate_status(self): + self.responses["actions/runs/7"]["path"] = ".github/workflows/tests.yml" + with self.assertRaisesRegex(ValueError, "Unexpected candidate"): + self.select() + + def test_environment_export_rejects_multiline_values_before_writing(self): + self.manifest["requirements"]["versions"]["protoc"] = "32.0\nINJECTED=yes" + with self.assertRaisesRegex(ValueError, "newline-free"): + runner.export_environment(self.manifest, self.output) + self.assertFalse(self.output.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/kotlin-sdk-build.yml b/.github/workflows/kotlin-sdk-build.yml index cf2034ba0b7..b2470a6bc90 100644 --- a/.github/workflows/kotlin-sdk-build.yml +++ b/.github/workflows/kotlin-sdk-build.yml @@ -25,6 +25,8 @@ on: - '.github/workflows/kotlin-sdk-build.yml' - '.github/scripts/kotlin-instrumented-tests.sh' - '.github/actions/rust/**' + - '.github/runner-requirements.json' + - '.github/scripts/runner-image.py' permissions: contents: read @@ -34,9 +36,35 @@ concurrency: cancel-in-progress: true jobs: + select-runner: + name: Select compatible runner image + if: >- + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository + || github.event.pull_request.head.repo.owner.login == 'thepastaclaw' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + pull-requests: read + statuses: read + actions: read + outputs: + labels: ${{ steps.select.outputs.labels }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Select provisioned pool or wait for the exact PR candidate + id: select + env: + GH_TOKEN: ${{ github.token }} + run: python3 .github/scripts/runner-image.py select --kind kotlin + kotlin-sdk-build: name: Kotlin SDK build + tests (x86_64 emulator) - runs-on: [self-hosted, kotlin-ci] + needs: select-runner + runs-on: ${{ fromJSON(needs.select-runner.outputs.labels) }} # Fork PRs must not execute on a persistent runner. Keep this guard in sync # with tests-rs-workspace.yml. if: >- @@ -56,6 +84,9 @@ jobs: # dependency installation out of the job: it required sudo and made a # CI job capable of mutating its container. Rebuild the runner image when # one of these requirements changes instead. + - name: Verify exact runner requirements and load versions + run: python3 .github/scripts/runner-image.py verify + - name: Verify runner image dependencies run: | set -euo pipefail @@ -78,12 +109,12 @@ jobs: python3 scripts/check_sdk_parity_manifest.py python3 -m unittest discover -s scripts/tests -p 'test_*.py' - - name: Verify JDK 17 + - name: Verify repository-pinned JDK run: | JAVA_HOME_RESOLVED=$(dirname "$(dirname "$(readlink -f "$(command -v java)")")") JAVA_VERSION_OUTPUT=$("$JAVA_HOME_RESOLVED/bin/java" -version 2>&1) printf '%s\n' "$JAVA_VERSION_OUTPUT" - printf '%s\n' "$JAVA_VERSION_OUTPUT" | grep -Eq 'version "17([.]|\")' + printf '%s\n' "$JAVA_VERSION_OUTPUT" | grep -E "version \"${CI_JAVA_MAJOR}([.]|\")" echo "JAVA_HOME=$JAVA_HOME_RESOLVED" >> "$GITHUB_ENV" echo "$JAVA_HOME_RESOLVED/bin" >> "$GITHUB_PATH" @@ -91,10 +122,10 @@ jobs: run: | set -euo pipefail test "${ANDROID_HOME:-}" = /opt/android-sdk - test -x "$ANDROID_HOME/ndk/28.1.13356709/ndk-build" - test -x "$ANDROID_HOME/build-tools/35.0.0/aapt2" - test -f "$ANDROID_HOME/platforms/android-35/android.jar" - test -f "$ANDROID_HOME/system-images/android-35/default/x86_64/system.img" + test -x "$ANDROID_HOME/ndk/$CI_ANDROID_NDK/ndk-build" + test -x "$ANDROID_HOME/build-tools/$CI_ANDROID_BUILD_TOOLS/aapt2" + test -f "$ANDROID_HOME/platforms/android-$CI_ANDROID_API/android.jar" + test -f "$ANDROID_HOME/system-images/android-$CI_ANDROID_API/default/x86_64/system.img" command -v ci-android-emulator command -v adb # Do not run setup-android/sdkmanager: the SDK is pinned and root-owned. @@ -107,19 +138,19 @@ jobs: # Pinned: this runner is persistent, so cargo-ndk is provisioned in the # image and checked here before the NDK build can start. - - name: Verify cargo-ndk v4.1.2 + - name: Verify repository-pinned cargo-ndk run: | set -euo pipefail cargo ndk --version - cargo ndk --version | grep -qx 'cargo-ndk 4.1.2' + cargo ndk --version | grep -Fx "cargo-ndk $CI_CARGO_NDK_VERSION" - - name: Verify protoc v32.0 (repo-standard; apt's 3.21 breaks tenderdash-proto) + - name: Verify repository-pinned protoc run: | set -euo pipefail protoc --version # protoc is part of the runner image. Installing it into /usr/local # from a job required sudo and made the CI container mutable. - protoc --version | grep -qx 'libprotoc 32.0' + protoc --version | grep -Fx "libprotoc $CI_PROTOC_VERSION" # KVM is the only host device this job needs. The container receives it # through the kvm group; no Docker socket or host package-management @@ -150,7 +181,7 @@ jobs: working-directory: packages/kotlin-sdk run: ./gradlew :sdk:compileDebugAndroidTestKotlin --stacktrace - - name: Run instrumented FFI smoke test (prebaked API 35 emulator) + - name: Run instrumented FFI smoke test (prebaked emulator) working-directory: packages/kotlin-sdk run: ci-android-emulator bash ../../.github/scripts/kotlin-instrumented-tests.sh diff --git a/.github/workflows/tests-rs-wallet.yml b/.github/workflows/tests-rs-wallet.yml index ec9d2e5a62d..564a2819b47 100644 --- a/.github/workflows/tests-rs-wallet.yml +++ b/.github/workflows/tests-rs-wallet.yml @@ -130,7 +130,7 @@ jobs: - name: Find unused dependencies run: | - cargo machete --version | grep -Eq '^(cargo-machete )?0[.]9[.]2$' + test "$(cargo machete --version | awk '{print $NF}')" = "$CI_CARGO_MACHETE_VERSION" cargo machete - name: Detect immutable structure changes diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index 61f87fd7d6f..2a2755a9224 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -18,6 +18,31 @@ on: default: true jobs: + select-runner: + name: Select compatible runner image + if: >- + github.event_name != 'pull_request' + || github.event.pull_request.head.repo.full_name == github.repository + || github.event.pull_request.head.repo.owner.login == 'thepastaclaw' + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + pull-requests: read + statuses: read + actions: read + outputs: + labels: ${{ steps.select.outputs.labels }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Select provisioned pool or wait for the exact PR candidate + id: select + env: + GH_TOKEN: ${{ github.token }} + run: python3 .github/scripts/runner-image.py select --kind rust + test: name: Tests # Scheduled onto whichever self-hosted runner is free — the macOS boxes or @@ -25,7 +50,8 @@ jobs: # runners; pairing it with `self-hosted` keeps the job off GitHub-hosted # runners entirely, so untrusted code never reaches a hosted Linux VM by # way of a label collision. - runs-on: [self-hosted, rust-ci] + needs: select-runner + runs-on: ${{ fromJSON(needs.select-runner.outputs.labels) }} # Fork PRs must not execute on any persistent runner, macOS or Linux. if: >- github.event_name != 'pull_request' @@ -148,12 +174,12 @@ jobs: # These helpers are part of the pinned runner image. Version checks make # image drift fail before the test phase instead of installing tools from # a job or silently swallowing a failed installation. - - name: Verify cargo-llvm-cov v0.9.1 - run: cargo llvm-cov --version | grep -qx 'cargo-llvm-cov 0.9.1' + - name: Verify repository-pinned cargo-llvm-cov + run: cargo llvm-cov --version | grep -Fx "cargo-llvm-cov $CI_CARGO_LLVM_COV_VERSION" - - name: Verify cargo-nextest v0.9.144 + - name: Verify repository-pinned cargo-nextest # Release binaries include commit/host metadata after the version. - run: cargo nextest --version | sed -n '1p' | grep -Eq '^cargo-nextest 0[.]9[.]144( |$)' + run: test "$(cargo nextest --version | awk 'NR == 1 {print $2}')" = "$CI_CARGO_NEXTEST_VERSION" - name: Check formatting run: cargo fmt --check --all @@ -179,7 +205,7 @@ jobs: - name: Find unused dependencies run: | - cargo machete --version | grep -Eq '^(cargo-machete )?0[.]9[.]2$' + test "$(cargo machete --version | awk '{print $NF}')" = "$CI_CARGO_MACHETE_VERSION" cargo machete # The transport-free cuts are how embedders with their own networking diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 86d9675f743..fae6453f8c2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -61,6 +61,11 @@ jobs: - name: Verify self-hosted Swift runner policy run: python3 .github/scripts/check-swift-self-hosted-runner.py + - name: Verify candidate-image routing and manifest inputs + run: | + python3 .github/scripts/runner-image.py env + python3 -m unittest discover -s .github/scripts/tests -v + - uses: dorny/paths-filter@v4 id: filter-js if: ${{ github.event_name != 'workflow_dispatch' }} @@ -89,6 +94,9 @@ jobs: - .github/workflows/tests-rs-wallet.yml - .github/workflows/tests.yml - .github/scripts/check-wallet-closure.py + - .github/runner-requirements.json + - .github/scripts/runner-image.py + - .github/actions/rust/** - uses: dorny/paths-filter@v4 id: filter-e2e @@ -271,7 +279,7 @@ jobs: exit 0 fi - if echo "$CHANGED" | grep -qE '(^|/)Cargo\.(toml|lock)$|^rust-toolchain\.toml$|^\.github/actions/rust/|^\.github/workflows/tests\.yml$|^\.github/workflows/tests-rs-workspace\.yml$'; then + if echo "$CHANGED" | grep -qE '(^|/)Cargo\.(toml|lock)$|^rust-toolchain\.toml$|^\.github/runner-requirements\.json$|^\.github/scripts/runner-image\.py$|^\.github/actions/rust/|^\.github/workflows/tests\.yml$|^\.github/workflows/tests-rs-workspace\.yml$'; then echo "shielded-changed=true" >> "$GITHUB_OUTPUT" echo "Build configuration changed — shielded tests will run" exit 0 From fb3d76cbe300c8a7bc25dd1993c38e7e74875f81 Mon Sep 17 00:00:00 2001 From: infraclaw-dash <283232465+infraclaw-dash@users.noreply.github.com> Date: Tue, 22 Sep 2026 15:15:24 +0000 Subject: [PATCH 7/7] ci: allow image publication and build artifact uploads to finish --- .github/scripts/runner-image.py | 2 +- .github/workflows/kotlin-sdk-build.yml | 3 ++- .github/workflows/tests-build-js.yml | 3 ++- .github/workflows/tests-rs-workspace.yml | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/scripts/runner-image.py b/.github/scripts/runner-image.py index 9ee7dd1b2bd..258cf6c442c 100644 --- a/.github/scripts/runner-image.py +++ b/.github/scripts/runner-image.py @@ -131,7 +131,7 @@ def main(): parser.add_argument("--env", default=os.environ.get("GITHUB_ENV")) parser.add_argument("--output", default=os.environ.get("GITHUB_OUTPUT")) parser.add_argument("--kind", choices=["rust", "kotlin"]) - parser.add_argument("--wait-seconds", type=int, default=2400) + parser.add_argument("--wait-seconds", type=int, default=7200) args = parser.parse_args() manifest = read_manifest(args.manifest) if args.command == "select": diff --git a/.github/workflows/kotlin-sdk-build.yml b/.github/workflows/kotlin-sdk-build.yml index b2470a6bc90..a6f6fe1c6ff 100644 --- a/.github/workflows/kotlin-sdk-build.yml +++ b/.github/workflows/kotlin-sdk-build.yml @@ -43,7 +43,8 @@ jobs: || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.head.repo.owner.login == 'thepastaclaw' runs-on: ubuntu-24.04 - timeout-minutes: 45 + # Allow the separate 90-minute build and publication/queue window to finish. + timeout-minutes: 135 permissions: contents: read pull-requests: read diff --git a/.github/workflows/tests-build-js.yml b/.github/workflows/tests-build-js.yml index 75d1324ee40..992ab762929 100644 --- a/.github/workflows/tests-build-js.yml +++ b/.github/workflows/tests-build-js.yml @@ -5,7 +5,8 @@ jobs: build-js: name: Build JS runs-on: ubuntu-24.04 - timeout-minutes: 15 + # A cold build can finish near 15 minutes; leave time for artifact upload. + timeout-minutes: 20 steps: - uses: softwareforgood/check-artifact-v4-existence@v0 id: check-artifact diff --git a/.github/workflows/tests-rs-workspace.yml b/.github/workflows/tests-rs-workspace.yml index 2a2755a9224..70514800643 100644 --- a/.github/workflows/tests-rs-workspace.yml +++ b/.github/workflows/tests-rs-workspace.yml @@ -25,7 +25,8 @@ jobs: || github.event.pull_request.head.repo.full_name == github.repository || github.event.pull_request.head.repo.owner.login == 'thepastaclaw' runs-on: ubuntu-24.04 - timeout-minutes: 45 + # Allow the separate 90-minute build and publication/queue window to finish. + timeout-minutes: 135 permissions: contents: read pull-requests: read