From 52423215be5c85ec6794379e7eec9059ce3adbbc Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 8 Aug 2026 20:21:28 +0800 Subject: [PATCH] =?UTF-8?q?=F0=9F=91=B7=20ci:=20make=20hardware=20benchmar?= =?UTF-8?q?k=20self-contained?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hardware.yml | 20 +-------- benchmarks/cpu-inventory/run-container.py | 55 +++++++++++++++++++++++ benchmarks/cpu-inventory/run-container.sh | 21 +++++++++ docs/benchmarks.md | 9 ++-- docs/development.md | 13 +++--- justfile | 3 ++ 6 files changed, 94 insertions(+), 27 deletions(-) create mode 100755 benchmarks/cpu-inventory/run-container.py create mode 100755 benchmarks/cpu-inventory/run-container.sh diff --git a/.github/workflows/hardware.yml b/.github/workflows/hardware.yml index 6848f9e..3654ec5 100644 --- a/.github/workflows/hardware.yml +++ b/.github/workflows/hardware.yml @@ -16,18 +16,9 @@ jobs: with: components: clippy,rustfmt - uses: taiki-e/install-action@just - - name: Verify CPU profiler prerequisites - run: | - command -v perf - command -v py-spy - sudo -n true - /usr/bin/python3 -X perf -c \ - "import sys; assert sys._xoptions.get('perf') is True" - run: just test-bpf-live - name: Test CPU and Python sampling - run: >- - sudo --preserve-env=PATH,CARGO_HOME,RUSTUP_HOME - "$(command -v just)" test-cpu-live + run: just test-cpu-live - run: just test-cupti-live-cuda12 - run: just test-nvtx-live-cuda12 - run: just test-cupti-live-cuda12-min @@ -44,14 +35,7 @@ jobs: - run: just benchmark-multiprocess - run: just benchmark-pytorch - name: Benchmark CPU inventory - run: >- - sudo --preserve-env=PATH - /usr/bin/python3 benchmarks/cpu-inventory/run.py - --xprobe "$PWD/target/debug/xprobe" - --python /usr/bin/python3 - --perf "$(command -v perf)" - --py-spy "$(command -v py-spy)" - | tee cpu-inventory-benchmark.json + run: just benchmark-cpu | tee cpu-inventory-benchmark.json - uses: actions/upload-artifact@v7 with: name: cpu-inventory-benchmark diff --git a/benchmarks/cpu-inventory/run-container.py b/benchmarks/cpu-inventory/run-container.py new file mode 100755 index 0000000..f0c715b --- /dev/null +++ b/benchmarks/cpu-inventory/run-container.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +import json +import pathlib +import subprocess +import sys + + +def main() -> None: + if len(sys.argv) != 2: + raise SystemExit("usage: run-container.py ") + + workspace = pathlib.Path(__file__).resolve().parents[2] + completed = subprocess.run( + [ + "docker", + "run", + "--rm", + "--cap-add", + "BPF", + "--cap-add", + "PERFMON", + "--cap-add", + "SYS_ADMIN", + "--cap-add", + "SYS_RESOURCE", + "--cap-add", + "SYS_PTRACE", + "--security-opt", + "seccomp=unconfined", + "--volume", + f"{workspace}:/workspace:ro", + "--workdir", + "/workspace", + sys.argv[1], + "/workspace/benchmarks/cpu-inventory/run-container.sh", + ], + check=False, + capture_output=True, + text=True, + ) + if completed.returncode != 0: + sys.stdout.write(completed.stdout) + sys.stderr.write(completed.stderr) + raise SystemExit(completed.returncode) + + reports = [line for line in completed.stdout.splitlines() if line.startswith("{")] + if not reports: + raise AssertionError( + f"CPU inventory benchmark emitted no JSON report:\n{completed.stdout}" + ) + print(json.dumps(json.loads(reports[-1]), sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/cpu-inventory/run-container.sh b/benchmarks/cpu-inventory/run-container.sh new file mode 100755 index 0000000..7e4cbc3 --- /dev/null +++ b/benchmarks/cpu-inventory/run-container.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y --no-install-recommends \ + linux-tools-common linux-tools-generic python3 python3-pip +python3 -m pip install --break-system-packages --disable-pip-version-check \ + py-spy==0.4.2 + +perf_path=$(find /usr/lib/linux-tools-* -type f -name perf -print | sort -V | tail -n 1) +[[ -x "${perf_path}" ]] || { + echo "installed perf executable was not found" >&2 + exit 1 +} + +exec python3 /workspace/benchmarks/cpu-inventory/run.py \ + --xprobe /workspace/target/debug/xprobe \ + --python /usr/bin/python3 \ + --perf "${perf_path}" \ + --py-spy "$(command -v py-spy)" diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 2f5a5fd..12a93e6 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,13 +1,16 @@ # Precision and overhead benchmarks -Run the CPU and Python inventory benchmark on a Linux host with `perf`, -`py-spy`, CPython 3.12 or newer with `-X perf`, perf-event access, and eBPF -attach permission: +Run the CPU and Python inventory benchmark through the pinned +capability-limited container: ```bash just benchmark-cpu ``` +For an already-provisioned Linux host with `perf`, `py-spy`, CPython 3.12 or +newer with `-X perf`, perf-event access, and eBPF attach permission, use +`just benchmark-cpu-host`. + The benchmark uses a fresh target for every case. Native and Python workloads are each measured without a profiler, with bounded xprobe CPU sampling, and with their corresponding broad profiler (`perf` or `py-spy`). A native syscall diff --git a/docs/development.md b/docs/development.md index 69be06b..619840f 100644 --- a/docs/development.md +++ b/docs/development.md @@ -21,9 +21,9 @@ SONAMEs, and rejects ABI-only output or build-time RPATHs. Live CUDA behavior remains a hardware test on an NVIDIA runner. The self-hosted hardware runner must use Actions Runner 2.329.0 or newer and -provide passwordless `sudo`, `perf`, `py-spy`, and `/usr/bin/python3` with -`-X perf`. These are host profiler prerequisites, not release archive -dependencies. +provide Docker and NVIDIA Container Toolkit access. CPU comparison tools and a +USDT-enabled CPython are installed inside the pinned benchmark container; they +are not runner or release archive dependencies. ## Release packaging @@ -100,9 +100,10 @@ broad-to-narrow route: just benchmark-cpu ``` -The benchmark requires `perf` and `py-spy` and may require root on hosts whose -perf-event or eBPF policy denies attachment. See `docs/benchmarks.md` for its -reported metrics and interpretation. +The default recipe installs `perf`, `py-spy`, and a USDT-enabled CPython inside +the pinned capability-limited container. Use `just benchmark-cpu-host` when the +same tools and attach permissions are already available on the host. See +`docs/benchmarks.md` for reported metrics and interpretation. Resolve real CPython, native extension, and libtorch C++ symbols with a local Python environment containing PyTorch: diff --git a/justfile b/justfile index 9fb10c0..f40ac67 100644 --- a/justfile +++ b/justfile @@ -103,6 +103,9 @@ benchmark-pytorch: build if [[ -n "${PYTORCH_ENV:-}" ]]; then python3 benchmarks/pytorch/run.py --image "{{cuda12_devel_image}}" --pytorch-env "${PYTORCH_ENV}"; else python3 benchmarks/pytorch/run.py --image "{{pytorch_image}}"; fi benchmark-cpu: build + python3 benchmarks/cpu-inventory/run-container.py "{{cuda12_devel_image}}" + +benchmark-cpu-host: build python3 benchmarks/cpu-inventory/run.py fmt: