From db006b9e0c20137da8ae05819ccbc10387569af6 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 8 Aug 2026 23:29:34 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=F0=9F=90=9B=20fix:=20bound=20multiprocess?= =?UTF-8?q?=20failure=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmarks/cuda-multiprocess/run-container.pl | 1 + benchmarks/cuda-multiprocess/run.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/benchmarks/cuda-multiprocess/run-container.pl b/benchmarks/cuda-multiprocess/run-container.pl index 9f36780..e7bb828 100755 --- a/benchmarks/cuda-multiprocess/run-container.pl +++ b/benchmarks/cuda-multiprocess/run-container.pl @@ -958,6 +958,7 @@ sub run_batch { sub main { require_condition(@ARGV == 1, 'usage: run-container.pl '); + umask 0000; my $output_dir = $ARGV[0]; $OUTPUT_DIRECTORY = $output_dir; mkdir $output_dir unless -d $output_dir; diff --git a/benchmarks/cuda-multiprocess/run.py b/benchmarks/cuda-multiprocess/run.py index 78fd631..e31ac8d 100755 --- a/benchmarks/cuda-multiprocess/run.py +++ b/benchmarks/cuda-multiprocess/run.py @@ -6,13 +6,21 @@ import sys import tempfile +FAILURE_ARTIFACT_LIMIT = 16 * 1024 + def dump_failure_artifacts(output: pathlib.Path) -> None: for path in sorted(output.rglob("*")): if not path.is_file() or path.suffix not in {".json", ".stderr", ".stdout"}: continue sys.stderr.write(f"\n--- {path.relative_to(output)} ---\n") - sys.stderr.write(path.read_text(errors="replace")) + with path.open(errors="replace") as artifact: + content = artifact.read(FAILURE_ARTIFACT_LIMIT + 1) + sys.stderr.write(content[:FAILURE_ARTIFACT_LIMIT]) + if len(content) > FAILURE_ARTIFACT_LIMIT: + sys.stderr.write( + f"\n... artifact truncated at {FAILURE_ARTIFACT_LIMIT} bytes ...\n" + ) def main() -> None: From 732b8c4d85216bfe6e8cc48456546d52d997f938 Mon Sep 17 00:00:00 2001 From: itdevwu Date: Sat, 8 Aug 2026 23:29:45 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=F0=9F=91=B7=20ci:=20keep=20hardware=20vali?= =?UTF-8?q?dation=20local?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/hardware.yml | 43 ---------------------------------- .github/workflows/release.yml | 22 ----------------- docs/development.md | 30 ++++++++++++++---------- justfile | 21 +++++++++++++++++ 4 files changed, 39 insertions(+), 77 deletions(-) delete mode 100644 .github/workflows/hardware.yml diff --git a/.github/workflows/hardware.yml b/.github/workflows/hardware.yml deleted file mode 100644 index 3654ec5..0000000 --- a/.github/workflows/hardware.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: Hardware Integration - -on: - workflow_dispatch: - -permissions: - contents: read - -jobs: - gpu-and-bpf: - runs-on: [self-hosted, linux, x64, nvidia] - timeout-minutes: 120 - steps: - - uses: actions/checkout@v6 - - uses: dtolnay/rust-toolchain@stable - with: - components: clippy,rustfmt - - uses: taiki-e/install-action@just - - run: just test-bpf-live - - name: Test CPU and Python sampling - run: just test-cpu-live - - run: just test-cupti-live-cuda12 - - run: just test-nvtx-live-cuda12 - - run: just test-cupti-live-cuda12-min - - run: just test-injection-live-cuda12 - - run: just test-multisource-live-cuda12 - - run: just test-cupti-live - - run: just test-nvtx-live - - run: just test-injection-live - - run: just test-multisource-live - - run: just test-pytorch-live - - run: just test-pytorch-cuda-live - - run: just benchmark-gpu - - run: just benchmark-aggregate - - run: just benchmark-multiprocess - - run: just benchmark-pytorch - - name: Benchmark CPU inventory - run: just benchmark-cpu | tee cpu-inventory-benchmark.json - - uses: actions/upload-artifact@v7 - with: - name: cpu-inventory-benchmark - path: cpu-inventory-benchmark.json - if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d97efcb..ea074cf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,32 +6,10 @@ on: - "v*" permissions: - actions: read contents: write jobs: - hardware-gate: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v6 - - name: Require successful hardware validation for this commit - env: - GH_TOKEN: ${{ github.token }} - run: | - commit=$(git rev-parse HEAD) - runs=$(gh api --method GET \ - "repos/${GITHUB_REPOSITORY}/actions/workflows/hardware.yml/runs" \ - -f head_sha="$commit" \ - -f status=success \ - -f per_page=100 \ - --jq '.total_count') - if [ "$runs" -lt 1 ]; then - echo "No successful Hardware Integration run exists for $commit" >&2 - exit 1 - fi - cupti-agent: - needs: hardware-gate runs-on: ubuntu-24.04 strategy: fail-fast: false diff --git a/docs/development.md b/docs/development.md index 619840f..6698344 100644 --- a/docs/development.md +++ b/docs/development.md @@ -18,12 +18,9 @@ zlib. A system C compiler and Linux UAPI/multiarch headers are also required. CUDA is not installed into the Mamba environment. CI compiles CUDA 12 and CUDA 13 CUPTI Agents without a GPU in pinned NVIDIA devel images, checks their SONAMEs, and rejects ABI-only output or build-time RPATHs. Live CUDA behavior -remains a hardware test on an NVIDIA runner. - -The self-hosted hardware runner must use Actions Runner 2.329.0 or newer and -provide Docker and NVIDIA Container Toolkit access. CPU comparison tools and a -USDT-enabled CPython are installed inside the pinned benchmark container; they -are not runner or release archive dependencies. +is tested locally with Docker and NVIDIA Container Toolkit. CPU comparison +tools and a USDT-enabled CPython are installed inside the pinned benchmark +container; they are not host or release archive dependencies. ## Release packaging @@ -53,10 +50,19 @@ the public archive and checksum again, repeats the installation test, and inspects every shipped ELF. This final gate verifies the artifact users can actually download rather than the workflow's local copy. -Before creating a tag, manually run `Hardware Integration` for the exact commit -to be tagged. The release workflow queries GitHub Actions for a successful run -whose `head_sha` matches that commit and fails before building artifacts when -the hardware result is absent. +GitHub Actions runs only tests and builds supported by hosted runners. Before +creating a tag, run the complete live suite on the local NVIDIA development +machine: + +```bash +PYTORCH_ENV=/path/to/pytorch-env just test-release-live +``` + +This local gate covers BPF and perf attachment, CUDA 12 and 13 behavior, +injection, NVTX, mixed host/GPU collection, PyTorch, concurrent workers, and +the release benchmarks. The tag workflow independently rebuilds both CUPTI +Agents, packages on Ubuntu 22.04, enforces the GLIBC ceiling, and verifies the +published archive. ## eBPF tests @@ -120,8 +126,8 @@ just test-pytorch-live ``` By default the live recipes use a pinned NVIDIA PyTorch image with Ubuntu 24.04 -and CUDA 12.9, so hardware CI does not depend on a runner-local Mamba -environment. During local development, set `PYTORCH_ENV=/path/to/env` to mount +and CUDA 12.9, so live checks do not depend on the host Mamba environment. +During local development, set `PYTORCH_ENV=/path/to/env` to mount an existing environment into the already-pinned CUDA fixtures instead of pulling the PyTorch image. Run eager matrix multiplication, convolution, compiled Triton, bidirectional transfer, selected-kernel, and diff --git a/justfile b/justfile index f40ac67..e9882cb 100644 --- a/justfile +++ b/justfile @@ -90,6 +90,27 @@ test-nvtx-live: build test-nvtx-live-cuda12: build python3 tests/integration/test_nvtx.py --image "{{cuda12_devel_image}}" +test-release-live: + [[ -n "${PYTORCH_ENV:-}" ]] || { echo "set PYTORCH_ENV to an existing environment with PyTorch" >&2; exit 2; } + just test-bpf-live + just test-cpu-live + just test-cupti-live-cuda12 + just test-nvtx-live-cuda12 + just test-cupti-live-cuda12-min + just test-injection-live-cuda12 + just test-multisource-live-cuda12 + just test-cupti-live + just test-nvtx-live + just test-injection-live + just test-multisource-live + just test-pytorch-live + just test-pytorch-cuda-live + just benchmark-gpu + just benchmark-aggregate + just benchmark-multiprocess + just benchmark-pytorch + just benchmark-cpu + benchmark-gpu: python3 benchmarks/cuda-callback/run.py "{{cuda13_devel_image}}"