👷 ci: keep hardware validation local

.github/workflows/release.yml

@@ -6,32 +6,10 @@ on:
       - "v*"
 
 permissions:
-  actions: read
   contents: write
 
 jobs:
-  hardware-gate:
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-      - name: Require successful hardware validation for this commit
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          commit=$(git rev-parse HEAD)
-          runs=$(gh api --method GET \
-            "repos/${GITHUB_REPOSITORY}/actions/workflows/hardware.yml/runs" \
-            -f head_sha="$commit" \
-            -f status=success \
-            -f per_page=100 \
-            --jq '.total_count')
-          if [ "$runs" -lt 1 ]; then
-            echo "No successful Hardware Integration run exists for $commit" >&2
-            exit 1
-          fi
-
   cupti-agent:
-    needs: hardware-gate
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false

benchmarks/cuda-multiprocess/run-container.pl

@@ -958,6 +958,7 @@ sub run_batch {
 sub main {
     require_condition(@ARGV == 1,
         'usage: run-container.pl <output-directory>');
+    umask 0000;
     my $output_dir = $ARGV[0];
     $OUTPUT_DIRECTORY = $output_dir;
     mkdir $output_dir unless -d $output_dir;

benchmarks/cuda-multiprocess/run.py

@@ -6,13 +6,21 @@
 import sys
 import tempfile
 
+FAILURE_ARTIFACT_LIMIT = 16 * 1024
+
 
 def dump_failure_artifacts(output: pathlib.Path) -> None:
     for path in sorted(output.rglob("*")):
         if not path.is_file() or path.suffix not in {".json", ".stderr", ".stdout"}:
             continue
         sys.stderr.write(f"\n--- {path.relative_to(output)} ---\n")
-        sys.stderr.write(path.read_text(errors="replace"))
+        with path.open(errors="replace") as artifact:
+            content = artifact.read(FAILURE_ARTIFACT_LIMIT + 1)
+        sys.stderr.write(content[:FAILURE_ARTIFACT_LIMIT])
+        if len(content) > FAILURE_ARTIFACT_LIMIT:
+            sys.stderr.write(
+                f"\n... artifact truncated at {FAILURE_ARTIFACT_LIMIT} bytes ...\n"
+            )
 
 
 def main() -> None:

docs/development.md

@@ -18,12 +18,9 @@ zlib. A system C compiler and Linux UAPI/multiarch headers are also required.
 CUDA is not installed into the Mamba environment. CI compiles CUDA 12 and CUDA
 13 CUPTI Agents without a GPU in pinned NVIDIA devel images, checks their
 SONAMEs, and rejects ABI-only output or build-time RPATHs. Live CUDA behavior
-remains a hardware test on an NVIDIA runner.
-
-The self-hosted hardware runner must use Actions Runner 2.329.0 or newer and
-provide Docker and NVIDIA Container Toolkit access. CPU comparison tools and a
-USDT-enabled CPython are installed inside the pinned benchmark container; they
-are not runner or release archive dependencies.
+is tested locally with Docker and NVIDIA Container Toolkit. CPU comparison
+tools and a USDT-enabled CPython are installed inside the pinned benchmark
+container; they are not host or release archive dependencies.
 
 ## Release packaging
 
@@ -53,10 +50,19 @@ the public archive and checksum again, repeats the installation test, and
 inspects every shipped ELF. This final gate verifies the artifact users can
 actually download rather than the workflow's local copy.
 
-Before creating a tag, manually run `Hardware Integration` for the exact commit
-to be tagged. The release workflow queries GitHub Actions for a successful run
-whose `head_sha` matches that commit and fails before building artifacts when
-the hardware result is absent.
+GitHub Actions runs only tests and builds supported by hosted runners. Before
+creating a tag, run the complete live suite on the local NVIDIA development
+machine:
+
+```bash
+PYTORCH_ENV=/path/to/pytorch-env just test-release-live
+```
+
+This local gate covers BPF and perf attachment, CUDA 12 and 13 behavior,
+injection, NVTX, mixed host/GPU collection, PyTorch, concurrent workers, and
+the release benchmarks. The tag workflow independently rebuilds both CUPTI
+Agents, packages on Ubuntu 22.04, enforces the GLIBC ceiling, and verifies the
+published archive.
 
 ## eBPF tests
 
@@ -120,8 +126,8 @@ just test-pytorch-live
 ```
 
 By default the live recipes use a pinned NVIDIA PyTorch image with Ubuntu 24.04
-and CUDA 12.9, so hardware CI does not depend on a runner-local Mamba
-environment. During local development, set `PYTORCH_ENV=/path/to/env` to mount
+and CUDA 12.9, so live checks do not depend on the host Mamba environment.
+During local development, set `PYTORCH_ENV=/path/to/env` to mount
 an existing environment into the already-pinned CUDA fixtures instead of
 pulling the PyTorch image. Run eager matrix multiplication, convolution,
 compiled Triton, bidirectional transfer, selected-kernel, and

justfile

@@ -90,6 +90,27 @@ test-nvtx-live: build
 test-nvtx-live-cuda12: build
     python3 tests/integration/test_nvtx.py --image "{{cuda12_devel_image}}"
 
+test-release-live:
+    [[ -n "${PYTORCH_ENV:-}" ]] || { echo "set PYTORCH_ENV to an existing environment with PyTorch" >&2; exit 2; }
+    just test-bpf-live
+    just test-cpu-live
+    just test-cupti-live-cuda12
+    just test-nvtx-live-cuda12
+    just test-cupti-live-cuda12-min
+    just test-injection-live-cuda12
+    just test-multisource-live-cuda12
+    just test-cupti-live
+    just test-nvtx-live
+    just test-injection-live
+    just test-multisource-live
+    just test-pytorch-live
+    just test-pytorch-cuda-live
+    just benchmark-gpu
+    just benchmark-aggregate
+    just benchmark-multiprocess
+    just benchmark-pytorch
+    just benchmark-cpu
+
 benchmark-gpu:
     python3 benchmarks/cuda-callback/run.py "{{cuda13_devel_image}}"