From 84bdd318751595dce89154b781e69bdd3680b9be Mon Sep 17 00:00:00 2001 From: Jeremy Walker Date: Mon, 21 Sep 2026 16:05:38 +0200 Subject: [PATCH] Publish and test the client-side runner from this repo Co-Authored-By: Claude Opus 5 --- .github/CODEOWNERS | 3 + .github/clientside/run-in-kernel.mjs | 118 ++++++++++++++ .github/workflows/ci.yml | 6 + .github/workflows/clientside-test.yml | 90 +++++++++++ .github/workflows/publish-clientside.yml | 190 +++++++++++++++++++++++ README.md | 29 ++++ bin/build-clientside-tarball.sh | 114 ++++++++++++++ clientside.json | 13 ++ 8 files changed, 563 insertions(+) create mode 100644 .github/clientside/run-in-kernel.mjs create mode 100644 .github/workflows/clientside-test.yml create mode 100644 .github/workflows/publish-clientside.yml create mode 100755 bin/build-clientside-tarball.sh create mode 100644 clientside.json diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 691de80..c1bb586 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,4 @@ * @exercism/guardians + +# Writes to the test-runners bucket, so changes to it need sign-off. +/.github/workflows/publish-clientside.yml @iHiD @exercism/maintainers-admin diff --git a/.github/clientside/run-in-kernel.mjs b/.github/clientside/run-in-kernel.mjs new file mode 100644 index 0000000..6c92ffb --- /dev/null +++ b/.github/clientside/run-in-kernel.mjs @@ -0,0 +1,118 @@ +#!/usr/bin/env node +// Boot a kernel in headless Chromium, untar a runner into it, run one command. +// +// run-in-kernel.mjs [args...] +// +// Exits with the command's status. stdout/stderr are relayed as they arrive. +// +// This is the browser equivalent of `docker run --entrypoint `. +// The kernel is threaded wasm and only runs in a cross-origin isolated page, +// so there is no way to drive it from Node directly: everything is served +// to a real headless Chromium, which is also the runtime students get. +// +// Needs `playwright` on the module path and a Chromium it can launch +// (`npx playwright install --with-deps chromium`, or CHROMIUM_PATH). + +import fs from "node:fs"; +import http from "node:http"; +import path from "node:path"; +import { chromium } from "playwright"; + +const [kernelDir, sysrootDir, bootJson, tarball, ...argv] = process.argv.slice(2); +if (!argv.length) { + console.error("usage: run-in-kernel.mjs [args...]"); + process.exit(64); +} + +const DEBUG = !!process.env.KERNEL_DEBUG; + +const files = { + "/kernel/kernel.js": [path.join(kernelDir, "kernel.js"), "text/javascript"], + "/kernel/kernel_bg.wasm": [path.join(kernelDir, "kernel_bg.wasm"), "application/wasm"], + "/kernel/kernel_client.mjs": [path.join(kernelDir, "kernel_client.mjs"), "text/javascript"], + "/sysroot.tar": [path.join(sysrootDir, "sysroot.tar"), "application/x-tar"], + "/boot.json": [bootJson, "application/json"], + "/runner.tar": [tarball, "application/x-tar"], +}; +for (const [p] of Object.values(files)) { + if (!fs.existsSync(p)) { console.error(`missing: ${p}`); process.exit(66); } +} + +// The kernel is threaded wasm: the page must be cross-origin isolated, which +// means every response carries COOP/COEP. Same pair the website sends. +const ISOLATION = { + "Cross-Origin-Opener-Policy": "same-origin", + "Cross-Origin-Embedder-Policy": "credentialless", +}; + +const PAGE = `kernel +`; + +const server = http.createServer((req, res) => { + const p = new URL(req.url, "http://localhost").pathname; + if (p === "/") { res.writeHead(200, { ...ISOLATION, "Content-Type": "text/html" }); return res.end(PAGE); } + const entry = files[p]; + if (!entry) { res.writeHead(404, ISOLATION); return res.end(); } + res.writeHead(200, { ...ISOLATION, "Content-Type": entry[1] }); + fs.createReadStream(entry[0]).pipe(res); +}); +await new Promise((r) => server.listen(0, "127.0.0.1", r)); +const origin = `http://127.0.0.1:${server.address().port}`; + +const browser = await chromium.launch({ executablePath: process.env.CHROMIUM_PATH, args: ["--no-sandbox"] }); +const page = await browser.newPage(); +await page.exposeFunction("__out", (s) => process.stdout.write(s)); +await page.exposeFunction("__err", (s) => process.stderr.write(s)); +await page.exposeFunction("__event", (s) => { if (DEBUG) console.error("[event]", s); }); +page.on("pageerror", (e) => console.error("[page]", e.message)); +page.on("console", (m) => { if (m.type() === "error") console.error("[console]", m.text()); }); + +let status = 1; +try { + await page.goto(origin, { waitUntil: "load" }); + if (!(await page.evaluate(() => crossOriginIsolated))) throw new Error("page is not cross-origin isolated"); + await page.waitForFunction(() => window.__ready, null, { timeout: 30_000 }); + const raw = await page.evaluate((a) => window.__run(a), argv).then( + (s) => s, + (e) => { console.error("[kernel]", e.message); return 1; }, + ); + if (DEBUG) console.error("[run returned]", JSON.stringify(raw)); + status = raw ?? 0; +} finally { + await browser.close().catch(() => {}); + server.close(); +} +process.exit(status); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d427a3..b1e6acd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,3 +18,9 @@ jobs: - name: Run Tests in Docker run: bin/run-tests-in-docker.sh + + # The same suite, run the way a student's browser runs it. The workflow is + # written to move to exercism/github-actions; this line then becomes + # uses: exercism/github-actions/.github/workflows/clientside-test.yml@main + clientside: + uses: ./.github/workflows/clientside-test.yml diff --git a/.github/workflows/clientside-test.yml b/.github/workflows/clientside-test.yml new file mode 100644 index 0000000..3335d3d --- /dev/null +++ b/.github/workflows/clientside-test.yml @@ -0,0 +1,90 @@ +# Runs a track's test-runner suite inside the wasm kernel, on the kernel and +# sysroot its clientside.json names: the track's bin/run.sh, driven by its own +# bin/run-tests.sh, in headless Chromium. It is the browser equivalent of +# `bin/run-tests-in-docker.sh`, and what's under test is byte-for-byte the +# tarball publish-clientside.yml would ship. +# +# Lives here for now but is written as the reusable workflow it will become +# in exercism/github-actions. Nothing in it is track-specific except what it +# reads from clientside.json. "Get the harness" is the only step that changes +# on the move. +name: Client-side tests + +on: + workflow_call: + +jobs: + test: + name: Tests (client-side) + runs-on: ubuntu-26.04 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + # Once this workflow lives in exercism/github-actions, this becomes a + # sparse checkout of that repo's clientside/ into harness/. Today the + # harness is beside this file. + - name: Get the harness + run: cp -r .github/clientside harness + + - name: Read clientside.json + id: config + run: | + track="${GITHUB_REPOSITORY##*/}" + track="${track%-test-runner}" + kernel="$(jq -er '.kernel' clientside.json)" + version="$(jq -er '.sysroot.version' clientside.json)" + id="$(jq -er '.sysroot.id' clientside.json)" + { + echo "kernel=kernel/${kernel}" + echo "sysroot=sysroot/${track}/${version}/${id}" + } >> "$GITHUB_OUTPUT" + + # From where the website serves them, so what's tested is exactly what + # students get. Nothing here is published until the track points at it, + # so a PR naming an unpublished kernel or sysroot fails right here. + - name: Fetch the kernel and sysroot + run: | + base="https://exercism.org/test-runners" + fetch() { + mkdir -p "$(dirname "$1")" + curl --fail --silent --show-error --location --retry 3 "${base}/$1" --output "$1" + } + for f in kernel.js kernel_bg.wasm kernel_client.mjs; do + fetch "${{ steps.config.outputs.kernel }}/${f}" + done + for f in sysroot.tar boot.json; do + fetch "${{ steps.config.outputs.sysroot }}/${f}" + done + ls -l "${{ steps.config.outputs.kernel }}" "${{ steps.config.outputs.sysroot }}" + + # The exact tarball publish-clientside.yml would ship, with the suite + # laid on top. + - name: Build the runner tarball + run: | + bin/build-clientside-tarball.sh test-runner.tar + tar --append --file test-runner.tar --transform 's|^|opt/test-runner/|' bin/run-tests.sh tests + + # Same merge as publish-clientside.yml: the sysroot describes itself, + # the track adds its env and preload. + - name: Build boot.json + run: | + jq -s '.[0] as $base | .[1] as $track | + $base + { + env: ($base.env + ($track.env // {})), + preload: ($track.preload // $base.preload) + }' "${{ steps.config.outputs.sysroot }}/boot.json" clientside.json > boot.json + + - name: Install Playwright + working-directory: harness + run: | + npm install --no-save --no-package-lock playwright@1.63.0 + npx playwright install --with-deps chromium + + - name: Run the suite in the kernel + run: | + node harness/run-in-kernel.mjs \ + "${{ steps.config.outputs.kernel }}" \ + "${{ steps.config.outputs.sysroot }}" \ + boot.json test-runner.tar \ + /opt/test-runner/bin/run-tests.sh diff --git a/.github/workflows/publish-clientside.yml b/.github/workflows/publish-clientside.yml new file mode 100644 index 0000000..f14b8c8 --- /dev/null +++ b/.github/workflows/publish-clientside.yml @@ -0,0 +1,190 @@ +# Publishes the browser test runner: a tarball of this repo's bin/run.sh that +# the wasm kernel unpacks at boot, the boot.json that goes with it, and +# the latest.json that points the website at both. +# +# This is deliberately NOT part of deploy.yml. That file is synced into every +# tooling repo from exercism/org-wide-files, so edits to it here are reverted +# on the next sync. It should move there once a second track has one of these +# and it is clear what is actually common between them. +name: Publish client-side runner + +on: + # Chained off Deploy rather than run on push, so the Docker image and the + # browser tarball always ship from the same commit. They are the same + # bin/run.sh, and a student running tests in the browser should be running + # exactly what the server would have run. + workflow_run: + workflows: [Deploy] + types: [completed] + workflow_dispatch: + +permissions: + contents: read + id-token: write # for the OIDC token exchanged for the AWS role + +env: + BUCKET: exercism-test-runners + +jobs: + publish: + name: Publish + # Forks have no role to assume. A failed Deploy means the image never + # shipped, and publishing the tarball then would put the browser ahead of + # the server. And only main publishes, however the run was started: + # latest.json is production, and a branch must not be able to repoint it. + if: >- + github.repository_owner == 'exercism' && + ( + (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') || + (github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' && + github.event.workflow_run.head_branch == 'main') + ) + runs-on: ubuntu-26.04 + steps: + - name: Checkout code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + # workflow_run starts from the default branch, so check out the + # commit Deploy actually built. + ref: ${{ github.event.workflow_run.head_sha || github.ref }} + + - name: Read clientside.json + id: config + run: | + # The track is the repo name, so this file is identical in every + # test runner repo. + track="${GITHUB_REPOSITORY##*/}" + track="${track%-test-runner}" + echo "TRACK=${track}" >> "$GITHUB_ENV" + + # These end up in S3 keys and in later steps' shell, so they are held + # to a strict shape even though the file is on main and trusted. + segment() { + local key="$1" value="$2" + if ! [[ "${value}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then + echo "::error::clientside.json: ${key} must be a single path segment, got '${value}'" + exit 1 + fi + } + + version="$(jq -er '.sysroot.version' clientside.json)" + segment sysroot.version "${version}" + id="$(jq -er '.sysroot.id | select(type == "number" and . == floor and . >= 1)' clientside.json)" + + kernel="$(jq -er '.kernel' clientside.json)" + segment kernel "${kernel}" + + timeout="$(jq -er '.timeout | select(type == "number")' clientside.json)" + + { + echo "sysroot_version=${version}" + echo "sysroot_id=${id}" + # Where this track's sysroot lives, relative to the bucket. One + # place, because it is needed to read the sysroot here and to + # point the browser at it later. + echo "sysroot_prefix=test-runners/sysroot/${track}/${version}/${id}" + echo "kernel=${kernel}" + echo "timeout=${timeout}" + } >> "$GITHUB_OUTPUT" + + - name: Build the tarball + run: bin/build-clientside-tarball.sh test-runner.tar + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@cbe3b392738ccf3f987d68400dafcf4b0624a56c + with: + role-to-assume: ${{ secrets.AWS_CLIENTSIDE_PUBLISH_ROLE_ARN }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Check the sysroot has what we preload + run: | + # preload names paths inside the sysroot. A typo, or a version bump that + # moves a binary, otherwise fails at boot in a student's browser with + # nothing useful in the console. + aws s3 cp "s3://${BUCKET}/${{ steps.config.outputs.sysroot_prefix }}/sysroot.tar" sysroot.tar --quiet + tar -tf sysroot.tar | sed 's|^|/|' | sort > sysroot-paths.txt + + status=0 + while read -r path; do + grep -qxF "${path}" sysroot-paths.txt || { echo "::error::${path} is in preload but not in the sysroot"; status=1; } + done < <(jq -r '.preload[]' clientside.json) + exit "${status}" + + - name: Build boot.json + run: | + # The sysroot publishes the half that describes itself - PATH, HOME, + # the licence - and this repo owns the half that describes the track. + aws s3 cp "s3://${BUCKET}/${{ steps.config.outputs.sysroot_prefix }}/boot.json" base-boot.json --quiet + + jq -s '.[0] as $base | .[1] as $track | + $base + { + env: ($base.env + ($track.env // {})), + preload: ($track.preload // $base.preload) + }' base-boot.json clientside.json > boot.json + + - name: Derive the build id + id: build + run: | + # Content-addressed over everything that lands at the prefix - the + # tarball AND boot.json. Re-running a deploy that changed nothing then + # republishes to the same prefix rather than minting a new one and + # busting every student's cache. And a change to only one of them + # (an env var in clientside.json, say) still gets a new prefix, which + # matters because the old one is cached as immutable for a year. + hash="$(cat test-runner.tar boot.json | sha256sum | cut -c1-12)" + { + echo "hash=${hash}" + # Under the sysroot it was built and checked against, so a listing + # of runner//// is every runner made for that sysroot. + echo "prefix=test-runners/runner/${TRACK}/${{ steps.config.outputs.sysroot_version }}/${{ steps.config.outputs.sysroot_id }}/${hash}" + # latest.json wants a single string for the version. + echo "version=${{ steps.config.outputs.sysroot_version }}-${{ steps.config.outputs.sysroot_id }}-${hash}" + } >> "$GITHUB_OUTPUT" + + - name: Publish the build + id: publish + run: | + prefix="${{ steps.build.outputs.prefix }}" + # Immutable: nothing at this prefix is ever rewritten, because the + # prefix ends in a hash of the contents. + cache="public, max-age=31536000, immutable" + + aws s3 cp test-runner.tar "s3://${BUCKET}/${prefix}/test-runner.tar" \ + --content-type application/x-tar --cache-control "${cache}" + aws s3 cp boot.json "s3://${BUCKET}/${prefix}/boot.json" \ + --content-type application/json --cache-control "${cache}" + + echo "prefix=/${prefix}" >> "$GITHUB_OUTPUT" + + - name: Point the track at it + run: | + # Written last. latest.json is the only mutable object in the tree, + # so until this lands the new build is invisible and students keep + # getting the old one - rather than briefly getting a half-published + # one. + jq -n \ + --arg version "${{ steps.build.outputs.version }}" \ + --argjson timeout "${{ steps.config.outputs.timeout }}" \ + --arg kernel "/test-runners/kernel/${{ steps.config.outputs.kernel }}/" \ + --arg boot "${{ steps.publish.outputs.prefix }}/boot.json" \ + --arg sysroot "/${{ steps.config.outputs.sysroot_prefix }}/sysroot.tar" \ + --arg testRunner "${{ steps.publish.outputs.prefix }}/test-runner.tar" \ + '{type: "kernel", version: $version, timeout: $timeout, + kernel: $kernel, boot: $boot, sysroot: $sysroot, testRunner: $testRunner}' \ + > latest.json + + cat latest.json + + aws s3 cp latest.json "s3://${BUCKET}/test-runners/${TRACK}/latest.json" \ + --content-type application/json --cache-control "public, max-age=60" + + - name: Summarise + run: | + { + echo "### Published \`${{ steps.build.outputs.version }}\`" + echo + echo '```json' + cat latest.json + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/README.md b/README.md index 64ba49e..c88f5f7 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,38 @@ Catch can report the tests results in [JUnit][junit] formatted xml when enabled, This file is parsed with Python and the [junitparser][junitparser-lib] library in the `process.py` script that outputs a `results.json` file that respects the test runners specifications. +## The client-side runner + +The same `bin/run.sh` also runs in the browser, on a wasm kernel that provides a real Linux userland. +There is no second implementation: the kernel's sysroot carries the toolchain, this repo's `bin/run.sh` is untarred into `/opt/test-runner`, and executed exactly as the Docker image runs it. + +`clientside.json` is everything this track says about that: + +| Key | Meaning | +| --- | --- | +| `sysroot` | which published sysroot to run on: the version it carries, and an id to tell rebuilds of the same version apart | +| `kernel` | which kernel build to run on | +| `timeout` | seconds a single run may take before the worker is killed | +| `env` | environment variables this track needs on top of the sysroot's own | +| `preload` | binaries to load at boot rather than fault in on first use | + +Kernels and sysroots are published from [exercism/clientside-tooling][tooling]. +The `kernel` and `sysroot` values are directory names in that repo. + +To build the tarball locally: + +```bash +./bin/build-clientside-tarball.sh test-runner.tar +``` + +It needs GNU tar, because the archive has to be byte-identical between runs: the published path is derived from its hash. + +Publishing happens in `.github/workflows/publish-clientside.yml`, which runs after a successful Deploy so that the Docker image and the browser tarball always come from the same commit. + [test-runner-interface]: https://exercism.org/docs/building/tooling/test-runners/interface [test-runner-docker]: https://exercism.org/docs/building/tooling/test-runners/docker [cmake]: https://cmake.org/ [catch-lib]: https://github.com/catchorg/Catch2 [junit]: https://junit.org/junit5/ [junitparser-lib]: https://github.com/gastlygem/junitparser +[tooling]: https://github.com/exercism/clientside-tooling diff --git a/bin/build-clientside-tarball.sh b/bin/build-clientside-tarball.sh new file mode 100755 index 0000000..debe53b --- /dev/null +++ b/bin/build-clientside-tarball.sh @@ -0,0 +1,114 @@ +#! /bin/bash -e + +# Synopsis: +# Build the tarball that the browser test runner untars into its kernel. +# +# This is the browser's equivalent of the Dockerfile's +# +# WORKDIR /opt/test-runner +# COPY . . +# +# The kernel's sysroot carries the toolchain but not this runner, so the +# runner ships as a tarball that is unpacked at boot. Staging it here rather +# than tarring the repo directly means the archive's layout is exactly what the +# kernel should end up with, and nothing that only makes sense on a developer's +# machine goes along for the ride. +# +# Arguments: +# $1: path to write the tarball to (default: ./test-runner.tar) +# +# Example: +# ./bin/build-clientside-tarball.sh test-runner.tar + +# Only what runs when a student runs their tests. The rest of bin/ is drivers +# for Docker and for this runner's own golden tests in tests/. +# TODO: run.sh calls bin/exercism_parser, which the Dockerfile compiles from +# src/ and include/. It has to be a binary the kernel can execute, so it +# either comes from the sysroot or gets added here once built for it. +CONTENTS=(bin/run.sh) + +# Everything lands under here, matching the Docker image's WORKDIR. +# Explicitly not an absolute path: see the `stage` function. +PREFIX="opt/test-runner" + +main() { + # Made absolute now, because tar's --directory would otherwise resolve a + # relative path inside the staging tree. Not via realpath, which on BSD + # refuses a path that does not exist yet. + local output="${1:-test-runner.tar}" + [[ "${output}" == /* ]] || output="${PWD}/${output}" + mkdir -p "$(dirname "${output}")" + + local tar + tar="$(find_gnu_tar)" + + local staging + staging="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '${staging}'" EXIT + + stage "${staging}" + archive "${tar}" "${staging}" "${output}" + + echo "Wrote ${output} ($(wc -c < "${output}" | tr -d ' ') bytes)" + echo "sha256: $(sha256 "${output}")" +} + +# The reproducibility flags below are GNU-only. BSD tar takes different ones and +# writes different headers, so rather than silently produce a tarball that +# hashes differently on a maintainer's Mac than in CI, insist on GNU tar. +find_gnu_tar() { + local candidate + for candidate in tar gtar; do + if command -v "${candidate}" > /dev/null && "${candidate}" --version 2> /dev/null | grep -q "GNU tar"; then + echo "${candidate}" + return + fi + done + + echo "GNU tar is required (on macOS: brew install gnu-tar)" >&2 + exit 1 +} + +stage() { + local staging="$1" + + local entry + for entry in "${CONTENTS[@]}"; do + mkdir -p "${staging}/${PREFIX}/$(dirname "${entry}")" + # -a preserves the executable bits, which bin/run.sh needs. + cp -a "${entry}" "${staging}/${PREFIX}/${entry}" + done +} + +archive() { + local tar="$1" staging="$2" output="$3" + + # The archive has to be byte-identical between runs: the published S3 path + # is derived from its hash, so an unstable tarball would mint a new + # immutable prefix on every deploy and re-publish the same bytes under a + # new URL. --sort fixes the entry order, --mtime the timestamps, the + # ownership flags strip whoever happened to run the build, and ustar has + # no extension headers that could carry anything else. + "${tar}" \ + --create \ + --file "${output}" \ + --directory "${staging}" \ + --sort=name \ + --mtime="@0" \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + --format=ustar \ + "${PREFIX%%/*}" +} + +sha256() { + if command -v sha256sum > /dev/null; then + sha256sum "$1" | cut -d' ' -f1 + else + shasum -a 256 "$1" | cut -d' ' -f1 + fi +} + +main "$@" diff --git a/clientside.json b/clientside.json new file mode 100644 index 0000000..135acbf --- /dev/null +++ b/clientside.json @@ -0,0 +1,13 @@ +{ + "sysroot": { + "version": "TODO", + "id": 1 + }, + "kernel": "TODO", + "timeout": 30, + "env": {}, + "preload": [ + "/usr/bin/bash", + "/usr/bin/coreutils" + ] +}