From 0fca02f6c8755ff5fbcf508c669e759febae59f8 Mon Sep 17 00:00:00 2001 From: HugoCLSC Date: Thu, 24 Sep 2026 10:54:05 +0100 Subject: [PATCH 1/2] Add GitHub Actions CI/CD and daemon smoke test - ci.yml: scripts/build.sh + smoke test in debian:bookworm (amd64, arm64), full Debug build of all targets, pip install on Python 3.11/3.12 - release.yml: on v* tags, publish prebuilt bookworm daemons as release assets - scripts/smoke_test.py: checks the hello/snapshot handshake and clean shutdown - docs/WORKFLOW.md: what CI checks, how to run it locally, how releases work - pyproject.toml: fix build backend name (scikit_build_core.build), which made `pip install .` fail Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 105 +++++++++++++++++++++++++++++++++ .github/workflows/release.yml | 75 ++++++++++++++++++++++++ README.md | 5 ++ docs/WORKFLOW.md | 106 ++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- scripts/smoke_test.py | 93 +++++++++++++++++++++++++++++ 6 files changed, 385 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 docs/WORKFLOW.md create mode 100755 scripts/smoke_test.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b59184c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,105 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + # The on-device path: scripts/build.sh inside Debian bookworm (what the + # printers run), on both architectures, then a daemon handshake check. + daemon: + name: daemon (bookworm ${{ matrix.arch }}) + strategy: + fail-fast: false + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + container: debian:bookworm + steps: + - name: Install dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config git ca-certificates python3 \ + libusb-1.0-0-dev libudev-dev nlohmann-json3-dev + + - uses: actions/checkout@v4 + + - name: Build (scripts/build.sh) + # Runner speed isn't a Pi's, so this only reports the time; the + # updater's real limit is 60 s on the device. + run: time scripts/build.sh + + - name: Smoke test + run: scripts/smoke_test.py build/device_discoveryd + + # Every target in Debug, including the CLI and the pybind11 module. + full-build: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake pkg-config libusb-1.0-0-dev libudev-dev nlohmann-json3-dev + + - name: Configure + run: cmake -S . -B build-dev -DCMAKE_BUILD_TYPE=Debug + + - name: Build all targets + run: cmake --build build-dev -j"$(nproc)" + + - name: Run CLI + run: build-dev/device_discovery + + - name: Smoke test (Debug daemon) + run: python3 scripts/smoke_test.py build-dev/device_discoveryd + + python: + name: python ${{ matrix.python }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + python: ["3.11", "3.12"] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + cmake pkg-config libusb-1.0-0-dev libudev-dev nlohmann-json3-dev + + - name: pip install . + run: pip install -v . + + - name: Import and scan + run: | + cd /tmp + python - <<'EOF' + import DeviceDiscovery as dd + serial = dd.scan_serial() + usb = dd.scan_all_usb() + print(f"serial={serial}\nusb={usb}") + assert isinstance(serial, list) and isinstance(usb, list) + EOF diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..3e8b6cc --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,75 @@ +name: Release + +# Printers update by git pull + scripts/build.sh, so nothing here is deployed +# automatically. Pushing a v* tag publishes prebuilt bookworm daemons as +# GitHub Release assets, for machines that shouldn't build from source. + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +jobs: + build: + name: build (bookworm ${{ matrix.arch }}) + strategy: + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + container: debian:bookworm + steps: + - name: Install dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config git ca-certificates python3 \ + libusb-1.0-0-dev libudev-dev nlohmann-json3-dev + + - uses: actions/checkout@v4 + + - name: Build + run: scripts/build.sh + + - name: Smoke test + run: scripts/smoke_test.py build/device_discoveryd + + - name: Package + run: | + name="device_discoveryd-${GITHUB_REF_NAME}-bookworm-${{ matrix.arch }}" + mkdir "$name" + cp build/device_discoveryd "$name/" + strip "$name/device_discoveryd" + cp -r systemd udev docs/PROTOCOL.md LICENSE "$name/" + tar czf "$name.tar.gz" "$name" + sha256sum "$name.tar.gz" > "$name.tar.gz.sha256" + + - uses: actions/upload-artifact@v4 + with: + name: daemon-${{ matrix.arch }} + path: device_discoveryd-*.tar.gz* + + publish: + needs: build + runs-on: ubuntu-24.04 + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + merge-multiple: true + + - name: Create release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$GITHUB_REF_NAME" \ + --repo "$GITHUB_REPOSITORY" \ + --title "$GITHUB_REF_NAME" \ + --generate-notes \ + device_discoveryd-*.tar.gz* diff --git a/README.md b/README.md index 2015715..43af68f 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ udev/ optional USB permission rule scripts/install.sh one-time machine setup (sudo) scripts/build.sh incremental daemon-only build (run on every update) docs/PROTOCOL.md socket protocol: the contract with BlocksScreen +docs/WORKFLOW.md CI/CD: what GitHub Actions checks, how releases work +scripts/smoke_test.py daemon handshake check (used by CI) +.github/workflows/ CI and release workflows ANALYSIS.md code analysis and known problems ``` @@ -56,3 +59,5 @@ pip install . ``` Logs under systemd: `journalctl -u device-discoveryd -f`. + +CI runs on every pull request; see [docs/WORKFLOW.md](docs/WORKFLOW.md) for what it checks and how to run the same checks locally. diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md new file mode 100644 index 0000000..2bcae56 --- /dev/null +++ b/docs/WORKFLOW.md @@ -0,0 +1,106 @@ +# CI/CD workflow + +This page covers what GitHub Actions checks on every change, how releases are published, and how both relate to the way printers actually get updates. + +Workflows live in `.github/workflows/`: + +| Workflow | File | Runs on | Purpose | +|---|---|---|---| +| CI | `ci.yml` | push to `main`, every pull request, manual run | Build every target and check that the daemon works | +| Release | `release.yml` | push of a `v*` tag | Publish prebuilt daemons as GitHub Release assets | + +## How printers get updates (not through Actions) + +Printers don't download anything from GitHub Actions. The BlocksScreen updater does a `git pull` of this repo, runs `scripts/build.sh` on the printer, and restarts `device-discoveryd.service` (see the README). So **whatever is on `main` reaches printers on their next update**. CI is the gate that keeps a broken `main` from reaching them, which is why every pull request should be green before merging. + +``` +pull request ──► CI ──► merge to main ──► BlocksScreen updater on each printer + git pull → scripts/build.sh → restart service +tag vX.Y.Z ──► Release ──► GitHub Release with prebuilt daemons (optional) +``` + +## CI (`ci.yml`) + +Three jobs run in parallel. A new push to the same branch or PR cancels the previous run. + +### `daemon (bookworm amd64 / arm64)` + +This job reproduces the on-printer build as closely as possible: + +- It runs in a `debian:bookworm` container, the same Debian release as the printers, on both x86-64 and ARM64 runners. +- It installs the same packages as `scripts/install.sh`. +- It runs `scripts/build.sh`, the exact script the updater runs: a Release build of the daemon only. +- It prints the build time. The updater kills `build.sh` after 60 s on a printer. GitHub runners are faster than a Pi, so this time doesn't enforce that limit, but a jump in it is a warning sign. +- It runs `scripts/smoke_test.py` against the built daemon (see [Smoke test](#smoke-test)). + +### `full-build` + +This job builds every target in Debug on Ubuntu 24.04: the daemon, the `device_discovery` CLI and the pybind11 module. It then runs the CLI once and the smoke test against the Debug daemon. It catches breakage in targets that `build.sh` skips. + +### `python 3.11 / 3.12` + +This job runs `pip install .` (scikit-build-core) and imports `DeviceDiscovery`, calling `scan_serial()` and `scan_all_usb()`. Python 3.11 is the minimum in `pyproject.toml`. + +### Smoke test + +There are no unit tests yet, so `scripts/smoke_test.py` is the main behavioural check. It needs no hardware. It: + +1. starts `device_discoveryd --socket ` +2. connects and reads the first two lines +3. checks that they are `hello` with `proto` 1, then a `snapshot` whose devices have every field listed in [PROTOCOL.md](PROTOCOL.md) +4. sends SIGTERM and checks that the daemon exits with 0 and removes its socket file + +CI runners have no serial boards and few or no USB devices, so the snapshot is usually empty. Hotplug (`added`/`removed`) is **not** tested, because that needs real udev events. + +If you bump `kProtocolVersion` or rename a device field, update `EXPECTED_PROTO` / `DEVICE_FIELDS` in the smoke test along with `docs/PROTOCOL.md`. The test failing there is intended: it's a reminder that BlocksScreen's client has to change too. + +## Running the checks locally + +```bash +# daemon job +scripts/build.sh +scripts/smoke_test.py build/device_discoveryd + +# full-build job +cmake -S . -B build-dev -DCMAKE_BUILD_TYPE=Debug +cmake --build build-dev -j +build-dev/device_discovery +scripts/smoke_test.py build-dev/device_discoveryd + +# python job (inside a venv) +pip install . +cd /tmp && python -c "import DeviceDiscovery as dd; print(dd.scan_serial(), dd.scan_all_usb())" +``` + +To reproduce the bookworm environment exactly: + +```bash +git ls-files -co --exclude-standard | tar cf - -T - | docker run --rm -i debian:bookworm bash -c ' + mkdir /src && cd /src && tar xf - && + apt-get update && apt-get install -y --no-install-recommends build-essential cmake pkg-config git \ + ca-certificates python3 libusb-1.0-0-dev libudev-dev nlohmann-json3-dev && + scripts/build.sh && scripts/smoke_test.py build/device_discoveryd' +``` + +## Releases (`release.yml`) + +To publish a release: + +```bash +git tag v0.1.0 +git push origin v0.1.0 +``` + +The workflow builds the daemon with `scripts/build.sh` in bookworm on amd64 and arm64, runs the smoke test, and creates a GitHub Release with auto-generated notes. Each architecture gets two assets: + +- `device_discoveryd--bookworm-.tar.gz`: the stripped daemon, `systemd/`, `udev/`, `PROTOCOL.md` and `LICENSE` +- a matching `.sha256` checksum file + +Releases are optional and nothing installs them automatically. They exist for machines that shouldn't build from source. Note that the systemd unit in the tarball expects the binary at `/home/blocks/DeviceDiscovery/build/device_discoveryd`, so adjust `ExecStart` if you install it elsewhere. + +Keep the tag in step with the version in `pyproject.toml` and `CMakeLists.txt`. The workflows don't check this. + +## Notes + +- **ARM64 runners** (`ubuntu-24.04-arm`) are free for public repositories. If the repo is private and the organisation's plan doesn't include them, the arm64 jobs will stay queued. Remove those matrix entries, or switch to a self-hosted runner. +- **Branch protection:** to make CI a real gate, require the `CI` checks on `main` under *Settings → Branches*. diff --git a/pyproject.toml b/pyproject.toml index e5e0bc7..3657ad6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires-python = ">=3.11" [build-system] requires = ["scikit-build-core>=0.5", "pybind11>=2.12"] -build-backend = "scikit-build-core.build" +build-backend = "scikit_build_core.build" [tool.scikit-build] cmake.build-type = "Dev" diff --git a/scripts/smoke_test.py b/scripts/smoke_test.py new file mode 100755 index 0000000..437a375 --- /dev/null +++ b/scripts/smoke_test.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Start device_discoveryd, check the handshake from docs/PROTOCOL.md, stop it. + +There are no unit tests, so CI uses this to catch a daemon that crashes on +startup, breaks the hello/snapshot handshake, or doesn't shut down cleanly. +It needs no real hardware: an empty snapshot is fine. + +Usage: scripts/smoke_test.py [path/to/device_discoveryd] +""" +import json +import os +import signal +import socket +import subprocess +import sys +import tempfile +import time + +EXPECTED_PROTO = 1 +DEVICE_FIELDS = { + "name", "manufacturer", "product", "serial_number", "symlink_name", + "device_path", "mcu_type", "interface", "video_path", "can_uuid", + "can_interface", "connection", "firmware", "usb_class", "vendor_id", + "product_id", "vid_hex", "pid_hex", "usb_bus_number", + "usb_device_address", "is_klipper", "is_katapult", +} + + +def fail(msg): + print(f"FAIL: {msg}", file=sys.stderr) + sys.exit(1) + + +def connect(path, timeout=5.0): + deadline = time.monotonic() + timeout + while True: + s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + s.connect(path) + return s + except (FileNotFoundError, ConnectionRefusedError): + s.close() + if time.monotonic() > deadline: + fail(f"daemon never opened {path}") + time.sleep(0.05) + + +def read_lines(sock, count, timeout=5.0): + sock.settimeout(timeout) + buf = b"" + while buf.count(b"\n") < count: + chunk = sock.recv(65536) + if not chunk: + got = buf.count(b"\n") + fail(f"socket closed after {got} of {count} lines") + buf += chunk + return [json.loads(line) for line in buf.split(b"\n")[:count]] + + +def main(): + daemon = sys.argv[1] if len(sys.argv) > 1 else "build/device_discoveryd" + with tempfile.TemporaryDirectory() as tmp: + sock_path = os.path.join(tmp, "dd.sock") + proc = subprocess.Popen([daemon, "--socket", sock_path]) + try: + with connect(sock_path) as s: + hello, snapshot = read_lines(s, 2) + finally: + if proc.poll() is None: + proc.send_signal(signal.SIGTERM) + code = proc.wait(timeout=5) + + if hello.get("type") != "hello": + fail(f"first message is not hello: {hello}") + if hello.get("proto") != EXPECTED_PROTO: + fail(f"proto {hello.get('proto')}, expected {EXPECTED_PROTO} " + "(bumped kProtocolVersion? update this test and PROTOCOL.md)") + if snapshot.get("type") != "snapshot" or not isinstance(snapshot.get("devices"), list): + fail(f"second message is not a snapshot: {snapshot}") + for d in snapshot["devices"]: + missing = DEVICE_FIELDS - d.keys() + if missing: + fail(f"device {d.get('name')!r} missing fields {sorted(missing)}") + if code != 0: + fail(f"daemon exited with {code} on SIGTERM") + if os.path.exists(sock_path): + fail("socket file left behind after shutdown") + + print(f"OK: proto {hello['proto']}, {len(snapshot['devices'])} device(s) in snapshot") + + +if __name__ == "__main__": + main() From 92f361fe204cdc8e0c2fd579e96db8dc744660b6 Mon Sep 17 00:00:00 2001 From: HugoCLSC Date: Thu, 24 Sep 2026 11:01:15 +0100 Subject: [PATCH 2/2] Support prebuilt release installs alongside on-device builds - scripts/fetch_release.sh: download device_discoveryd for this arch from GitHub Releases, verify sha256, install into bin/ - install.sh --prebuilt: runtime deps only, saves the mode in .install-mode; build.sh delegates to fetch_release.sh in that mode so the BlocksScreen updater hook works unchanged - build.sh and fetch_release.sh both install atomically to bin/device_discoveryd and record its origin in bin/VERSION; the systemd unit now runs bin/device_discoveryd - release.yml: version-less asset names (so releases/latest/download works), VERSION file in each tarball, verify job that installs the published release on a clean bookworm container and smoke-tests it - docs: README install modes, WORKFLOW.md update flow and releases Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 4 +-- .github/workflows/release.yml | 54 ++++++++++++++++++++++------ .gitignore | 2 ++ README.md | 22 ++++++++++-- docs/WORKFLOW.md | 50 +++++++++++++++++++------- scripts/build.sh | 18 ++++++++-- scripts/fetch_release.sh | 59 +++++++++++++++++++++++++++++++ scripts/install.sh | 37 +++++++++++++++---- systemd/device-discoveryd.service | 4 ++- 9 files changed, 212 insertions(+), 38 deletions(-) create mode 100755 scripts/fetch_release.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b59184c..027342d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,8 +43,8 @@ jobs: # updater's real limit is 60 s on the device. run: time scripts/build.sh - - name: Smoke test - run: scripts/smoke_test.py build/device_discoveryd + - name: Smoke test (installed binary) + run: scripts/smoke_test.py bin/device_discoveryd # Every target in Debug, including the CLI and the pybind11 module. full-build: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3e8b6cc..ef6174b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,8 +1,11 @@ name: Release -# Printers update by git pull + scripts/build.sh, so nothing here is deployed -# automatically. Pushing a v* tag publishes prebuilt bookworm daemons as -# GitHub Release assets, for machines that shouldn't build from source. +# Pushing a v* tag publishes prebuilt bookworm daemons as GitHub Release +# assets. Printers set up with `install.sh --prebuilt` download them through +# scripts/fetch_release.sh; the rest keep compiling with scripts/build.sh. +# +# Asset names carry no version, so releases/latest/download/ always +# resolves; the version is in the VERSION file inside each tarball. on: push: @@ -37,22 +40,24 @@ jobs: run: scripts/build.sh - name: Smoke test - run: scripts/smoke_test.py build/device_discoveryd + run: scripts/smoke_test.py bin/device_discoveryd - name: Package run: | - name="device_discoveryd-${GITHUB_REF_NAME}-bookworm-${{ matrix.arch }}" - mkdir "$name" - cp build/device_discoveryd "$name/" - strip "$name/device_discoveryd" - cp -r systemd udev docs/PROTOCOL.md LICENSE "$name/" + name="device_discoveryd-bookworm-${{ matrix.arch }}" + mkdir -p dist/"$name" + cp bin/device_discoveryd dist/"$name"/ + strip dist/"$name"/device_discoveryd + echo "$GITHUB_REF_NAME $GITHUB_SHA" > dist/"$name"/VERSION + cp -r systemd udev docs/PROTOCOL.md LICENSE dist/"$name"/ + cd dist tar czf "$name.tar.gz" "$name" sha256sum "$name.tar.gz" > "$name.tar.gz.sha256" - uses: actions/upload-artifact@v4 with: name: daemon-${{ matrix.arch }} - path: device_discoveryd-*.tar.gz* + path: dist/device_discoveryd-*.tar.gz* publish: needs: build @@ -73,3 +78,32 @@ jobs: --title "$GITHUB_REF_NAME" \ --generate-notes \ device_discoveryd-*.tar.gz* + + # Installs the just-published release the way a --prebuilt printer does, + # then runs it. + verify: + name: verify (bookworm ${{ matrix.arch }}) + needs: publish + strategy: + matrix: + include: + - arch: amd64 + runner: ubuntu-24.04 + - arch: arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} + container: debian:bookworm + steps: + - name: Install runtime dependencies + run: | + apt-get update + apt-get install -y --no-install-recommends \ + curl ca-certificates git python3 libusb-1.0-0 libudev1 + + - uses: actions/checkout@v4 + + - name: Fetch release + run: DD_GH_REPO="$GITHUB_REPOSITORY" scripts/fetch_release.sh "$GITHUB_REF_NAME" + + - name: Smoke test + run: scripts/smoke_test.py bin/device_discoveryd diff --git a/.gitignore b/.gitignore index 9413f90..ac6e1d1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ compile_commands.json __pycache__/ *.so *.pyc +bin/ +.install-mode diff --git a/README.md b/README.md index 43af68f..80cca5c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ systemd/ device-discoveryd.service (linked into /etc/systemd/system udev/ optional USB permission rule scripts/install.sh one-time machine setup (sudo) scripts/build.sh incremental daemon-only build (run on every update) +scripts/fetch_release.sh download the prebuilt daemon from GitHub Releases docs/PROTOCOL.md socket protocol: the contract with BlocksScreen docs/WORKFLOW.md CI/CD: what GitHub Actions checks, how releases work scripts/smoke_test.py daemon handshake check (used by CI) @@ -32,16 +33,31 @@ ANALYSIS.md code analysis and known problems ```bash git clone https://github.com/BlocksTechnology/DeviceDiscovery.git ~/DeviceDiscovery -~/DeviceDiscovery/scripts/install.sh # add --usb-access to also install the udev rule +~/DeviceDiscovery/scripts/install.sh # build from source on the printer +~/DeviceDiscovery/scripts/install.sh --prebuilt # or: download the release binary (amd64/arm64) ``` -This installs the build dependencies (`cmake`, `libusb-1.0-0-dev`, `libudev-dev`, `nlohmann-json3-dev`), builds the daemon, and links and enables `device-discoveryd.service`. The unit expects the repo at `/home/blocks/DeviceDiscovery` and runs as `blocks:blocksscreen`. +Add `--usb-access` to either command to also install the udev rule. + +There are two install modes: + +| Mode | Installs | How the daemon gets onto the printer | +|---|---|---| +| **Source** (default) | build deps (`cmake`, `libusb-1.0-0-dev`, `libudev-dev`, `nlohmann-json3-dev`) | `scripts/build.sh` compiles it | +| **Prebuilt** (`--prebuilt`) | runtime libs and `curl` only, no compiler | `scripts/fetch_release.sh` downloads it from [GitHub Releases](https://github.com/BlocksTechnology/DeviceDiscovery/releases) and checks its sha256 | + +Both modes put the daemon in `bin/device_discoveryd` and record where it came from in `bin/VERSION` (`source ` or `release `). The installer then links and enables `device-discoveryd.service`. The unit expects the repo at `/home/blocks/DeviceDiscovery` and runs as `blocks:blocksscreen`. + +The chosen mode is saved in `.install-mode`. To switch, run `install.sh` again with or without `--prebuilt`. ## Updates The BlocksScreen updater manages this repo as the `DeviceDiscovery` component (see BlocksScreen's `updater/components.yaml`). After each git update, BlocksScreen's `updater/hooks/DeviceDiscovery.sh` runs `scripts/build.sh`, and then the updater restarts `device-discoveryd.service`. No sudo is needed after the first install. -`scripts/build.sh` must finish within the updater's 60 s hook timeout. That's why it builds only the daemon, and why it keeps `build/` between updates so only changed files recompile. +`scripts/build.sh` follows the install mode: + +- **Source:** compiles the daemon. It must finish within the updater's 60 s hook timeout. That's why it builds only the daemon, and why it keeps `build/` between updates so only changed files recompile. +- **Prebuilt:** runs `scripts/fetch_release.sh`. If HEAD is exactly on a release tag, it installs that release, and skips the download if it's already installed. Otherwise it installs the latest release, which can be older than the checked-out source. ## Development diff --git a/docs/WORKFLOW.md b/docs/WORKFLOW.md index 2bcae56..6bc832b 100644 --- a/docs/WORKFLOW.md +++ b/docs/WORKFLOW.md @@ -7,18 +7,24 @@ Workflows live in `.github/workflows/`: | Workflow | File | Runs on | Purpose | |---|---|---|---| | CI | `ci.yml` | push to `main`, every pull request, manual run | Build every target and check that the daemon works | -| Release | `release.yml` | push of a `v*` tag | Publish prebuilt daemons as GitHub Release assets | +| Release | `release.yml` | push of a `v*` tag | Publish prebuilt daemons as GitHub Release assets, then test-install them | -## How printers get updates (not through Actions) +## How printers get updates -Printers don't download anything from GitHub Actions. The BlocksScreen updater does a `git pull` of this repo, runs `scripts/build.sh` on the printer, and restarts `device-discoveryd.service` (see the README). So **whatever is on `main` reaches printers on their next update**. CI is the gate that keeps a broken `main` from reaching them, which is why every pull request should be green before merging. +On every update, the BlocksScreen updater does a `git pull` of this repo, runs `scripts/build.sh` on the printer, and restarts `device-discoveryd.service` (see the README). What `build.sh` does depends on how the printer was installed: + +- **Source mode (default):** it compiles the daemon from the pulled code. So **whatever is on `main` reaches these printers on their next update**. CI is the gate that keeps a broken `main` from reaching them, which is why every pull request should be green before merging. +- **Prebuilt mode (`install.sh --prebuilt`):** it runs `scripts/fetch_release.sh`, which downloads the daemon from GitHub Releases. These printers only get new daemon code when you **publish a release**. If HEAD is exactly on a release tag, that release is installed; otherwise the latest release is. ``` -pull request ──► CI ──► merge to main ──► BlocksScreen updater on each printer - git pull → scripts/build.sh → restart service -tag vX.Y.Z ──► Release ──► GitHub Release with prebuilt daemons (optional) +pull request ──► CI ──► merge to main ──► updater on each printer: git pull → scripts/build.sh + ├─ source mode: compile ─┐ +tag vX.Y.Z ──► Release ──► GitHub Release ─────┴─ prebuilt mode: fetch_release.sh ─┴─► bin/device_discoveryd + → restart service ``` +Either way, the daemon ends up in `bin/device_discoveryd`, which is where the systemd unit runs it from. `bin/VERSION` records its origin: `source ` or `release `. + ## CI (`ci.yml`) Three jobs run in parallel. A new push to the same branch or PR cancels the previous run. @@ -31,7 +37,7 @@ This job reproduces the on-printer build as closely as possible: - It installs the same packages as `scripts/install.sh`. - It runs `scripts/build.sh`, the exact script the updater runs: a Release build of the daemon only. - It prints the build time. The updater kills `build.sh` after 60 s on a printer. GitHub runners are faster than a Pi, so this time doesn't enforce that limit, but a jump in it is a warning sign. -- It runs `scripts/smoke_test.py` against the built daemon (see [Smoke test](#smoke-test)). +- It runs `scripts/smoke_test.py` against the installed `bin/device_discoveryd` (see [Smoke test](#smoke-test)). ### `full-build` @@ -59,7 +65,7 @@ If you bump `kProtocolVersion` or rename a device field, update `EXPECTED_PROTO` ```bash # daemon job scripts/build.sh -scripts/smoke_test.py build/device_discoveryd +scripts/smoke_test.py bin/device_discoveryd # full-build job cmake -S . -B build-dev -DCMAKE_BUILD_TYPE=Debug @@ -79,7 +85,7 @@ git ls-files -co --exclude-standard | tar cf - -T - | docker run --rm -i debian: mkdir /src && cd /src && tar xf - && apt-get update && apt-get install -y --no-install-recommends build-essential cmake pkg-config git \ ca-certificates python3 libusb-1.0-0-dev libudev-dev nlohmann-json3-dev && - scripts/build.sh && scripts/smoke_test.py build/device_discoveryd' + scripts/build.sh && scripts/smoke_test.py bin/device_discoveryd' ``` ## Releases (`release.yml`) @@ -91,16 +97,34 @@ git tag v0.1.0 git push origin v0.1.0 ``` -The workflow builds the daemon with `scripts/build.sh` in bookworm on amd64 and arm64, runs the smoke test, and creates a GitHub Release with auto-generated notes. Each architecture gets two assets: +The workflow has three stages: + +1. **build:** runs `scripts/build.sh` in bookworm on amd64 and arm64, smoke-tests the result, and packages it. +2. **publish:** creates the GitHub Release with auto-generated notes. +3. **verify:** on a clean bookworm container per architecture, with runtime libraries only and no compiler, installs the published release with `scripts/fetch_release.sh `, exactly as a prebuilt printer would, and smoke-tests it. -- `device_discoveryd--bookworm-.tar.gz`: the stripped daemon, `systemd/`, `udev/`, `PROTOCOL.md` and `LICENSE` -- a matching `.sha256` checksum file +Each architecture gets two assets: -Releases are optional and nothing installs them automatically. They exist for machines that shouldn't build from source. Note that the systemd unit in the tarball expects the binary at `/home/blocks/DeviceDiscovery/build/device_discoveryd`, so adjust `ExecStart` if you install it elsewhere. +- `device_discoveryd-bookworm-.tar.gz`: the stripped daemon, a `VERSION` file (` `), `systemd/`, `udev/`, `PROTOCOL.md` and `LICENSE` +- a matching `.sha256` checksum, which `fetch_release.sh` checks before installing + +The asset names don't include the version on purpose. That way `https://github.com/BlocksTechnology/DeviceDiscovery/releases/latest/download/` always points at the newest release. Keep the tag in step with the version in `pyproject.toml` and `CMakeLists.txt`. The workflows don't check this. +### Installing a release by hand + +```bash +scripts/fetch_release.sh # the tag at HEAD, or the latest release +scripts/fetch_release.sh v0.1.0 # a specific release +sudo systemctl restart device-discoveryd +``` + +`DD_GH_REPO` points it at a fork. `DD_RELEASE_BASE_URL` downloads from any directory URL holding the assets, for a mirror or for testing. + ## Notes - **ARM64 runners** (`ubuntu-24.04-arm`) are free for public repositories. If the repo is private and the organisation's plan doesn't include them, the arm64 jobs will stay queued. Remove those matrix entries, or switch to a self-hosted runner. +- **Private repo:** `fetch_release.sh` downloads without authentication, so prebuilt mode only works while the repository (and its releases) are public. +- **32-bit ARM** (`armhf`) has no prebuilt binary. On such a printer, `fetch_release.sh` refuses and you need source mode. - **Branch protection:** to make CI a real gate, require the `CI` checks on `main` under *Settings → Branches*. diff --git a/scripts/build.sh b/scripts/build.sh index 320bfca..2a24ef9 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,17 +1,31 @@ #!/bin/bash -# Incremental, daemon-only build into build/. +# Incremental, daemon-only build into build/, installed into bin/ (where +# the systemd unit runs it from). # # Called by BlocksScreen's updater hook (updater/hooks/DeviceDiscovery.sh) # after every git update of this repo, and by install.sh. The updater kills # hooks after 60 s, so this deliberately skips the pybind11 module (slowest # target) and relies on build/ being kept between updates so only changed # files recompile. +# +# On machines set up with `install.sh --prebuilt` it downloads the release +# binary instead (scripts/fetch_release.sh), so the hook works in both modes. set -euo pipefail REPO="$(cd "$(dirname "$0")/.." && pwd)" cd "$REPO" +if [[ "$(cat .install-mode 2>/dev/null)" == "prebuilt" ]]; then + exec scripts/fetch_release.sh +fi + cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DDD_BUILD_PYTHON=OFF >/dev/null cmake --build build --target device_discoveryd -j"$(nproc)" -echo "[DeviceDiscovery] built $REPO/build/device_discoveryd" +# Replace via rename: a running daemon keeps its old inode. +mkdir -p bin +install -m 0755 build/device_discoveryd bin/device_discoveryd.new +mv -f bin/device_discoveryd.new bin/device_discoveryd +echo "source $(git rev-parse --short HEAD 2>/dev/null || echo unknown)" > bin/VERSION + +echo "[DeviceDiscovery] built $REPO/bin/device_discoveryd ($(cat bin/VERSION))" diff --git a/scripts/fetch_release.sh b/scripts/fetch_release.sh new file mode 100755 index 0000000..85a2f8a --- /dev/null +++ b/scripts/fetch_release.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Installs a prebuilt device_discoveryd from GitHub Releases into bin/, +# the alternative to compiling it on the device with scripts/build.sh. +# +# Called by build.sh when the machine was set up with `install.sh --prebuilt`, +# so the BlocksScreen updater hook works unchanged in either mode. +# +# Usage: scripts/fetch_release.sh [TAG] +# TAG release to install, e.g. v0.1.0. Default: the tag at HEAD if HEAD is +# exactly on one (binary matches the checked-out source), otherwise +# the latest release. +# +# Environment: DD_GH_REPO (default BlocksTechnology/DeviceDiscovery), +# DD_RELEASE_BASE_URL (download from here instead of GitHub). +set -euo pipefail + +REPO="$(cd "$(dirname "$0")/.." && pwd)" +GH_REPO="${DD_GH_REPO:-BlocksTechnology/DeviceDiscovery}" + +arch="$(dpkg --print-architecture)" +case "$arch" in + amd64|arm64) ;; + *) + echo "[DeviceDiscovery] no prebuilt daemon for $arch, use scripts/build.sh" >&2 + exit 1 + ;; +esac + +tag="${1:-$(git -C "$REPO" describe --tags --exact-match 2>/dev/null || true)}" +if [[ -n "$tag" ]]; then + if grep -qx "release $tag .*" "$REPO/bin/VERSION" 2>/dev/null; then + echo "[DeviceDiscovery] release $tag already installed" + exit 0 + fi + base="https://github.com/$GH_REPO/releases/download/$tag" +else + base="https://github.com/$GH_REPO/releases/latest/download" +fi +# Override for mirrors and testing: a directory URL holding the assets. +base="${DD_RELEASE_BASE_URL:-$base}" + +name="device_discoveryd-bookworm-$arch" +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT + +# The updater kills hooks after 60 s; keep the downloads well inside that. +curl -fsSL --retry 2 --max-time 20 -o "$tmp/$name.tar.gz" "$base/$name.tar.gz" +curl -fsSL --retry 2 --max-time 10 -o "$tmp/$name.tar.gz.sha256" "$base/$name.tar.gz.sha256" +(cd "$tmp" && sha256sum --check --quiet "$name.tar.gz.sha256") +tar xzf "$tmp/$name.tar.gz" -C "$tmp" + +# Replace via rename so a running daemon keeps its (old) inode and a crash +# mid-copy never leaves a half-written binary for the service to start. +mkdir -p "$REPO/bin" +install -m 0755 "$tmp/$name/device_discoveryd" "$REPO/bin/device_discoveryd.new" +mv -f "$REPO/bin/device_discoveryd.new" "$REPO/bin/device_discoveryd" +echo "release $(cat "$tmp/$name/VERSION")" > "$REPO/bin/VERSION" + +echo "[DeviceDiscovery] installed $(cat "$REPO/bin/VERSION") into $REPO/bin/device_discoveryd" diff --git a/scripts/install.sh b/scripts/install.sh index f9f8268..146db99 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -1,9 +1,13 @@ #!/bin/bash -# One-time machine setup (needs sudo): build dependencies, first build, +# One-time machine setup (needs sudo): dependencies, first daemon install, # systemd unit. After this, updates arrive through the BlocksScreen updater, # which runs scripts/build.sh and restarts the service - no sudo needed. # -# Usage: scripts/install.sh [--usb-access] +# Usage: scripts/install.sh [--prebuilt] [--usb-access] +# --prebuilt download release binaries from GitHub instead of compiling +# on this machine (amd64/arm64 only). Remembered in +# .install-mode, so later updates do the same. Run without +# it to switch back to building from source. # --usb-access also install udev/70-device-discovery-usb.rules # (read the trade-off in that file first) set -euo pipefail @@ -12,24 +16,43 @@ REPO="$(cd "$(dirname "$0")/.." && pwd)" UNIT="device-discoveryd.service" EXPECTED_REPO="/home/blocks/DeviceDiscovery" +prebuilt=0 +usb_access=0 +for arg in "$@"; do + case "$arg" in + --prebuilt) prebuilt=1 ;; + --usb-access) usb_access=1 ;; + *) echo "[DeviceDiscovery] unknown option: $arg" >&2; exit 2 ;; + esac +done + if [[ "$REPO" != "$EXPECTED_REPO" ]]; then echo "[DeviceDiscovery] $UNIT expects the repo at $EXPECTED_REPO, found $REPO" >&2 echo "[DeviceDiscovery] clone it there, or edit ExecStart in systemd/$UNIT" >&2 exit 1 fi -sudo apt-get install -y --no-install-recommends \ - build-essential cmake pkg-config \ - libusb-1.0-0-dev libudev-dev nlohmann-json3-dev +if (( prebuilt )); then + sudo apt-get install -y --no-install-recommends \ + curl ca-certificates libusb-1.0-0 libudev1 + echo prebuilt > "$REPO/.install-mode" +else + sudo apt-get install -y --no-install-recommends \ + build-essential cmake pkg-config \ + libusb-1.0-0-dev libudev-dev nlohmann-json3-dev + rm -f "$REPO/.install-mode" +fi "$REPO/scripts/build.sh" # `link` keeps the unit file in the repo, so unit changes arrive with # git updates and only need a daemon-reload. sudo systemctl link "$REPO/systemd/$UNIT" -sudo systemctl enable --now "$UNIT" +sudo systemctl daemon-reload +sudo systemctl enable "$UNIT" +sudo systemctl restart "$UNIT" -if [[ "${1:-}" == "--usb-access" ]]; then +if (( usb_access )); then sudo install -m 0644 "$REPO/udev/70-device-discovery-usb.rules" /etc/udev/rules.d/ sudo udevadm control --reload-rules sudo udevadm trigger --subsystem-match=usb diff --git a/systemd/device-discoveryd.service b/systemd/device-discoveryd.service index 433aff4..fd3b6bf 100644 --- a/systemd/device-discoveryd.service +++ b/systemd/device-discoveryd.service @@ -18,7 +18,9 @@ RuntimeDirectory=device-discovery RuntimeDirectoryMode=0750 # Socket ends up 0770 -> readable by the blocksscreen group (BlocksScreen.service). UMask=0007 -ExecStart=/home/blocks/DeviceDiscovery/build/device_discoveryd --socket /run/device-discovery/device_discovery.sock +# bin/ holds whichever daemon was installed last: compiled here by +# scripts/build.sh or downloaded by scripts/fetch_release.sh. +ExecStart=/home/blocks/DeviceDiscovery/bin/device_discoveryd --socket /run/device-discovery/device_discovery.sock Restart=on-failure RestartSec=2