diff --git a/.ci/detect-vmnet-network.sh b/.ci/device-smoke/detect-vmnet-network.sh similarity index 100% rename from .ci/detect-vmnet-network.sh rename to .ci/device-smoke/detect-vmnet-network.sh diff --git a/.ci/test-gpu.sh b/.ci/device-smoke/test-gpu.sh similarity index 99% rename from .ci/test-gpu.sh rename to .ci/device-smoke/test-gpu.sh index cda6762d..3fdeb939 100755 --- a/.ci/test-gpu.sh +++ b/.ci/device-smoke/test-gpu.sh @@ -3,7 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "${SCRIPT_DIR}/common.sh" +. "${SCRIPT_DIR}/../common.sh" # Override timeout and sleep duration for macOS - emulation is significantly slower case "${OS_TYPE}" in @@ -15,7 +15,7 @@ case "${OS_TYPE}" in DFB_SLEEP=5 ;; esac -export DFB_SLEEP +export TIMEOUT DFB_SLEEP SEMU_DIRECTFB2_TEST="${SEMU_DIRECTFB2_TEST:-1}" export SEMU_DIRECTFB2_TEST MAKE_CHECK_DISKIMG_ARG="" diff --git a/.ci/test-netdev.sh b/.ci/device-smoke/test-netdev.sh similarity index 97% rename from .ci/test-netdev.sh rename to .ci/device-smoke/test-netdev.sh index fbb72b0c..56250a1d 100755 --- a/.ci/test-netdev.sh +++ b/.ci/device-smoke/test-netdev.sh @@ -3,7 +3,7 @@ # Source common functions and settings SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export SCRIPT_DIR -source "${SCRIPT_DIR}/common.sh" +source "${SCRIPT_DIR}/../common.sh" # Override timeout for netdev tests # Network tests need different timeout: 30s for Linux, 900s for macOS @@ -22,11 +22,8 @@ esac # Clean up any existing semu processes before starting tests cleanup -# Platform detection -UNAME_S=$(uname -s) - # Check if running on macOS -if [[ ${UNAME_S} == "Darwin" ]]; then +if [[ ${OS_TYPE} == "Darwin" ]]; then IS_MACOS=1 else IS_MACOS=0 @@ -123,7 +120,7 @@ else fi fi -echo "Platform: ${UNAME_S}" +echo "Platform: ${OS_TYPE}" echo "Network devices to test: ${NETWORK_DEVICES[@]}" for NETDEV in "${NETWORK_DEVICES[@]}"; do diff --git a/.ci/test-sound.sh b/.ci/device-smoke/test-sound.sh similarity index 96% rename from .ci/test-sound.sh rename to .ci/device-smoke/test-sound.sh index d89ab7e5..448737e3 100755 --- a/.ci/test-sound.sh +++ b/.ci/device-smoke/test-sound.sh @@ -5,7 +5,7 @@ set -euo pipefail # Source common functions and settings SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" export SCRIPT_DIR -source "${SCRIPT_DIR}/common.sh" +source "${SCRIPT_DIR}/../common.sh" SAMPLE_SOUND="/usr/share/sounds/alsa/Front_Center.wav" diff --git a/.ci/test-vinput.sh b/.ci/device-smoke/test-vinput.sh similarity index 98% rename from .ci/test-vinput.sh rename to .ci/device-smoke/test-vinput.sh index 7c0dae15..9ed7f58b 100755 --- a/.ci/test-vinput.sh +++ b/.ci/device-smoke/test-vinput.sh @@ -3,7 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -. "${SCRIPT_DIR}/common.sh" +. "${SCRIPT_DIR}/../common.sh" # Override timeout for macOS - emulation is significantly slower case "${OS_TYPE}" in diff --git a/.ci/github/append-github-output.sh b/.ci/github/append-github-output.sh new file mode 100755 index 00000000..15fc73b5 --- /dev/null +++ b/.ci/github/append-github-output.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# GitHub Actions adapter for resolver/materializer KEY=VALUE files. The +# prebuilt scripts write machine-readable env-style plans to normal files; this +# helper is the only place that copies selected keys into $GITHUB_OUTPUT. It +# rejects non-output lines and fails if a requested key is missing, so workflow +# plumbing breaks at the boundary instead of silently producing empty outputs. + +set -euo pipefail + +if [ "$#" -lt 1 ]; then + echo "Usage: append-github-output.sh [allowed-key ...]" >&2 + exit 1 +fi + +file=$1 +shift + +: "${GITHUB_OUTPUT:?Need GITHUB_OUTPUT from GitHub Actions}" + +tmp=$(mktemp) +cleanup() { + rm -f "$tmp" +} +trap cleanup EXIT + +if awk -v keys="$*" ' + BEGIN { + has_filter = (keys != "") + count = split(keys, key_list, " ") + for (i = 1; i <= count; i++) { + if (key_list[i] != "") allowed[key_list[i]] = 1 + } + } + /^[A-Za-z_][A-Za-z0-9_]*=/ { + key = $0 + sub(/=.*/, "", key) + if (!has_filter || key in allowed) { + print + seen[key] = 1 + } + next + } + { + print "::error::Unexpected non-output line: " $0 > "/dev/stderr" + bad = 1 + } + END { + if (has_filter) { + for (i = 1; i <= count; i++) { + key = key_list[i] + if (key != "" && !(key in seen)) { + print "::error::Missing requested output key: " key > "/dev/stderr" + bad = 1 + } + } + } + exit bad + } +' "$file" > "$tmp"; then + cat "$tmp" >> "$GITHUB_OUTPUT" +else + exit 1 +fi diff --git a/.ci/suggest-format.sh b/.ci/github/suggest-format.sh similarity index 92% rename from .ci/suggest-format.sh rename to .ci/github/suggest-format.sh index bf1d6a36..813ae2fd 100755 --- a/.ci/suggest-format.sh +++ b/.ci/github/suggest-format.sh @@ -14,8 +14,8 @@ set -euo pipefail -# Collect all C/C++ sources tracked by git (POSIX-compatible array build -# for bash 3.2 compatibility on macOS runners). +# Collect all C/C++ sources tracked by git into a Bash array without +# word-splitting filenames. SOURCES=() while IFS= read -r file; do SOURCES+=("$file") diff --git a/.ci/prebuilt/artifact-inputs.sh b/.ci/prebuilt/artifact-inputs.sh new file mode 100755 index 00000000..07b8a6cd --- /dev/null +++ b/.ci/prebuilt/artifact-inputs.sh @@ -0,0 +1,356 @@ +#!/usr/bin/env bash + +# Helper library for the CI prebuilt flow. Resolver, materializer, packager, +# stamp helper, and tests source this file to share CI recipe-key helpers and +# resolver plan naming. Local default make flows do not use these helpers. +# +# Stable artifact class identity and raw output filenames live in +# scripts/prebuilt/artifact-classes.sh because they are source-build contract, +# not CI policy. This file adapts that contract for CI recipe keys and plans. +# +# File recipe inputs stay in prebuilt_class_inputs() because they are CI +# cache-key metadata for the shared source-build API and are easier to review as +# explicit per-class lists. For variables in +# scripts/prebuilt/artifact-recipe.env, each class hashes only the KEY=VALUE +# lines that affect it. That keeps CI invalidation per-class without splitting +# the local build recipe into multiple env files. + +# New artifact classes are declared in scripts/prebuilt/artifact-classes.sh and +# built in scripts/prebuilt/artifact-recipes.sh. This file only adds the CI +# recipe-key inputs/env vars for those shared classes. +. "$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../scripts/prebuilt" && pwd)/artifact-classes.sh" + +# test-tools recipe selection is build behavior first and CI key metadata +# second. Source the build-owned helper so local builds and CI keys cannot drift. +. "$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")/../../scripts/prebuilt" && pwd)/test-tools-recipe.sh" + +# List real files that affect one class recipe key. Recipe variables from +# scripts/prebuilt/artifact-recipe.env are handled separately by +# prebuilt_class_recipe_env_vars(). Keep this synchronized with +# scripts/prebuilt/artifact-recipes.sh: if a source-build recipe starts reading +# a new file, add it here or CI can silently reuse a stale raw artifact. +prebuilt_class_inputs() { + case "$1" in + image) + printf '%s\n' \ + configs/buildroot.config \ + configs/linux.config \ + scripts/prebuilt/artifact-recipes.sh + ;; + rootfs) + printf '%s\n' \ + configs/buildroot.config \ + configs/busybox.config \ + scripts/prebuilt/artifact-recipes.sh \ + target/init + ;; + test-tools) + prebuilt_test_tools_recipe_key >/dev/null || return 1 + printf '%s\n' \ + configs/buildroot.config \ + configs/busybox.config \ + scripts/prebuilt/artifact-recipes.sh \ + scripts/prebuilt/test-tools-recipe.sh \ + scripts/rootfs_ext4.sh \ + target/init + if prebuilt_test_tools_recipe_includes x11; then + printf '%s\n' configs/x11.config + fi + if prebuilt_test_tools_recipe_includes directfb2; then + printf '%s\n' \ + configs/meson-riscv-cross-file \ + target/local-env.sh + fi + ;; + *) + echo "[!] Unknown prebuilt input class: $1" >&2 + return 1 + ;; + esac +} + +# List the de-duplicated real-file inputs across all classes. Use this when +# a caller needs a whole-system source-input set, such as package validation or +# test fixture setup. +prebuilt_inputs() { + local class + + while IFS= read -r class; do + prebuilt_class_inputs "$class" + done < <(source_artifact_classes) | awk '!seen[$0]++' +} + +# Resolver plans are shell-friendly KEY=VALUE files. Class-scoped keys are +# derived from the class name, using underscores where shell variable names +# cannot use dashes. The resolver emits action and current_recipe_key fields; +# release/local fields are internal CI decision state. +prebuilt_plan_class_var_name() { + local class=$1 + local kind=$2 + local prefix + + source_artifact_class_outputs "$class" >/dev/null || return 1 + prefix=${class//-/_} + case "$kind" in + action) printf '%s_action\n' "$prefix" ;; + current_recipe_key) printf 'current_%s_recipe_key\n' "$prefix" ;; + release_recipe_key) printf 'release_%s_recipe_key\n' "$prefix" ;; + local_recipe_key) printf 'local_%s_recipe_key\n' "$prefix" ;; + local_status) printf 'local_%s_status\n' "$prefix" ;; + *) echo "[!] Unknown prebuilt plan variable kind: $kind" >&2; return 1 ;; + esac +} + +# Set one generated class-scoped plan variable. Use this instead of spelling +# out image_action/rootfs_action/test_tools_action style names at each caller. +prebuilt_plan_set_class_var() { + local class=$1 + local kind=$2 + local value=$3 + local var + + var=$(prebuilt_plan_class_var_name "$class" "$kind") || return 1 + printf -v "$var" '%s' "$value" +} + +# Read one generated class-scoped plan variable. Missing variables read as an +# empty value so parsers can validate required fields explicitly. +prebuilt_plan_get_class_var() { + local class=$1 + local kind=$2 + local var + + var=$(prebuilt_plan_class_var_name "$class" "$kind") || return 1 + printf '%s\n' "${!var-}" +} + +# Print one plan field for every class as KEY=VALUE lines. Resolver output +# uses this to stay machine-readable without hard-coded class names. +prebuilt_plan_print_class_values() { + local kind=$1 + local class + + while IFS= read -r class; do + printf '%s=%s\n' \ + "$(prebuilt_plan_class_var_name "$class" "$kind")" \ + "$(prebuilt_plan_get_class_var "$class" "$kind")" + done < <(source_artifact_classes) +} + +# Test whether a plan key is one of the generated variable names for a class. +# Plan parsers use this to recognize class-scoped fields without duplicating the +# naming rules. +prebuilt_plan_class_var_matches() { + local plan_key=$1 + local class=$2 + local kind + shift 2 + + for kind in "$@"; do + if [ "$plan_key" = "$(prebuilt_plan_class_var_name "$class" "$kind")" ]; then + return 0 + fi + done + + return 1 +} + +# List prebuilt.sha1 recipe-key entries for all artifacts in one class. +prebuilt_class_recipe_entries() { + local artifact + + while IFS= read -r artifact; do + printf '%s.recipe-key\n' "$artifact" + done < <(source_artifact_class_outputs "$1") +} + +# Local recipe-key stamps live with other generated build metadata. The raw +# artifacts stay at the repository root; this directory stores only metadata +# used to decide whether restored local artifacts match the current recipe. +prebuilt_local_stamp_dir=.stamps/prebuilt-local + +# Map a raw artifact filename to the CI cache stamp that records its recipe key. +prebuilt_artifact_recipe_stamp() { + printf '%s/%s.recipe-key\n' "$prebuilt_local_stamp_dir" "$1" +} + +# Recipe keys are produced on Linux and consumed again on Linux/macOS test +# jobs. Keep this helper limited to sha1sum-compatible output over explicit +# relative paths so GNU sha1sum and macOS shasum generate identical manifests. +prebuilt_sha1sum() { + if command -v sha1sum >/dev/null 2>&1; then + sha1sum "$@" + elif command -v shasum >/dev/null 2>&1; then + shasum -a 1 "$@" + else + echo "[!] Need sha1sum or shasum on PATH" >&2 + return 1 + fi +} + +# CI raw artifact cache keys combine artifact recipe keys with this schema tag. +# Recipe keys cover artifact bytes; these files cover CI planning and +# materialization behavior that can change how restored raw artifacts are +# interpreted. +prebuilt_ci_cache_schema_inputs() { + printf '%s\n' \ + .ci/prebuilt/artifact-inputs.sh \ + .ci/prebuilt/build-plan-artifacts.sh \ + .ci/prebuilt/resolve-artifacts.sh \ + .ci/prebuilt/stamp-artifacts.sh \ + .ci/prebuilt/materialize-artifacts.sh +} + +prebuilt_ci_cache_schema_tag() { + local -a inputs=() + local input + + while IFS= read -r input; do + inputs+=("$input") + done < <(prebuilt_ci_cache_schema_inputs) + + for input in "${inputs[@]}"; do + if [ ! -f "$input" ]; then + echo "[!] Missing prebuilt CI cache schema input $input" >&2 + return 1 + fi + done + + prebuilt_sha1sum "${inputs[@]}" | prebuilt_sha1sum | awk '{print $1}' +} + +# Return the shared local build recipe env file path. CI hashes selected lines +# from this file, but the local build keeps one recipe file. +prebuilt_recipe_env_file() { + printf '%s\n' scripts/prebuilt/artifact-recipe.env +} + +# Read exactly one KEY=VALUE line from the recipe env file. Missing or +# duplicate variables are errors because recipe keys must be deterministic. +prebuilt_recipe_env_line() { + local name=$1 + local env_file + local line + + env_file=$(prebuilt_recipe_env_file) + if [ ! -f "$env_file" ]; then + echo "[!] Missing prebuilt recipe env $env_file" >&2 + return 1 + fi + line=$(grep -E "^${name}=" "$env_file" || true) + if [ -z "$line" ]; then + echo "[!] Missing prebuilt recipe variable $name in $env_file" >&2 + return 1 + fi + if [ "$(printf '%s\n' "$line" | wc -l | tr -d ' ')" != 1 ]; then + echo "[!] Duplicate prebuilt recipe variable $name in $env_file" >&2 + return 1 + fi + printf '%s\n' "$line" +} + +# List recipe env variables that affect one class. Keep this synchronized with +# artifact-recipes.sh and artifact-recipe.env; missed variables make CI +# invalidation stale even though local source builds still see the new value. +prebuilt_class_recipe_env_vars() { + local class=$1 + + case "$class" in + image) + printf '%s\n' \ + BUILDROOT_REPO \ + BUILDROOT_REV \ + LINUX_REPO \ + LINUX_REV + ;; + rootfs) + printf '%s\n' \ + BUILDROOT_REPO \ + BUILDROOT_REV + ;; + test-tools) + printf '%s\n' \ + BUILDROOT_REPO \ + BUILDROOT_REV \ + TEST_TOOLS_SIZE_MB + if prebuilt_test_tools_recipe_includes directfb2; then + printf '%s\n' \ + DIRECTFB2_REPO \ + DIRECTFB2_REV \ + DIRECTFB_EXAMPLES_REPO \ + DIRECTFB_EXAMPLES_REV + fi + ;; + *) + echo "[!] Unknown prebuilt input class: $class" >&2 + return 1 + ;; + esac +} + +# Hash one recipe env KEY=VALUE line and label it in the class recipe manifest. +# The label records which recipe variable affected the key. +prebuilt_recipe_env_virtual_entry() { + local name=$1 + local line + + line=$(prebuilt_recipe_env_line "$name") || return 1 + printf '%s\n' "$line" | prebuilt_sha1sum | \ + awk -v name="$name" '{print $1 " artifact-recipe.env:" name}' +} + +# Emit non-file recipe inputs for one class, such as selected recipe env lines +# and the test-tools recipe selection. +prebuilt_class_recipe_virtual_entries() { + local class=$1 + local value + local recipe_var + + while IFS= read -r recipe_var; do + prebuilt_recipe_env_virtual_entry "$recipe_var" + done < <(prebuilt_class_recipe_env_vars "$class") + + case "$class" in + test-tools) + value=$(prebuilt_test_tools_recipe_key) || return 1 + printf '%s' "$value" | prebuilt_sha1sum | \ + awk '{print $1 " test-tools.recipe"}' + ;; + esac +} + +# Build the full recipe manifest for one class. It contains hashes for real +# input files plus the non-file inputs that also affect the artifact bytes. +prebuilt_class_recipe_manifest() { + local class=$1 + local -a inputs=() + local input + + while IFS= read -r input; do + inputs+=("$input") + done < <(prebuilt_class_inputs "$class") + + if [ "${#inputs[@]}" -eq 0 ]; then + echo "[!] Prebuilt input class $class has no inputs" >&2 + return 1 + fi + + for input in "${inputs[@]}"; do + if [ ! -f "$input" ]; then + echo "[!] Missing prebuilt input $input" >&2 + return 1 + fi + done + + prebuilt_sha1sum "${inputs[@]}" + prebuilt_class_recipe_virtual_entries "$class" +} + +# Collapse one class recipe manifest into the single SHA-1 key used by resolver +# decisions, release manifests, and CI cache stamps. +prebuilt_class_recipe_key() { + local manifest + + manifest=$(prebuilt_class_recipe_manifest "$1") || return 1 + printf '%s\n' "$manifest" | prebuilt_sha1sum | awk '{print $1}' +} diff --git a/.ci/prebuilt/build-plan-artifacts.sh b/.ci/prebuilt/build-plan-artifacts.sh new file mode 100755 index 00000000..67ee5374 --- /dev/null +++ b/.ci/prebuilt/build-plan-artifacts.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Internal prebuilt builder. It builds raw artifacts requested by a resolver plan +# and deliberately does not write .stamps/prebuilt-local stamps; the materializer commits +# stamps after the plan action succeeds. Keeping this separate from the +# user-facing build-artifacts.sh avoids feeding local CLI side effects back +# into the CI materialization layer. + +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH='' cd -- "$SCRIPT_DIR/../.." && pwd) +cd "$REPO_ROOT" + +# Reuse the build recipe library, but keep stamping outside this wrapper. The +# materializer owns the commit point because only it knows which resolver action +# completed successfully. +. "$REPO_ROOT/scripts/prebuilt/artifact-recipes.sh" + +build_prebuilt_artifact_classes "$@" diff --git a/.ci/prebuilt/materialize-artifacts.sh b/.ci/prebuilt/materialize-artifacts.sh new file mode 100755 index 00000000..c2de571d --- /dev/null +++ b/.ci/prebuilt/materialize-artifacts.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +# +# CI executor for a resolver plan. It deliberately does not run the resolver or +# accept class arguments: the resolver plans all registered classes, then callers +# pass PREBUILT_PLAN_FILE here. Local Make builds bypass this layer entirely. + +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH='' cd -- "$SCRIPT_DIR/../.." && pwd) +cd "$REPO_ROOT" + +# Load the class/plan registry and stamp writer as the materializer's shared +# contract with the resolver. The materializer consumes plan keys from the +# registry and commits stamps only after download/build actions succeed. +{ + . "$SCRIPT_DIR/artifact-inputs.sh" + . "$SCRIPT_DIR/stamp-artifacts.sh" +} + +if [ "$#" -ne 0 ]; then + echo "[!] materialize-artifacts.sh does not accept class arguments; pass them when creating the plan." >&2 + exit 1 +fi + +# Downloading consumes an already-decided release action. After every raw +# artifact in the class lands, the materializer commits the recipe-key stamp so +# later CI cache restores can be trusted as use-action-cache actions. Release +# archive downloads are transport, not recipe verification; the resolver has +# already checked the release recipe key against current source state. +download_release_class() { + local class=$1 + local recipe_key=$2 + local artifact + + while IFS= read -r artifact; do + scripts/prebuilt/download-release-artifact.sh "$artifact" "$recipe_key" || { + echo "[!] Failed to materialize $class from release" >&2 + exit 1 + } + done < <(source_artifact_class_outputs "$class") + prebuilt_stamp_artifacts_for_class "$class" "$recipe_key" +} + +# Build actions go through the internal CI builder, not scripts/build-artifacts.sh. +# build-artifacts.sh is the user-facing local CLI; the materializer stamps only +# after the plan build action succeeds. +build_plan_classes() { + local -a args=() + local class + + while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = build ]; then + args+=("$class") + fi + done < <(source_artifact_classes) + + if [ "${#args[@]}" -eq 0 ]; then + return 0 + fi + + .ci/prebuilt/build-plan-artifacts.sh "${args[@]}" >&2 +} + +# Materialization consumes plan_version, fixed workflow scalars, *_action, and +# current_*_recipe_key. Unknown future keys fail fast so the resolver/materializer +# plan schema cannot silently drift. +parse_resolver_output() { + local line + local plan_key + local value + local class + local matched + + while IFS= read -r line; do + plan_key=${line%%=*} + value=${line#*=} + case "$plan_key" in + plan_version) plan_version=$value; continue ;; + requires_build) + continue + ;; + platform_action_cache_tag|\ + ci_cache_schema_tag|\ + release_needs_update) + continue + ;; + esac + + matched=false + while IFS= read -r class; do + if prebuilt_plan_class_var_matches "$plan_key" "$class" action current_recipe_key; then + printf -v "$plan_key" '%s' "$value" + matched=true + break + fi + done < <(source_artifact_classes) + + if [ "$matched" = true ]; then + continue + fi + + echo "[!] Unknown prebuilt plan key: $plan_key" >&2 + return 1 + done +} + +# The resolver is intentionally a separate step. Requiring an explicit +# plan file makes materialization reproducible and lets workflow steps pass the +# same plan through cache restore, build, and test jobs. +if [ -z "${PREBUILT_PLAN_FILE:-}" ]; then + echo "[!] PREBUILT_PLAN_FILE is required; create a CI resolver plan before materializing." >&2 + exit 1 +fi +if [ ! -f "$PREBUILT_PLAN_FILE" ]; then + echo "[!] PREBUILT_PLAN_FILE does not exist: $PREBUILT_PLAN_FILE" >&2 + exit 1 +fi +resolver_output=$(cat "$PREBUILT_PLAN_FILE") + +# Test jobs use this guard when they are supposed to consume only existing +# workflow artifacts or the rolling release. A build action in that context is +# a planner/publish-state problem, not something the test job should repair. +case "${PREBUILT_FORBID_BUILD:-0}" in + 1|true) + forbid_build=true + ;; + 0|false|'') + forbid_build=false + ;; + *) + echo "[!] PREBUILT_FORBID_BUILD must be 0/1 or true/false, got: ${PREBUILT_FORBID_BUILD:-}" >&2 + exit 1 + ;; +esac + +# Initialize the class-scoped fields that are required for execution. +# parse_resolver_output fills them from the plan and leaves missing values empty +# so the validation below can report a precise schema error. +plan_version= +while IFS= read -r class; do + prebuilt_plan_set_class_var "$class" action '' + prebuilt_plan_set_class_var "$class" current_recipe_key '' +done < <(source_artifact_classes) + +parse_resolver_output <<< "$resolver_output" + +# Version the plan format even though it is just KEY=VALUE text. That lets a +# future resolver change fail loudly instead of being interpreted by an older +# materializer with subtly different semantics. +if [ "$plan_version" != 1 ]; then + echo "[!] Unsupported prebuilt plan version: ${plan_version:-missing}" >&2 + exit 1 +fi + +# Validate every class before doing any work. This keeps a malformed plan +# from partially downloading or building one class before failing on another. +while IFS= read -r class; do + action=$(prebuilt_plan_get_class_var "$class" action) + case "$action" in + build|use-action-cache|download-release) + ;; + '') + echo "[!] Missing action for prebuilt class: $class" >&2 + exit 1 + ;; + *) + echo "[!] Unsupported action for prebuilt class $class: $action" >&2 + exit 1 + ;; + esac + + case "$action" in + build|download-release) + if [ -z "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" ]; then + echo "[!] Missing current recipe key for prebuilt class $class with action $action" >&2 + exit 1 + fi + ;; + esac +done < <(source_artifact_classes) + +# Collapse per-class actions into a single branch for dependency setup and +# PREBUILT_FORBID_BUILD enforcement. use-action-cache and download-release do +# not need source-build dependencies. +requires_build=false +while IFS= read -r class; do + case "$(prebuilt_plan_get_class_var "$class" action)" in + build) requires_build=true ;; + esac +done < <(source_artifact_classes) + +# When source builds are forbidden, report the exact class actions that +# violated the contract so the workflow log points back to the resolver state. +if [ "$requires_build" = true ] && [ "$forbid_build" = true ]; then + echo "[!] PREBUILT_FORBID_BUILD forbids source builds, but the plan contains:" >&2 + while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = build ]; then + printf '[!] %s=build\n' "$(prebuilt_plan_class_var_name "$class" action)" >&2 + fi + done < <(source_artifact_classes) + exit 1 +fi + +# Build all planned source-build classes first. The internal builder only +# creates raw artifacts; stamps are written here after the whole build action +# succeeds so future cache restores do not trust half-built outputs. +if [ "$requires_build" = true ]; then + build_plan_classes + while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = build ]; then + prebuilt_stamp_artifacts_for_class "$class" "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" + fi + done < <(source_artifact_classes) +fi + +# Finally materialize release-backed classes. use-action-cache classes need +# no work here because the workflow cache restore already placed both raw +# artifacts and .stamps/prebuilt-local stamps in the workspace. +while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = download-release ]; then + download_release_class "$class" "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" + fi +done < <(source_artifact_classes) diff --git a/.ci/prebuilt/package.sh b/.ci/prebuilt/package.sh new file mode 100755 index 00000000..72139c84 --- /dev/null +++ b/.ci/prebuilt/package.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Compress raw prebuilt guest artifacts and write prebuilt.sha1. +# This script does not publish anything to a remote service. The manifest records +# both archive byte checksums for transport integrity and recipe-key virtual +# entries for resolver freshness decisions. +# +# Inputs (in cwd): +# raw guest artifacts listed by .ci/prebuilt/artifact-inputs.sh +# plus the source inputs listed by .ci/prebuilt/artifact-inputs.sh +# +# Outputs (in cwd): +# .bz2 for each raw guest artifact +# prebuilt.sha1 -- sha1sum-format manifest. .bz2 lines record +# archive bytes; *.recipe-key lines record the source +# recipe fingerprint for each artifact class. +# +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) + +# Package layout is derived from the same class registry as planning and +# materialization, so the release manifest follows class additions without a +# second hard-coded artifact list in this script. +. "$SCRIPT_DIR/artifact-inputs.sh" + +require_raw_artifacts() { + local class + local artifact + + while IFS= read -r class; do + while IFS= read -r artifact; do + if [ ! -f "$artifact" ]; then + echo "[!] Missing $artifact -- materialize it in the CI guest-artifact-build job first" >&2 + exit 1 + fi + done < <(source_artifact_class_outputs "$class") + done < <(source_artifact_classes) +} + +require_source_inputs() { + local -a inputs=() + local input + + while IFS= read -r input; do + inputs+=("$input") + done < <(prebuilt_inputs) + + for input in "${inputs[@]}"; do + if [ ! -f "$input" ]; then + echo "[!] Missing prebuilt source input $input -- check repository checkout and artifact input lists" >&2 + exit 1 + fi + done +} + +tmp_archives=() +cleanup_package_parts() { + if [ "${#tmp_archives[@]}" -gt 0 ]; then + rm -f "${tmp_archives[@]}" + fi +} +trap cleanup_package_parts EXIT + +compress_artifact() { + local artifact=$1 + local archive=${artifact}.bz2 + local part=${archive}.part + + tmp_archives+=("$part") + if ! bzip2 -c "$artifact" > "$part"; then + rm -f "$part" + return 1 + fi + mv -f "$part" "$archive" +} + +compress_class_artifacts() { + local class=$1 + local artifact + + while IFS= read -r artifact; do + compress_artifact "$artifact" + done < <(source_artifact_class_outputs "$class") +} + +write_archive_checksum_entries_for_class() { + local class=$1 + local artifact + + while IFS= read -r artifact; do + prebuilt_sha1sum "${artifact}.bz2" + done < <(source_artifact_class_outputs "$class") +} + +write_recipe_key_entries_for_class() { + local class=$1 + local recipe_key=$2 + local entry + + while IFS= read -r entry; do + printf '%s %s\n' "$recipe_key" "$entry" + done < <(prebuilt_class_recipe_entries "$class") +} + +require_raw_artifacts +require_source_inputs + +# Package raw artifacts first, then write archive checksums and recipe-key +# entries from the same class registry so archive names and manifest entries stay +# aligned. +while IFS= read -r class; do + compress_class_artifacts "$class" +done < <(source_artifact_classes) + +{ + while IFS= read -r class; do + write_archive_checksum_entries_for_class "$class" + done < <(source_artifact_classes) + while IFS= read -r class; do + write_recipe_key_entries_for_class "$class" "$(prebuilt_class_recipe_key "$class")" + done < <(source_artifact_classes) +} > prebuilt.sha1 diff --git a/.ci/prebuilt/resolve-artifacts.sh b/.ci/prebuilt/resolve-artifacts.sh new file mode 100755 index 00000000..df6fdbfd --- /dev/null +++ b/.ci/prebuilt/resolve-artifacts.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +# +# CI planner for prebuilt guest artifacts. It never mutates raw artifacts; it +# compares the current recipe keys, the rolling-release recipe keys, and any +# restored CI-cache stamps for all registered classes, then prints a KEY=VALUE +# plan for the CI materializer. Local Make builds do not use this script. + +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH='' cd -- "$SCRIPT_DIR/../.." && pwd) +cd "$REPO_ROOT" + +# Load the shared artifact class contract plus CI recipe-key helpers. The build +# recipe itself lives under scripts/prebuilt; this file only describes +# cache/release keys and resolver decisions for CI. +. "$SCRIPT_DIR/artifact-inputs.sh" + +# This is only for fork/mirror first-run bootstrap when their rolling prebuilt +# release has never existed. Other manifest fetch failures stay fatal. +case "${PREBUILT_BOOTSTRAP_ON_404:-0}" in + 1|true) + bootstrap_on_404=true + ;; + 0|false|'') + bootstrap_on_404=false + ;; + *) + echo "[!] PREBUILT_BOOTSTRAP_ON_404 must be 0/1 or true/false, got: ${PREBUILT_BOOTSTRAP_ON_404:-}" >&2 + exit 1 + ;; +esac + +# Keep the resolver output comparable between jobs by always planning the +# full registered artifact set. Callers that only need one artifact should still +# consume the class they need from the complete plan. +if [ "$#" -ne 0 ]; then + echo "[!] resolve-artifacts.sh does not accept class arguments; it resolves all artifact classes." >&2 + exit 1 +fi + +tmpdir=$(mktemp -d) +trap 'rm -rf "$tmpdir"' EXIT + +# The release manifest is downloaded into a temp directory so failed or +# partial network fetches never touch workspace artifacts. +manifest=$tmpdir/prebuilt.sha1 +manifest_url= +release_manifest_http_code= + +while IFS= read -r class; do + prebuilt_plan_set_class_var "$class" release_recipe_key '' +done < <(source_artifact_classes) + +# prebuilt.sha1 stores archive checksum entries plus recipe-key virtual entries +# such as Image.recipe-key. The resolver uses recipe keys for freshness decisions +# and requires archive checksum entries to be present; manifests that lack them +# are outside the current release contract and resolve as stale. +manifest_entry_value() { + local entry=$1 + + awk -v f="$entry" '$2 == f {print $1; found = 1; exit} END {exit !found}' "$manifest" +} + +manifest_class_recipe_key() { + local class=$1 + local artifact + local next + local recipe_key= + local have_key=false + + while IFS= read -r artifact; do + if ! manifest_entry_value "${artifact}.bz2" >/dev/null; then + printf '\n' + return + fi + if ! next=$(manifest_entry_value "${artifact}.recipe-key"); then + printf '\n' + return + fi + if [ "$have_key" = false ]; then + recipe_key=$next + have_key=true + elif [ "$recipe_key" != "$next" ]; then + printf '\n' + return + fi + done < <(source_artifact_class_outputs "$class") + + printf '%s\n' "$recipe_key" +} + +# Fetch the rolling release manifest and project it into per-class +# release_*_recipe_key fields. Missing per-class entries are not fatal here; +# they simply make that class fall through to a source build decision. +fetch_release_manifest() { + local http_code + local curl_err=$tmpdir/curl.err + local class + + : "${PREBUILT_URL:?Need PREBUILT_URL pointing at the prebuilt artifact directory}" + manifest_url="${PREBUILT_URL}/prebuilt.sha1" + if http_code=$(curl --fail --silent --show-error --retry 3 --retry-delay 1 \ + --write-out '%{http_code}' -L -o "$manifest" "$manifest_url" 2>"$curl_err"); then + release_manifest_http_code=$http_code + while IFS= read -r class; do + prebuilt_plan_set_class_var "$class" release_recipe_key "$(manifest_class_recipe_key "$class")" + done < <(source_artifact_classes) + else + release_manifest_http_code=$http_code + # Forks and mirrors may bootstrap only when the rolling release has not + # been created yet. Network failures and other HTTP errors stay fatal so + # external release or network problems remain visible. + if [ "$bootstrap_on_404" = false ] || [ "$release_manifest_http_code" != 404 ]; then + cat "$curl_err" >&2 + fi + echo "prebuilt manifest is unavailable at $manifest_url" >&2 + return 1 + fi +} + +class_any_artifact_exists() { + local artifact + + while IFS= read -r artifact; do + if [ -f "$artifact" ]; then + return 0 + fi + done < <(source_artifact_class_outputs "$1") + + return 1 +} + +# Read recipe-key stamps from a restored GitHub Actions raw-artifact cache. +# A class may contain multiple raw artifact files, so every file must exist, +# have a stamp, and agree on the same recipe key before the cache is trusted. +class_local_recipe_key() { + local class=$1 + local artifact + local recipe_stamp + local recipe_key= + local next= + local have_key=false + + while IFS= read -r artifact; do + recipe_stamp=$(prebuilt_artifact_recipe_stamp "$artifact") + if [ ! -f "$artifact" ] || [ ! -f "$recipe_stamp" ]; then + return 1 + fi + IFS= read -r next < "$recipe_stamp" || return 1 + if [ -z "$next" ]; then + return 1 + fi + if [ "$have_key" = false ]; then + recipe_key=$next + have_key=true + elif [ "$recipe_key" != "$next" ]; then + return 1 + fi + done < <(source_artifact_class_outputs "$class") + + if [ "$have_key" = false ]; then + return 1 + fi + + printf '%s\n' "$recipe_key" +} + +# Local state here means a CI cache restore, not a developer workspace. +# CI trusts a matching recipe-key stamp as the cache contract; raw artifact +# content corruption is intentionally left as a normal CI failure to inspect. +# Status values are: missing = no raw artifact, valid = artifact plus matching +# stamps, unmanaged = raw artifact exists but cannot be tied to a recipe key. +resolve_local_class() { + local class=$1 + local recipe_key= + local status=missing + + if class_any_artifact_exists "$class"; then + if recipe_key=$(class_local_recipe_key "$class" 2>/dev/null); then + status=valid + else + status=unmanaged + fi + fi + + prebuilt_plan_set_class_var "$class" local_recipe_key "$recipe_key" + prebuilt_plan_set_class_var "$class" local_status "$status" +} + +# The action is the only authority consumed by materialize-artifacts.sh: +# use-action-cache, download-release, or build. Prefer already-restored raw +# artifacts first, then the rolling release, and build from source only when +# neither source matches the current checkout recipe. +decide_action() { + local current_recipe_key=$1 + local release_recipe_key=$2 + local local_recipe_key=$3 + local local_status=$4 + + if [ "$local_status" = valid ] && [ "$local_recipe_key" = "$current_recipe_key" ]; then + printf '%s\n' use-action-cache + return + fi + + if [ -n "$release_recipe_key" ] && [ "$release_recipe_key" = "$current_recipe_key" ]; then + printf '%s\n' download-release + else + printf '%s\n' build + fi +} + +# Collapse the current per-class recipe keys into a single cache namespace +# for the workflow-local raw artifact bundle. This is not an artifact checksum; +# materialization correctness still comes from each class action and stamp. +compute_platform_action_cache_tag() { + local class + + while IFS= read -r class; do + prebuilt_plan_get_class_var "$class" current_recipe_key + done < <(source_artifact_classes) | prebuilt_sha1sum | awk '{print $1}' +} + +# Compute checkout-derived recipe keys before consulting the release. These +# keys describe the artifacts this workflow wants, independent of what exists +# in cache or in the rolling release. +while IFS= read -r class; do + prebuilt_plan_set_class_var "$class" current_recipe_key "$(prebuilt_class_recipe_key "$class")" + resolve_local_class "$class" +done < <(source_artifact_classes) +platform_action_cache_tag=$(compute_platform_action_cache_tag) +ci_cache_schema_tag=$(prebuilt_ci_cache_schema_tag) + +# A missing manifest is only tolerated for fork/mirror bootstrap. In that +# case release_*_recipe_key fields stay empty, so every class that is not +# satisfied by restored cache will choose build. +if ! fetch_release_manifest; then + if [ "$bootstrap_on_404" = true ] && [ "$release_manifest_http_code" = 404 ]; then + echo "prebuilt manifest is missing; bootstrapping from source" >&2 + else + exit 1 + fi +fi + +# Convert the three observed states for each class into the materializer +# action that will be executed later in the workflow. Workflow plumbing uses +# requires_build to decide whether build dependencies are needed before +# materialization. +requires_build=false +while IFS= read -r class; do + action=$(decide_action \ + "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" \ + "$(prebuilt_plan_get_class_var "$class" release_recipe_key)" \ + "$(prebuilt_plan_get_class_var "$class" local_recipe_key)" \ + "$(prebuilt_plan_get_class_var "$class" local_status)") + prebuilt_plan_set_class_var "$class" action "$action" + if [ "$action" = build ]; then + requires_build=true + fi +done < <(source_artifact_classes) + +# Publishing cares about the rolling release as a whole. If any class in the +# release manifest does not match the current recipe, the release needs update. +release_needs_update=false +while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" release_recipe_key)" != "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" ]; then + release_needs_update=true + break + fi +done < <(source_artifact_classes) + +# Print a shell-friendly plan. materialize-artifacts.sh consumes the action and +# current recipe-key fields; the remaining scalar fields feed workflow +# conditions, cache keys, and publish gates. +printf 'plan_version=1\n' +prebuilt_plan_print_class_values action +prebuilt_plan_print_class_values current_recipe_key +printf 'platform_action_cache_tag=%s\n' "$platform_action_cache_tag" +printf 'ci_cache_schema_tag=%s\n' "$ci_cache_schema_tag" +printf 'requires_build=%s\n' "$requires_build" +printf 'release_needs_update=%s\n' "$release_needs_update" diff --git a/.ci/prebuilt/stamp-artifacts.sh b/.ci/prebuilt/stamp-artifacts.sh new file mode 100755 index 00000000..db1a55e6 --- /dev/null +++ b/.ci/prebuilt/stamp-artifacts.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# CI stamp helper. A stamp records which recipe key produced an existing raw +# artifact restored from CI cache, downloaded from release, or built in CI. +# It is not an archive checksum and is not a publication manifest. + +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + +# Stamping needs only the class registry and recipe-key helpers. The CI +# materializer sources this file and calls the stamp functions directly. +# shellcheck source=.ci/prebuilt/artifact-inputs.sh +. "$SCRIPT_DIR/artifact-inputs.sh" + +prebuilt_stamp_artifacts_for_class() { + local class=$1 + local recipe_key=$2 + local artifact + + if [ -z "$recipe_key" ]; then + echo "[!] Cannot stamp $class with an empty recipe key" >&2 + return 1 + fi + + mkdir -p "$prebuilt_local_stamp_dir" || return 1 + while IFS= read -r artifact; do + if [ ! -f "$artifact" ]; then + echo "[!] Cannot stamp missing artifact: $artifact" >&2 + return 1 + fi + printf '%s\n' "$recipe_key" > "$(prebuilt_artifact_recipe_stamp "$artifact")" + done < <(source_artifact_class_outputs "$class") +} diff --git a/.ci/prebuilt/verify-package.sh b/.ci/prebuilt/verify-package.sh new file mode 100755 index 00000000..fbe0c21e --- /dev/null +++ b/.ci/prebuilt/verify-package.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) + +# Verification follows the class registry and checks package shape: each raw +# artifact has a compressed archive, prebuilt.sha1 has the expected archive and +# recipe-key entries, and archive checksum entries match the packaged bytes. +. "$SCRIPT_DIR/artifact-inputs.sh" + +if [ ! -f prebuilt.sha1 ]; then + echo "[!] Missing prebuilt.sha1" >&2 + exit 1 +fi + +manifest_entry_value() { + local entry=$1 + + awk -v f="$entry" '$2 == f {print $1; found = 1; exit} END {exit !found}' prebuilt.sha1 +} + +verify_archive_checksum_entry() { + local archive=$1 + local expected + local actual + + if ! expected=$(manifest_entry_value "$archive"); then + echo "[!] prebuilt.sha1 missing $archive" >&2 + return 1 + fi + + actual=$(prebuilt_sha1sum "$archive" | awk '{print $1}') + if [ "$actual" != "$expected" ]; then + echo "[!] prebuilt.sha1 checksum mismatch for $archive" >&2 + echo "[!] expected $expected" >&2 + echo "[!] actual $actual" >&2 + return 1 + fi +} + +while IFS= read -r class; do + while IFS= read -r artifact; do + archive=${artifact}.bz2 + if [ ! -f "$archive" ]; then + echo "[!] Missing $archive" >&2 + exit 1 + fi + verify_archive_checksum_entry "$archive" + done < <(source_artifact_class_outputs "$class") + + while IFS= read -r entry; do + if ! manifest_entry_value "$entry" >/dev/null; then + echo "[!] prebuilt.sha1 missing $entry" >&2 + exit 1 + fi + done < <(prebuilt_class_recipe_entries "$class") +done < <(source_artifact_classes) diff --git a/.ci/publish-prebuilt.sh b/.ci/publish-prebuilt.sh deleted file mode 100755 index d4eeaeaf..00000000 --- a/.ci/publish-prebuilt.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# -# Compress the prebuilt Image, rootfs.cpio, and test-tools.img in cwd, write a -# sha1 manifest, hash the input files that define the prebuilt's contents, and -# print all four sums in KEY=VAL form on stdout so callers can splice them into -# release notes, GITHUB_OUTPUT, or whatever else. -# -# Inputs (in cwd): -# Image -# rootfs.cpio -# test-tools.img -# plus the source inputs listed in INPUTS below (config + scripts + -# target files that define the prebuilt content) -# -# Outputs (in cwd): -# Image.bz2 -# rootfs.cpio.bz2 -# test-tools.img.bz2 -# prebuilt.sha1 -- four-line manifest in sha1sum format. The -# first three lines verify the published archives; -# the fourth uses the virtual name 'inputs' to -# publish the SHA-1 of the concatenated input -# files so drift-detection consumers can read it -# directly from the release. -# -# Stdout (machine-readable, one assignment per line): -# kernel_sha1= -# initrd_sha1= -# test_tools_sha1= -# inputs_sha1= - -set -euo pipefail - -# Pick a SHA1 tool. macOS dropped sha1sum from the base system; the -# coreutils-style 'shasum -a 1' is the portable fallback. -if command -v sha1sum >/dev/null 2>&1; then - SHA1=(sha1sum) -elif command -v shasum >/dev/null 2>&1; then - SHA1=(shasum -a 1) -else - echo "[!] Need sha1sum (Linux) or shasum (macOS) on PATH" >&2 - exit 1 -fi - -# Keep this list in sync with PREBUILT_INPUTS in mk/external.mk and the -# 'paths:' filter in .github/workflows/prebuilt.yml. -INPUTS=( - configs/linux.config - configs/busybox.config - configs/buildroot.config - configs/x11.config - configs/riscv-cross-file - scripts/build-image.sh - scripts/rootfs_ext4.sh - target/init - target/local-env.sh -) - -for f in Image rootfs.cpio test-tools.img "${INPUTS[@]}"; do - if [ ! -f "$f" ]; then - echo "[!] Missing $f -- run scripts/build-image.sh --all --x11 --directfb2-test first" >&2 - exit 1 - fi -done - -bzip2 -k -f Image -bzip2 -k -f rootfs.cpio -bzip2 -k -f test-tools.img - -KERNEL_SHA1=$("${SHA1[@]}" Image.bz2 | awk '{print $1}') -INITRD_SHA1=$("${SHA1[@]}" rootfs.cpio.bz2 | awk '{print $1}') -TEST_TOOLS_SHA1=$("${SHA1[@]}" test-tools.img.bz2 | awk '{print $1}') -# Concatenate inputs in deterministic order and hash the stream. Matches -# the make-time computation in mk/external.mk so they compare directly. -INPUTS_SHA1=$(cat "${INPUTS[@]}" | "${SHA1[@]}" | awk '{print $1}') - -# Write the manifest. The first three lines match 'sha1sum -c' format for -# the real archives; the fourth line uses the virtual filename 'inputs' -# to publish the input-fingerprint hash so consumers (mk/external.mk's -# drift warning, .github/workflows/main.yml's PR drift detection) can -# read it from the release without parsing the release-body markdown. -{ - echo "$KERNEL_SHA1 Image.bz2" - echo "$INITRD_SHA1 rootfs.cpio.bz2" - echo "$TEST_TOOLS_SHA1 test-tools.img.bz2" - echo "$INPUTS_SHA1 inputs" -} > prebuilt.sha1 - -# Echo the manifest + inputs hash to stderr for visibility in CI logs -# without polluting the parseable stdout block below. -{ - cat prebuilt.sha1 - echo "inputs_sha1: $INPUTS_SHA1" -} >&2 - -echo "kernel_sha1=$KERNEL_SHA1" -echo "initrd_sha1=$INITRD_SHA1" -echo "test_tools_sha1=$TEST_TOOLS_SHA1" -echo "inputs_sha1=$INPUTS_SHA1" diff --git a/.ci/test-build-system.sh b/.ci/test-build-system.sh new file mode 100755 index 00000000..d931f15d --- /dev/null +++ b/.ci/test-build-system.sh @@ -0,0 +1,689 @@ +#!/usr/bin/env bash + +set -euo pipefail + +# Fast build-system contract tests. +# +# Keep this script cheap and deterministic. It should not download guest +# artifacts, unpack release assets, or trigger a real Buildroot/Linux build. +# Slower CI jobs cover those paths. The purpose here is to pin the shell +# contracts that glue those jobs together: artifact class registration, recipe +# inputs, local CLI validation, Make download recipes, materializer plan +# parsing, and the Buildroot source-build lock. +# +# Generated files live under TEST_TMP so the tests can freely create fake plans, +# lock paths, and temporary Makefiles without touching real artifacts in the +# checkout. +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +cd "$REPO_ROOT" + +# All fixtures are disposable. Logs are kept under the same directory so failure +# paths can dump useful context without leaving state in the repository. +TEST_TMP=$(mktemp -d "${TMPDIR:-/tmp}/semu-build-system-test.XXXXXX") +cleanup() { + rm -rf "$TEST_TMP" +} +trap cleanup EXIT + +# Assertion helpers print enough context to identify the broken contract from CI +# logs. Keep them small and POSIX-friendly because this script is run by macOS +# and Linux jobs. +fail() { + printf '[!] %s\n' "$*" >&2 + exit 1 +} + +assert_eq() { + local actual=$1 + local expected=$2 + local label=$3 + + if [ "$actual" != "$expected" ]; then + printf '[!] %s\n' "$label" >&2 + printf '[!] expected:\n%s\n' "$expected" >&2 + printf '[!] actual:\n%s\n' "$actual" >&2 + exit 1 + fi +} + +assert_contains() { + local haystack=$1 + local needle=$2 + local label=$3 + + case "$haystack" in + *"$needle"*) ;; + *) fail "$label: missing '$needle'" ;; + esac +} + +assert_not_contains() { + local haystack=$1 + local needle=$2 + local label=$3 + + case "$haystack" in + *"$needle"*) fail "$label: unexpected '$needle'" ;; + esac +} + +# Each check is a named function so CI output points at the failed build-system +# contract instead of only reporting a line number inside this driver. +run_test() { + local name=$1 + shift + + printf ' TEST\t%s\n' "$name" + "$@" +} + +# Diagnostic text is part of the contract. Capture stdout/stderr into a log so +# callers can assert that failures are actionable, not just non-zero. +expect_failure() { + local label=$1 + local log=$2 + shift 2 + + if "$@" >"$log" 2>&1; then + cat "$log" >&2 + fail "$label: command unexpectedly succeeded" + fi +} + +# Success checks use the same log convention; on failure the captured output is +# printed before the test exits. +expect_success() { + local label=$1 + local log=$2 + shift 2 + + if ! "$@" >"$log" 2>&1; then + cat "$log" >&2 + fail "$label: command failed" + fi +} + +# Source pure helper libraries in the parent for unit-style checks. The lock +# helper is sourced later because it installs traps only after acquiring a lock. +. scripts/prebuilt/artifact-classes.sh +. scripts/prebuilt/test-tools-recipe.sh +. .ci/prebuilt/artifact-inputs.sh + +# The artifact class registry is shared by source builds, release downloads, +# action-cache planning, and Make targets. A silent class/output rename would +# desynchronize those paths, so keep the public class contract pinned here. +test_artifact_class_contract() { + local log="$TEST_TMP/unknown-artifact-class.log" + + assert_eq "$(source_artifact_classes)" $'image\nrootfs\ntest-tools' \ + "artifact class list changed unexpectedly" + assert_eq "$(source_artifact_outputs)" $'Image\nrootfs.cpio\ntest-tools.img' \ + "artifact output list changed unexpectedly" + assert_eq "$(source_artifact_class_outputs test-tools)" "test-tools.img" \ + "test-tools class output changed unexpectedly" + + expect_failure "unknown artifact class" "$log" \ + scripts/prebuilt/artifact-classes.sh output does-not-exist + assert_contains "$(cat "$log")" "Unknown source artifact class" \ + "unknown artifact class diagnostic" +} + +# The test-tools disk is optional and recipe-driven. These checks pin the +# default recipe, canonical ordering, duplicate handling, and invalid-input +# diagnostics without building the image. +test_test_tools_recipe_selection() { + local log="$TEST_TMP/test-tools-recipe.log" + + assert_eq "$(env -u PREBUILT_TEST_TOOLS_RECIPE bash -c \ + '. scripts/prebuilt/test-tools-recipe.sh; prebuilt_test_tools_recipe_key')" \ + "x11,directfb2" \ + "default test-tools recipe key" + + assert_eq "$(PREBUILT_TEST_TOOLS_RECIPE='directfb2,x11' \ + prebuilt_test_tools_recipe_key)" \ + "x11,directfb2" \ + "test-tools recipe key should be canonicalized" + + assert_eq "$(PREBUILT_TEST_TOOLS_RECIPE='x11,x11' \ + prebuilt_test_tools_recipe_key)" \ + "x11" \ + "duplicate test-tools recipe entries should collapse" + + expect_failure "empty test-tools recipe" "$log" \ + env PREBUILT_TEST_TOOLS_RECIPE= bash -c \ + '. scripts/prebuilt/test-tools-recipe.sh; prebuilt_test_tools_recipe_key' + assert_contains "$(cat "$log")" "Empty test-tools recipe" \ + "empty test-tools recipe diagnostic" + + expect_failure "unknown test-tools recipe" "$log" \ + env PREBUILT_TEST_TOOLS_RECIPE=unknown bash -c \ + '. scripts/prebuilt/test-tools-recipe.sh; prebuilt_test_tools_recipe_key' + assert_contains "$(cat "$log")" "Unknown test-tools recipe entry" \ + "unknown test-tools recipe diagnostic" +} + +# CI plan files encode class names as shell variable prefixes. Test the helper +# mapping directly so class names such as test-tools keep using safe variable +# names and do not drift from the planner/materializer contract. +test_prebuilt_ci_helper_contract() { + local inputs + local total + local unique + + assert_eq "$(prebuilt_plan_class_var_name test-tools action)" \ + "test_tools_action" \ + "test-tools plan variable name" + prebuilt_plan_set_class_var test-tools action build + assert_eq "$(prebuilt_plan_get_class_var test-tools action)" "build" \ + "test-tools plan variable round trip" + + if ! prebuilt_plan_class_var_matches test_tools_action test-tools action current_recipe_key; then + fail "test-tools action key should match generated plan vars" + fi + if prebuilt_plan_class_var_matches test_tools_action rootfs action current_recipe_key; then + fail "test-tools action key should not match rootfs plan vars" + fi + + assert_eq "$(prebuilt_class_recipe_entries test-tools)" \ + "test-tools.img.recipe-key" \ + "test-tools recipe manifest entry" + assert_eq "$(prebuilt_artifact_recipe_stamp Image)" \ + ".stamps/prebuilt-local/Image.recipe-key" \ + "Image recipe stamp path" + + inputs=$(PREBUILT_TEST_TOOLS_RECIPE=x11 prebuilt_class_inputs test-tools) + assert_contains "$inputs" "configs/x11.config" \ + "x11 test-tools inputs" + assert_not_contains "$inputs" "configs/meson-riscv-cross-file" \ + "x11 test-tools inputs" + + inputs=$(PREBUILT_TEST_TOOLS_RECIPE=directfb2 prebuilt_class_inputs test-tools) + assert_not_contains "$inputs" "configs/x11.config" \ + "directfb2 test-tools inputs" + assert_contains "$inputs" "configs/meson-riscv-cross-file" \ + "directfb2 test-tools inputs" + + inputs=$(prebuilt_inputs) + total=$(printf '%s\n' "$inputs" | sed '/^$/d' | wc -l | tr -d ' ') + unique=$(printf '%s\n' "$inputs" | sed '/^$/d' | sort -u | wc -l | tr -d ' ') + assert_eq "$total" "$unique" "prebuilt_inputs should be de-duplicated" +} + +# The recipe manifest is more useful than checking only the final SHA: when a +# recipe key changes, the manifest should show exactly which selected input +# participated in the class recipe. +test_recipe_manifest_tracks_selected_inputs() { + local manifest + + manifest=$(PREBUILT_TEST_TOOLS_RECIPE=x11 \ + prebuilt_class_recipe_manifest test-tools) + assert_contains "$manifest" "configs/x11.config" \ + "x11 recipe manifest" + assert_not_contains "$manifest" "configs/meson-riscv-cross-file" \ + "x11 recipe manifest" + assert_contains "$manifest" "test-tools.recipe" \ + "x11 recipe manifest" + assert_contains "$manifest" "artifact-recipe.env:TEST_TOOLS_SIZE_MB" \ + "x11 recipe manifest" + + manifest=$(PREBUILT_TEST_TOOLS_RECIPE=directfb2 \ + prebuilt_class_recipe_manifest test-tools) + assert_not_contains "$manifest" "configs/x11.config" \ + "directfb2 recipe manifest" + assert_contains "$manifest" "configs/meson-riscv-cross-file" \ + "directfb2 recipe manifest" + assert_contains "$manifest" "test-tools.recipe" \ + "directfb2 recipe manifest" +} + +# Local CLI validation should fail before expensive source checkout/build work. +# These cases cover option combinations that are easy to break while extending +# scripts/build-artifacts.sh. +test_build_artifacts_cli_validation() { + local log="$TEST_TMP/build-artifacts.log" + + expect_success "build-artifacts help" "$log" \ + scripts/build-artifacts.sh --help + assert_contains "$(cat "$log")" "Usage:" "build-artifacts help" + + expect_failure "recipe option without target" "$log" \ + scripts/build-artifacts.sh image --x11 + assert_contains "$(cat "$log")" \ + "test-tools recipe options require the test-tools or all target" \ + "recipe option without target diagnostic" + + expect_failure "--no-ext4 without rootfs target" "$log" \ + scripts/build-artifacts.sh image --no-ext4 + assert_contains "$(cat "$log")" \ + "--no-ext4 requires the rootfs or all target" \ + "--no-ext4 diagnostic" + + expect_failure "unknown build-artifacts target" "$log" \ + scripts/build-artifacts.sh does-not-exist + assert_contains "$(cat "$log")" "Unknown option or target" \ + "unknown build-artifacts target diagnostic" +} + +# A dry-run Makefile is enough to inspect the generated recipe. This pins the +# previous review concern that PREBUILT_URL must be expanded by Make and passed +# to the shell as an environment value, without performing any network access. +test_make_download_rule_uses_make_expansion() { + local makefile="$TEST_TMP/external.mk" + local log="$TEST_TMP/external-make.log" + + printf 'include mk/external.mk\n' > "$makefile" + expect_success "mk/external Image rule dry run" "$log" \ + make -n -B -f "$makefile" Image \ + PREBUILT_URL=https://example.invalid/prebuilt + + assert_contains "$(cat "$log")" \ + 'PREBUILT_URL="https://example.invalid/prebuilt" scripts/prebuilt/download-release-artifact.sh "Image"' \ + "mk/external download command" + assert_not_contains "$(cat "$log")" '$(PREBUILT_URL)' \ + "mk/external download command" +} + +# The Buildroot source lock is a symlink in the current design. Keep the ignore +# rule matching symlinks as well as old-style directory locks so interrupted +# builds do not leave noisy untracked lock paths in developer worktrees. +test_buildroot_lock_path_is_ignored() { + local tmp="$TEST_TMP/gitignore-lock" + local status + + mkdir "$tmp" + set +e + ( + cd "$tmp" + git init -q + cp "$REPO_ROOT/.gitignore" .gitignore + ln -s "semu-buildroot:test:1:0:token" .semu-buildroot.lock + git check-ignore -q .semu-buildroot.lock + ) + status=$? + set -e + + [ "$status" -eq 0 ] || fail ".semu-buildroot.lock symlink should be ignored" +} + +# rootfs_ext4.sh intentionally normalizes the whole staged filesystem to root +# ownership before mkfs, matching the historical ext4.img policy. This static +# check avoids running fakeroot/mkfs while still catching the accidental case +# where chown is only applied when an extra overlay directory is provided. +test_rootfs_ext4_normalizes_ownership_unconditionally() { + local in_extra=false + local after_extra=false + local chown_in_extra=false + local chown_after_extra=false + local line + + while IFS= read -r line; do + case "$line" in + "if [ -n \"\$SEMU_EXTRA_DIR\" ]; then") + in_extra=true + ;; + "fi") + if [ "$in_extra" = true ]; then + in_extra=false + after_extra=true + fi + ;; + '"$SEMU_ROOTFS_CHOWN" -R 0:0 .'|' "$SEMU_ROOTFS_CHOWN" -R 0:0 .') + if [ "$in_extra" = true ]; then + chown_in_extra=true + fi + if [ "$after_extra" = true ]; then + chown_after_extra=true + fi + ;; + esac + done < scripts/rootfs_ext4.sh + + [ "$chown_after_extra" = true ] || \ + fail "rootfs_ext4.sh should chown the staged rootfs outside the extra-dir branch" + [ "$chown_in_extra" = false ] || \ + fail "rootfs_ext4.sh should not limit rootfs ownership normalization to extra files" +} + + +# rootfs_ext4.sh runs several external tools under fakeroot. Keep each tool +# selectable by the caller so CI can provide one coherent toolchain on hosts +# where mixing system and package-manager binaries breaks fakeroot injection. +test_rootfs_ext4_tools_are_selectable() { + local script + + script=$(cat scripts/rootfs_ext4.sh) + + assert_contains "$script" 'ROOTFS_CPIO="${ROOTFS_CPIO:-cpio}"' \ + "rootfs_ext4 cpio hook" + assert_contains "$script" 'ROOTFS_CP="${ROOTFS_CP:-cp}"' \ + "rootfs_ext4 cp hook" + assert_contains "$script" 'ROOTFS_CHOWN="${ROOTFS_CHOWN:-chown}"' \ + "rootfs_ext4 chown hook" + assert_contains "$script" 'ROOTFS_FAKEROOT_SHELL="${ROOTFS_FAKEROOT_SHELL:-/bin/sh}"' \ + "rootfs_ext4 fakeroot shell hook" + assert_contains "$script" '"$SEMU_ROOTFS_CP" -a "$SEMU_EXTRA_DIR"/. .' \ + "rootfs_ext4 cp command should use caller-selected tool" + assert_contains "$script" '"$SEMU_ROOTFS_CHOWN" -R 0:0 .' \ + "rootfs_ext4 chown command should use caller-selected tool" + assert_contains "$script" 'fakeroot "$ROOTFS_FAKEROOT_SHELL" "$FAKEROOT_SCRIPT"' \ + "rootfs_ext4 fakeroot shell command should use caller-selected tool" +} + +# The Makefile probe must exercise the same fakeroot tool hooks used by the +# ext4 image path. A trivial shell-only probe can pass even when the later +# fakeroot child fails to run cp/chown on a host such as macOS 26. +test_makefile_uses_rootfs_fakeroot_probe_helper() { + local makefile + + makefile=$(cat Makefile) + + assert_contains "$makefile" 'scripts/check-rootfs-fakeroot.sh' \ + "Makefile should delegate fakeroot probing to the rootfs helper" + assert_not_contains "$makefile" 'fakeroot /bin/sh -c :' \ + "Makefile should not use a shell-only fakeroot probe" + + local ext4_rule + ext4_rule=$(awk '/^ext4.img:/{capture=1} capture{print} capture && /^$/{exit}' Makefile) + assert_contains "$ext4_rule" 'ROOTFS_FAKEROOT_SHELL="$(ROOTFS_FAKEROOT_SHELL)"' \ + "Makefile ext4 rule should pass the fakeroot shell hook" + assert_contains "$ext4_rule" 'ROOTFS_CP="$(ROOTFS_CP)"' \ + "Makefile ext4 rule should pass the cp hook" + assert_contains "$ext4_rule" 'ROOTFS_CHOWN="$(ROOTFS_CHOWN)"' \ + "Makefile ext4 rule should pass the chown hook" +} + + +# macOS CI should use the rootfs_ext4.sh hooks instead of hardcoding platform +# branches in the helper itself. The selected binaries must come from one +# Homebrew toolchain so fakeroot injects into matching arm64 processes on both +# macOS 15 and macOS 26 runners. +test_macos_setup_exports_rootfs_tool_hooks() { + local action + + action=$(cat .github/actions/setup-semu/action.yml) + + assert_contains "$action" 'brew install make dtc expect fakeroot bash cpio coreutils e2fsprogs sdl2' \ + "macOS setup should install the rootfs toolchain" + assert_contains "$action" 'echo "ROOTFS_FAKEROOT_SHELL=$rootfs_fakeroot_shell" >> "$GITHUB_ENV"' \ + "macOS setup should export the fakeroot shell hook" + assert_contains "$action" 'echo "ROOTFS_CPIO=$rootfs_cpio" >> "$GITHUB_ENV"' \ + "macOS setup should export the cpio hook" + assert_contains "$action" 'echo "ROOTFS_CP=$rootfs_cp" >> "$GITHUB_ENV"' \ + "macOS setup should export the cp hook" + assert_contains "$action" 'echo "ROOTFS_CHOWN=$rootfs_chown" >> "$GITHUB_ENV"' \ + "macOS setup should export the chown hook" +} + +# The materializer consumes planner output in several CI jobs. Use tiny fake +# plans to check parser behavior, unknown-key rejection, and the guard that +# prevents source builds in jobs that should only materialize cached artifacts. +test_materializer_plan_validation() { + local plan="$TEST_TMP/prebuilt-plan.env" + local log="$TEST_TMP/materialize.log" + + { + printf 'plan_version=1\n' + printf 'image_action=use-action-cache\n' + printf 'rootfs_action=use-action-cache\n' + printf 'test_tools_action=use-action-cache\n' + } > "$plan" + expect_success "materializer accepts no-op cache plan" "$log" \ + env PREBUILT_PLAN_FILE="$plan" .ci/prebuilt/materialize-artifacts.sh + + { + printf 'plan_version=1\n' + printf 'image_action=use-action-cache\n' + printf 'rootfs_action=use-action-cache\n' + printf 'test_tools_action=use-action-cache\n' + printf 'unknown_key=value\n' + } > "$plan" + expect_failure "materializer rejects unknown plan key" "$log" \ + env PREBUILT_PLAN_FILE="$plan" .ci/prebuilt/materialize-artifacts.sh + assert_contains "$(cat "$log")" "Unknown prebuilt plan key" \ + "unknown materializer plan key diagnostic" + + { + printf 'plan_version=1\n' + printf 'image_action=use-action-cache\n' + printf 'rootfs_action=build\n' + printf 'current_rootfs_recipe_key=dummy-rootfs-key\n' + printf 'test_tools_action=use-action-cache\n' + } > "$plan" + expect_failure "materializer honors PREBUILT_FORBID_BUILD" "$log" \ + env PREBUILT_PLAN_FILE="$plan" PREBUILT_FORBID_BUILD=1 \ + .ci/prebuilt/materialize-artifacts.sh + assert_contains "$(cat "$log")" "PREBUILT_FORBID_BUILD forbids source builds" \ + "forbid-build diagnostic" + assert_contains "$(cat "$log")" "rootfs_action=build" \ + "forbid-build class diagnostic" +} + +# The Buildroot source tree can be shared by Image/rootfs/test-tools builds. The +# lock must publish an owner token atomically and remove only that token on a +# normal release. +test_lock_acquire_release_contract() ( + . scripts/prebuilt/artifact-recipes.sh + + local tmp="$TEST_TMP/lock-acquire-release" + mkdir "$tmp" + BUILDROOT_LOCK_PATH="$tmp/lock" + + buildroot_acquire_lock + [ -L "$BUILDROOT_LOCK_PATH" ] || fail "lock path should be a symlink" + assert_eq "$(readlink "$BUILDROOT_LOCK_PATH")" "$BUILDROOT_LOCK_TOKEN" \ + "lock owner token" + buildroot_release_lock + [ ! -e "$BUILDROOT_LOCK_PATH" ] || fail "lock should be removed after release" +) + +# Artifact helpers may nest source-build operations in one process. Reentrant +# acquisition should only bump depth and must not remove the symlink until the +# outermost release. +test_lock_reentrant_contract() ( + . scripts/prebuilt/artifact-recipes.sh + + local tmp="$TEST_TMP/lock-reentrant" + mkdir "$tmp" + BUILDROOT_LOCK_PATH="$tmp/lock" + + buildroot_acquire_lock + local token=$BUILDROOT_LOCK_TOKEN + buildroot_acquire_lock + assert_eq "$BUILDROOT_LOCK_DEPTH" "2" "reentrant lock depth" + buildroot_release_lock + [ -L "$BUILDROOT_LOCK_PATH" ] || fail "reentrant first release should keep lock" + assert_eq "$(readlink "$BUILDROOT_LOCK_PATH")" "$token" \ + "reentrant lock token" + buildroot_release_lock + [ ! -e "$BUILDROOT_LOCK_PATH" ] || fail "reentrant final release should remove lock" +) + +# Release uses compare-and-delete semantics. If another owner replaces the lock +# path before release, the old owner must leave the new token alone. +test_lock_release_only_removes_own_token() ( + . scripts/prebuilt/artifact-recipes.sh + + local tmp="$TEST_TMP/lock-release-token" + mkdir "$tmp" + BUILDROOT_LOCK_PATH="$tmp/lock" + + buildroot_acquire_lock + rm -f "$BUILDROOT_LOCK_PATH" + ln -s "semu-buildroot:foreign:12345:0:fresh" "$BUILDROOT_LOCK_PATH" + buildroot_release_lock + assert_eq "$(readlink "$BUILDROOT_LOCK_PATH")" \ + "semu-buildroot:foreign:12345:0:fresh" \ + "release must not delete a replaced lock" +) + +# Stale locks are reported but not automatically removed. That is intentional: +# deleting after a read/compare check can race with a fresh owner that acquired +# the path after the stale check. +test_lock_reports_stale_without_deleting() { + local tmp="$TEST_TMP/lock-stale" + local log="$tmp/stale.log" + local host + local token + local status + + mkdir "$tmp" + host=$(buildroot_lock_host) + token="semu-buildroot:$host:99999999:0:dead" + ln -s "$token" "$tmp/lock" + + set +e + ( + . scripts/prebuilt/artifact-recipes.sh + BUILDROOT_LOCK_PATH="$tmp/lock" + buildroot_acquire_lock + ) >"$log" 2>&1 + status=$? + set -e + + [ "$status" -ne 0 ] || fail "stale lock acquire should fail" + [ -L "$tmp/lock" ] || fail "stale lock should not be deleted" + assert_eq "$(readlink "$tmp/lock")" "$token" \ + "stale lock token should be preserved" + assert_contains "$(cat "$log")" "source build lock appears stale" \ + "stale lock diagnostic" +} + +# Old directory locks, hand-created files, and other foreign paths should stop +# the build before the protected payload runs. The diagnostic tells the user +# which path needs manual cleanup. +test_lock_reports_unowned_without_running_payload() { + local tmp="$TEST_TMP/lock-unowned" + local log="$tmp/unowned.log" + local status + + mkdir -p "$tmp/lock" + + set +e + ( + . scripts/prebuilt/artifact-recipes.sh + BUILDROOT_LOCK_PATH="$tmp/lock" + with_buildroot_lock touch "$tmp/payload" + ) >"$log" 2>&1 + status=$? + set -e + + [ "$status" -ne 0 ] || fail "unowned lock acquire should fail" + [ -d "$tmp/lock" ] || fail "unowned lock directory should not be deleted" + [ ! -e "$tmp/payload" ] || fail "payload ran despite lock acquisition failure" + assert_contains "$(cat "$log")" "not an owned semu lock" \ + "unowned lock diagnostic" + + rm -rf "$tmp/lock" + ( + . scripts/prebuilt/artifact-recipes.sh + BUILDROOT_LOCK_PATH="$tmp/lock" + with_buildroot_lock touch "$tmp/payload" + ) >"$log" 2>&1 + [ -e "$tmp/payload" ] || fail "payload command should run after acquiring a valid lock" +} + +# Contention should be safe and observable: one process waits on the existing +# owner token, then acquires only after the holder releases it. +test_lock_waiter_acquires_after_release() { + local tmp="$TEST_TMP/lock-contention" + local holder_log="$tmp/holder.log" + local waiter_log="$tmp/waiter.log" + local holder + local waiter + local saw_wait=false + local i + + mkdir "$tmp" + ( + . scripts/prebuilt/artifact-recipes.sh + BUILDROOT_LOCK_PATH="$tmp/lock" + buildroot_acquire_lock + touch "$tmp/holder-ready" + # Keep the holder locked until the parent has observed the waiter + # report contention. A fixed sleep makes the test depend on scheduler + # timing: under a slow runner the waiter could start only after the + # holder released, acquire immediately, and never print the wait + # message this test is specifically checking. + while [ ! -e "$tmp/release-holder" ]; do + sleep 0.05 + done + buildroot_release_lock + ) >"$holder_log" 2>&1 & + holder=$! + + while [ ! -e "$tmp/holder-ready" ]; do + sleep 0.05 + done + + ( + . scripts/prebuilt/artifact-recipes.sh + BUILDROOT_LOCK_PATH="$tmp/lock" + buildroot_acquire_lock + printf "waiter-acquired\n" + buildroot_release_lock + ) >"$waiter_log" 2>&1 & + waiter=$! + + # Wait until the waiter has actually observed the held lock and emitted the + # diagnostic. Only then let the holder release. This makes the later + # assertion deterministic instead of relying on the holder sleep duration. + for i in {1..100}; do + if grep -q "Waiting for Buildroot source build lock:" "$waiter_log" 2>/dev/null; then + saw_wait=true + break + fi + if ! kill -0 "$waiter" 2>/dev/null; then + break + fi + sleep 0.05 + done + + if [ "$saw_wait" != true ]; then + touch "$tmp/release-holder" + wait "$holder" || true + wait "$waiter" || true + cat "$holder_log" >&2 + cat "$waiter_log" >&2 + fail "waiter did not observe the held lock before acquiring" + fi + + touch "$tmp/release-holder" + wait "$waiter" + wait "$holder" + + assert_contains "$(cat "$waiter_log")" \ + "Waiting for Buildroot source build lock:" \ + "lock waiter diagnostic" + assert_contains "$(cat "$waiter_log")" "waiter-acquired" \ + "lock waiter completion" + [ ! -e "$tmp/lock" ] || fail "lock should be removed after contention test" +} + +# Source the lock helper into the parent for helper-only calls, such as +# buildroot_lock_host in the stale-lock test. Tests that acquire a lock source +# the helper again inside subshells, so buildroot_acquire_lock can install and +# own an EXIT trap without replacing the parent test runner cleanup trap. +. scripts/prebuilt/artifact-recipes.sh + +run_test "artifact class contract" test_artifact_class_contract +run_test "test-tools recipe selection" test_test_tools_recipe_selection +run_test "prebuilt CI helper contract" test_prebuilt_ci_helper_contract +run_test "recipe manifest selected inputs" test_recipe_manifest_tracks_selected_inputs +run_test "build-artifacts CLI validation" test_build_artifacts_cli_validation +run_test "mk/external download rule" test_make_download_rule_uses_make_expansion +run_test "buildroot lock gitignore rule" test_buildroot_lock_path_is_ignored +run_test "rootfs ext4 ownership policy" test_rootfs_ext4_normalizes_ownership_unconditionally +run_test "rootfs ext4 tool selection" test_rootfs_ext4_tools_are_selectable +run_test "rootfs fakeroot probe helper" test_makefile_uses_rootfs_fakeroot_probe_helper +run_test "macOS setup rootfs hooks" test_macos_setup_exports_rootfs_tool_hooks +run_test "materializer plan validation" test_materializer_plan_validation +run_test "buildroot lock acquire/release" test_lock_acquire_release_contract +run_test "buildroot lock reentrant release" test_lock_reentrant_contract +run_test "buildroot lock release token check" test_lock_release_only_removes_own_token +run_test "buildroot lock stale detection" test_lock_reports_stale_without_deleting +run_test "buildroot lock unowned rejection" test_lock_reports_unowned_without_running_payload +run_test "buildroot lock contention" test_lock_waiter_acquires_after_release + +printf ' PASS\tbuild system tests\n' diff --git a/.github/actions/provide-guest-artifacts/action.yml b/.github/actions/provide-guest-artifacts/action.yml new file mode 100644 index 00000000..a62cdf26 --- /dev/null +++ b/.github/actions/provide-guest-artifacts/action.yml @@ -0,0 +1,33 @@ +name: Provide guest artifacts +description: Provide semu test jobs with guest artifacts from either workflow artifacts or the rolling release. +inputs: + use-workflow-artifacts: + description: Whether test jobs should consume guest artifacts produced by this workflow run. + required: true +runs: + using: composite + steps: + - name: validate inputs + shell: bash + env: + USE_WORKFLOW_ARTIFACTS: ${{ inputs.use-workflow-artifacts }} + run: | + case "$USE_WORKFLOW_ARTIFACTS" in + true|false) ;; + *) echo "::error::use-workflow-artifacts must be true or false, got: $USE_WORKFLOW_ARTIFACTS"; exit 1 ;; + esac + - name: download workflow-built guest artifacts + if: inputs.use-workflow-artifacts == 'true' + uses: actions/download-artifact@v7 + with: + name: guest-artifacts-raw + - name: materialize release guest artifacts + if: inputs.use-workflow-artifacts != 'true' + shell: bash + env: + PREBUILT_BOOTSTRAP_ON_404: ${{ github.repository != 'sysprog21/semu' }} + run: | + .ci/prebuilt/resolve-artifacts.sh > "$RUNNER_TEMP/prebuilt-plan-test.env" + PREBUILT_FORBID_BUILD=1 \ + PREBUILT_PLAN_FILE="$RUNNER_TEMP/prebuilt-plan-test.env" \ + .ci/prebuilt/materialize-artifacts.sh >/dev/null diff --git a/.github/actions/setup-semu/action.yml b/.github/actions/setup-semu/action.yml index bf976b55..1421a8db 100644 --- a/.github/actions/setup-semu/action.yml +++ b/.github/actions/setup-semu/action.yml @@ -25,4 +25,34 @@ runs: HOMEBREW_NO_AUTO_UPDATE: 1 HOMEBREW_NO_ANALYTICS: 1 run: | - brew install make dtc expect fakeroot e2fsprogs sdl2 + brew install make dtc expect fakeroot bash cpio coreutils e2fsprogs sdl2 + # fakeroot injects Homebrew's libfakeroot into each child process. Keep + # the commands that run inside rootfs_ext4.sh's fakeroot child on the + # same Homebrew arm64 toolchain instead of mixing in macOS arm64e system + # binaries on newer runners. + rootfs_fakeroot_shell="$(brew --prefix bash)/bin/bash" + cpio_prefix="$(brew --prefix cpio)" + coreutils_prefix="$(brew --prefix coreutils)" + rootfs_cpio= + for candidate in "$cpio_prefix/bin/cpio" "$cpio_prefix/bin/gcpio"; do + if [ -x "$candidate" ]; then + rootfs_cpio="$candidate" + break + fi + done + if [ -z "$rootfs_cpio" ]; then + echo "Homebrew cpio formula did not provide a usable cpio binary" >&2 + exit 1 + fi + rootfs_cp="$coreutils_prefix/bin/gcp" + rootfs_chown="$coreutils_prefix/bin/gchown" + for tool in "$rootfs_fakeroot_shell" "$rootfs_cpio" "$rootfs_cp" "$rootfs_chown"; do + if [ ! -x "$tool" ]; then + echo "Missing Homebrew rootfs tool: $tool" >&2 + exit 1 + fi + done + echo "ROOTFS_FAKEROOT_SHELL=$rootfs_fakeroot_shell" >> "$GITHUB_ENV" + echo "ROOTFS_CPIO=$rootfs_cpio" >> "$GITHUB_ENV" + echo "ROOTFS_CP=$rootfs_cp" >> "$GITHUB_ENV" + echo "ROOTFS_CHOWN=$rootfs_chown" >> "$GITHUB_ENV" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef49b137..1e50c2ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,81 +1,88 @@ name: CI +# No manual force-publish trigger here. A broken or deleted rolling release is +# external release state; maintainers inspect and repair it outside this CI plan +# instead of asking resolver/materializer to self-heal it. on: [push, pull_request] concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + # Do not cancel master push runs because they may be publishing the rolling prebuilt release. + cancel-in-progress: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }} permissions: contents: read +env: + # Use github.repository so forks and mirrors resolve their own rolling prebuilt release. + PREBUILT_URL: https://github.com/${{ github.repository }}/releases/download/prebuilt + jobs: - # PR-only: rebuild Image/rootfs.cpio/test-tools.img from source when local - # guest-artifact inputs have drifted from the prebuilt's recorded inputs. - # Without this, make check on a PR that touches configs, scripts, or - # target would silently exercise the stale prebuilt instead of the - # contributor's actual change. - # - # The artifacts produced here are published as a workflow artifact - # (reliable cross-job and cross-runner handoff) and consumed by the - # test jobs below. They are NOT published to the prebuilt GitHub - # release; that path is owned by .github/workflows/prebuilt.yml on - # master push. - # - # The job always runs but short-circuits unless this is a PR with - # drifted inputs, so non-drifting PRs and master pushes pay only the - # cost of a checkout plus drift check. - pr-prebuilt-build: + # Resolve guest artifacts through the CI prebuilt layer. GitHub cache and + # workflow artifacts are transport mechanisms; local Make builds do not use + # the resolver/materializer path. + guest-artifact-build: runs-on: ubuntu-24.04 outputs: - should_build: ${{ steps.detect.outputs.should_build }} + release_needs_update: ${{ steps.resolve.outputs.release_needs_update }} steps: - name: checkout code - uses: actions/checkout@v4 - - name: detect input drift - id: detect + uses: actions/checkout@v6 + - name: resolve release update plan + id: resolve + shell: bash + env: + # Forks and mirrors may not have created their own rolling prebuilt + # release yet. Only those repos may bootstrap on HTTP 404; the + # canonical repo should fail fast so maintainers can inspect the + # external release state manually. + PREBUILT_BOOTSTRAP_ON_404: ${{ github.repository != 'sysprog21/semu' }} + run: | + .ci/prebuilt/resolve-artifacts.sh > "$RUNNER_TEMP/prebuilt-plan-main.env" + .ci/github/append-github-output.sh "$RUNNER_TEMP/prebuilt-plan-main.env" \ + release_needs_update \ + platform_action_cache_tag \ + ci_cache_schema_tag + - name: list guest artifact paths + # Keep cache/upload path lists derived from the shared source-build + # artifact contract instead of duplicating raw artifact filenames in YAML. + id: artifact_paths + shell: bash + run: | + { + echo 'raw_paths<> "$GITHUB_OUTPUT" + - name: cache workflow-local raw guest artifacts + # platform_action_cache_tag is only a GitHub Actions cache namespace for + # the registered raw artifact bundle; per-class recipe keys still drive + # the resolver's correctness decisions. + if: steps.resolve.outputs.release_needs_update == 'true' + id: guest_artifact_cache + uses: actions/cache@v5 + with: + path: ${{ steps.artifact_paths.outputs.raw_paths }} + key: prebuilt-raw-${{ steps.resolve.outputs.platform_action_cache_tag }}-${{ steps.resolve.outputs.ci_cache_schema_tag }} + - name: materialization plan after cache restore + id: post_cache_resolve + if: steps.resolve.outputs.release_needs_update == 'true' shell: bash + env: + # Keep the post-cache resolver on the same fork/mirror bootstrap policy + # as the initial publish-gate resolver. + PREBUILT_BOOTSTRAP_ON_404: ${{ github.repository != 'sysprog21/semu' }} run: | - if [ "${{ github.event_name }}" != "pull_request" ]; then - echo "should_build=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - # The prebuilt's input-fingerprint lives on the release as the - # third line of prebuilt.sha1 (virtual name 'inputs'). Pull it - # from there instead of mirroring it in the source tree. - manifest_url="https://github.com/sysprog21/semu/releases/download/prebuilt/prebuilt.sha1" - curl --fail --silent --show-error --retry 3 --retry-delay 1 \ - -L -o prebuilt.sha1 "$manifest_url" - expected=$(awk '$2 == "inputs" {print $1}' prebuilt.sha1) - if [ -z "$expected" ]; then - # Manifest predates the inputs line (transition window after - # this workflow lands). Skip drift detection until prebuilt.yml - # republishes with the new format; treat as no drift to avoid - # gratuitous 90-minute from-source rebuilds on every PR. - echo "manifest has no inputs hash yet; skipping drift detection" - echo "should_build=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - live=$(cat \ - configs/linux.config \ - configs/busybox.config \ - configs/buildroot.config \ - configs/x11.config \ - configs/riscv-cross-file \ - scripts/build-image.sh \ - scripts/rootfs_ext4.sh \ - target/init \ - target/local-env.sh \ - | sha1sum | awk '{print $1}') - if [ "$live" = "$expected" ]; then - echo "PR inputs match the prebuilt ($live); skipping rebuild" - echo "should_build=false" >> "$GITHUB_OUTPUT" - else - echo "PR inputs drifted ($live != $expected); will rebuild from source" - echo "should_build=true" >> "$GITHUB_OUTPUT" - fi + .ci/prebuilt/resolve-artifacts.sh > "$RUNNER_TEMP/prebuilt-plan-post-cache.env" + .ci/github/append-github-output.sh "$RUNNER_TEMP/prebuilt-plan-post-cache.env" \ + requires_build - name: install build dependencies - if: steps.detect.outputs.should_build == 'true' + if: steps.resolve.outputs.release_needs_update == 'true' && steps.post_cache_resolve.outputs.requires_build == 'true' run: | sudo apt-get update sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ @@ -84,24 +91,37 @@ jobs: git python3 libssl-dev libelf-dev wget \ meson ninja-build pkg-config timeout-minutes: 5 - - name: build kernel, rootfs, and test tools disk from source - if: steps.detect.outputs.should_build == 'true' - run: ./scripts/build-image.sh --all --x11 --directfb2-test + - name: materialize guest artifacts + if: steps.resolve.outputs.release_needs_update == 'true' + shell: bash + env: + PREBUILT_PLAN_FILE: ${{ runner.temp }}/prebuilt-plan-post-cache.env + run: .ci/prebuilt/materialize-artifacts.sh timeout-minutes: 180 - - name: upload PR-built artifacts - if: steps.detect.outputs.should_build == 'true' - uses: actions/upload-artifact@v4 + - name: package prebuilt artifacts + if: steps.resolve.outputs.release_needs_update == 'true' + shell: bash + run: .ci/prebuilt/package.sh + - name: upload raw guest artifacts + if: steps.resolve.outputs.release_needs_update == 'true' + uses: actions/upload-artifact@v7 with: - name: prebuilt-pr - path: | - Image - rootfs.cpio - test-tools.img + name: guest-artifacts-raw + path: ${{ steps.artifact_paths.outputs.raw_paths }} + include-hidden-files: true + retention-days: 1 + if-no-files-found: error + - name: upload packaged guest artifacts + if: steps.resolve.outputs.release_needs_update == 'true' + uses: actions/upload-artifact@v7 + with: + name: guest-artifacts-packaged + path: ${{ steps.artifact_paths.outputs.packaged_paths }} retention-days: 1 if-no-files-found: error semu-linux: - needs: pr-prebuilt-build + needs: guest-artifact-build runs-on: ubuntu-24.04 strategy: fail-fast: false @@ -112,30 +132,17 @@ jobs: - libjack-jackd2-dev steps: - name: checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: submodules: recursive - # Drift PR: pull the freshly-built Image/rootfs.cpio/test-tools.img from - # pr-prebuilt-build via workflow artifact (reliable, not cache). - - name: download PR-built external artifacts - if: needs.pr-prebuilt-build.outputs.should_build == 'true' - uses: actions/download-artifact@v4 + # provide-guest-artifacts places raw guest artifacts in the workspace before + # make runs; local Make targets then treat them as normal existing files. + - name: provide guest artifacts + uses: ./.github/actions/provide-guest-artifacts with: - name: prebuilt-pr - # Non-drift PR or master push: cache the release-downloaded artifacts - # across runs. Include mk/external.mk so checksum or input-pin bumps - # after a republish invalidate the old external artifacts. - - name: cache external downloads - if: needs.pr-prebuilt-build.outputs.should_build != 'true' - uses: actions/cache@v4 - with: - path: | - Image - rootfs.cpio - test-tools.img - key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'configs/x11.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} + use-workflow-artifacts: ${{ needs.guest-artifact-build.outputs.release_needs_update }} - name: cache submodule builds - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | mini-gdbstub/build @@ -143,13 +150,6 @@ jobs: key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules', 'mini-gdbstub/**', 'minislirp/**') }} restore-keys: | ${{ runner.os }}-submodules- - - name: cache apt packages - uses: actions/cache@v4 - with: - path: /var/cache/apt/archives - key: ${{ runner.os }}-apt-${{ hashFiles('.github/actions/setup-semu/action.yml') }} - restore-keys: | - ${{ runner.os }}-apt- - name: install-dependencies uses: ./.github/actions/setup-semu - name: install sound multiplexer ${{ matrix.dependency }} @@ -166,19 +166,28 @@ jobs: shell: bash timeout-minutes: 10 - name: netdev test - run: sudo .ci/test-netdev.sh + # Preserve the rootfs tool hooks across sudo so Makefile feature detection sees the + # same fakeroot-capable environment as the non-sudo build and test steps. + run: | + sudo -E env \ + PATH="$PATH" \ + ROOTFS_FAKEROOT_SHELL="${ROOTFS_FAKEROOT_SHELL:-/bin/sh}" \ + ROOTFS_CPIO="${ROOTFS_CPIO:-cpio}" \ + ROOTFS_CP="${ROOTFS_CP:-cp}" \ + ROOTFS_CHOWN="${ROOTFS_CHOWN:-chown}" \ + .ci/device-smoke/test-netdev.sh shell: bash timeout-minutes: 10 - name: sound test - run: .ci/test-sound.sh + run: .ci/device-smoke/test-sound.sh shell: bash timeout-minutes: 5 - name: virtio-input test - run: .ci/test-vinput.sh + run: .ci/device-smoke/test-vinput.sh shell: bash timeout-minutes: 5 - name: virtio-gpu test - run: .ci/test-gpu.sh + run: .ci/device-smoke/test-gpu.sh shell: bash timeout-minutes: 10 @@ -186,29 +195,21 @@ jobs: # default boot mode is /dev/vda. Single slim job: fresh build with # ENABLE_EXTERNAL_ROOT=0, then run autorun. semu-linux-initramfs: - needs: pr-prebuilt-build + needs: guest-artifact-build runs-on: ubuntu-24.04 steps: - name: checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v6 with: submodules: recursive - - name: download PR-built external artifacts - if: needs.pr-prebuilt-build.outputs.should_build == 'true' - uses: actions/download-artifact@v4 - with: - name: prebuilt-pr - - name: cache external downloads - if: needs.pr-prebuilt-build.outputs.should_build != 'true' - uses: actions/cache@v4 + # provide-guest-artifacts places raw guest artifacts in the workspace before + # make runs; local Make targets then treat them as normal existing files. + - name: provide guest artifacts + uses: ./.github/actions/provide-guest-artifacts with: - path: | - Image - rootfs.cpio - test-tools.img - key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'configs/x11.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} + use-workflow-artifacts: ${{ needs.guest-artifact-build.outputs.release_needs_update }} - name: cache submodule builds - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | mini-gdbstub/build @@ -216,13 +217,6 @@ jobs: key: ${{ runner.os }}-submodules-${{ hashFiles('.gitmodules', 'mini-gdbstub/**', 'minislirp/**') }} restore-keys: | ${{ runner.os }}-submodules- - - name: cache apt packages - uses: actions/cache@v4 - with: - path: /var/cache/apt/archives - key: ${{ runner.os }}-apt-${{ hashFiles('.github/actions/setup-semu/action.yml') }} - restore-keys: | - ${{ runner.os }}-apt- - name: install-dependencies uses: ./.github/actions/setup-semu - name: legacy initramfs build @@ -235,32 +229,20 @@ jobs: timeout-minutes: 10 semu-macOS: - needs: pr-prebuilt-build + needs: guest-artifact-build runs-on: macos-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: recursive - # macOS cannot run buildroot. On a drift PR it must consume - # pr-prebuilt-build's artifact (built on a linux runner) -- a cache - # miss here would silently fall back to downloading the stale - # release, defeating the whole point of the drift-detection logic. - - name: download PR-built external artifacts - if: needs.pr-prebuilt-build.outputs.should_build == 'true' - uses: actions/download-artifact@v4 - with: - name: prebuilt-pr - - name: cache external downloads - if: needs.pr-prebuilt-build.outputs.should_build != 'true' - uses: actions/cache@v4 + # provide-guest-artifacts places raw guest artifacts in the workspace before + # make runs; local Make targets then treat them as normal existing files. + - name: provide guest artifacts + uses: ./.github/actions/provide-guest-artifacts with: - path: | - Image - rootfs.cpio - test-tools.img - key: external-${{ hashFiles('mk/external.mk', 'configs/linux.config', 'configs/busybox.config', 'configs/buildroot.config', 'configs/x11.config', 'scripts/build-image.sh', 'scripts/rootfs_ext4.sh', 'target/**') }} + use-workflow-artifacts: ${{ needs.guest-artifact-build.outputs.release_needs_update }} - name: cache submodule builds - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | mini-gdbstub/build @@ -279,48 +261,58 @@ jobs: shell: bash timeout-minutes: 20 - name: netdev test - # NOTE: vmnet requires sudo privileges which may not be available in GitHub Actions - # This test is conditional and will be skipped if sudo is not available + # NOTE: vmnet requires sudo privileges which may not be available in GitHub Actions. + # Preserve the rootfs tool hooks across sudo so Makefile feature detection sees the + # same fakeroot-capable environment as the non-sudo build and test steps. run: | if sudo -n true 2>/dev/null; then echo "sudo available, running netdev test" - sudo .ci/test-netdev.sh + sudo -E env \ + PATH="$PATH" \ + ROOTFS_FAKEROOT_SHELL="${ROOTFS_FAKEROOT_SHELL:-/bin/sh}" \ + ROOTFS_CPIO="${ROOTFS_CPIO:-cpio}" \ + ROOTFS_CP="${ROOTFS_CP:-cp}" \ + ROOTFS_CHOWN="${ROOTFS_CHOWN:-chown}" \ + .ci/device-smoke/test-netdev.sh else echo "Skipping netdev test: sudo not available in GitHub Actions runner" echo "vmnet.framework requires root privileges or com.apple.vm.networking entitlement" fi shell: bash timeout-minutes: 20 - if: ${{ success() }} - name: sound test - run: .ci/test-sound.sh + run: .ci/device-smoke/test-sound.sh shell: bash timeout-minutes: 20 - name: virtio-input test - run: .ci/test-vinput.sh + run: .ci/device-smoke/test-vinput.sh shell: bash timeout-minutes: 20 - name: virtio-gpu test - run: .ci/test-gpu.sh + run: .ci/device-smoke/test-gpu.sh shell: bash timeout-minutes: 20 + build_system: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - name: Check build system scripts + run: .ci/test-build-system.sh + shell: bash + timeout-minutes: 5 + coding_style: runs-on: ubuntu-24.04 permissions: contents: read pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: submodules: recursive - - name: cache apt packages - uses: actions/cache@v4 - with: - path: /var/cache/apt/archives - key: ${{ runner.os }}-apt-clang-format-20 - restore-keys: | - ${{ runner.os }}-apt- - name: Install clang-format run: | sudo apt-get update @@ -337,10 +329,65 @@ jobs: if: github.event_name == 'pull_request' env: REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: .ci/suggest-format.sh + run: .ci/github/suggest-format.sh shell: bash timeout-minutes: 5 - name: Check formatting (fail on violations) run: .ci/check-format.sh shell: bash timeout-minutes: 5 + + publish-prebuilt: + needs: + - guest-artifact-build + - semu-linux + - semu-linux-initramfs + - semu-macOS + - build_system + - coding_style + # This gate follows resolver release drift: artifact recipe keys plus any + # release-contract checks the resolver explicitly enforces. + if: github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.guest-artifact-build.outputs.release_needs_update == 'true' + runs-on: ubuntu-24.04 + permissions: + contents: write + concurrency: + group: prebuilt-release + cancel-in-progress: false + steps: + - name: checkout code + uses: actions/checkout@v6 + - name: download packaged guest artifacts + uses: actions/download-artifact@v7 + with: + name: guest-artifacts-packaged + - name: verify packaged artifacts + run: .ci/prebuilt/verify-package.sh + - name: list packaged guest artifact paths + # Publish the same registered artifacts that the package job produced. + id: artifact_paths + shell: bash + run: | + { + echo 'packaged_paths<> "$GITHUB_OUTPUT" + - name: update prebuilt prerelease + uses: softprops/action-gh-release@v3 + with: + tag_name: prebuilt + name: Prebuilt images (rolling) + prerelease: true + fail_on_unmatched_files: true + files: ${{ steps.artifact_paths.outputs.packaged_paths }} + body: | + Rolling prerelease of the Linux kernel, Buildroot rootfs, + and optional test tools disk consumed by CI prebuilt jobs. + Re-published from CI whenever any input that defines the + prebuilt content changes on master. + + Recipe keys for each artifact class are published in the attached + `prebuilt.sha1` manifest; CI consumers read it directly, so no + source-tree edit is needed on a republish. diff --git a/.github/workflows/prebuilt.yml b/.github/workflows/prebuilt.yml deleted file mode 100644 index 5b52ac44..00000000 --- a/.github/workflows/prebuilt.yml +++ /dev/null @@ -1,113 +0,0 @@ -name: Publish prebuilt images - -# Builds the Linux kernel, Buildroot rootfs, and optional test tools disk that the -# rest of CI and make on a fresh checkout consume, then publishes them as -# assets on a fixed-tag GitHub prerelease so the download URL stays stable -# across rebuilds, keeping large binary artifacts out of the source tree. -# -# Triggers automatically on master pushes that touch any input listed -# in the paths filter below, and can be invoked manually via -# workflow_dispatch. The archive SHA1 sums and the input-fingerprint -# hash are all read at make-time / CI-time from the prebuilt.sha1 -# manifest uploaded with the archives, so no source-tree edit is -# needed on a republish. - -on: - workflow_dispatch: - push: - branches: [master] - # Republish the prebuilt artifacts whenever any input that defines - # what gets baked into Image / rootfs.cpio actually changes. The list - # mirrors PREBUILT_INPUTS in mk/external.mk -- keep them in sync. - paths: - - 'configs/linux.config' - - 'configs/busybox.config' - - 'configs/buildroot.config' - - 'configs/x11.config' - - 'configs/riscv-cross-file' - - 'scripts/build-image.sh' - - 'scripts/rootfs_ext4.sh' - - 'target/**' - - '.ci/publish-prebuilt.sh' - - '.github/workflows/prebuilt.yml' - -permissions: - contents: write - -concurrency: - group: prebuilt - cancel-in-progress: false - -jobs: - build-and-publish: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Install build dependencies - run: | - sudo apt-get update - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - build-essential \ - bc \ - bison \ - flex \ - cpio \ - fakeroot \ - e2fsprogs \ - git \ - python3 \ - libssl-dev \ - libelf-dev \ - wget \ - meson \ - ninja-build \ - pkg-config - - - name: Build Buildroot, Linux, and test tools disk - run: ./scripts/build-image.sh --all --x11 --directfb2-test - - - name: Compress and checksum artifacts - id: checksum - # The shell logic lives in .ci/publish-prebuilt.sh so it can be - # exercised locally and stays out of the YAML. - run: | - set -euo pipefail - .ci/publish-prebuilt.sh >> "$GITHUB_OUTPUT" - ls -la Image.bz2 rootfs.cpio.bz2 test-tools.img.bz2 prebuilt.sha1 - - - name: Update prebuilt prerelease - uses: softprops/action-gh-release@v2 - with: - tag_name: prebuilt - name: Prebuilt images (rolling) - prerelease: true - # Replace existing assets at the prebuilt tag rather than - # appending duplicates with new names. - fail_on_unmatched_files: true - files: | - Image.bz2 - rootfs.cpio.bz2 - test-tools.img.bz2 - prebuilt.sha1 - body: | - Rolling prerelease of the Linux kernel, Buildroot rootfs, - and optional test tools disk consumed by `mk/external.mk`. - Re-published whenever any input that defines the prebuilt - content changes. - - All checksums (archive hashes plus the source-input - fingerprint used for drift detection) are published in the - `prebuilt.sha1` manifest below; consumers read it directly, - so no source-tree edit is needed on a republish. - - ## Raw checksums - - ``` - ${{ steps.checksum.outputs.kernel_sha1 }} Image.bz2 - ${{ steps.checksum.outputs.initrd_sha1 }} rootfs.cpio.bz2 - ${{ steps.checksum.outputs.test_tools_sha1 }} test-tools.img.bz2 - ${{ steps.checksum.outputs.inputs_sha1 }} inputs (configs + scripts + target files, concatenated) - ``` diff --git a/.gitignore b/.gitignore index f387cb41..1a8f6b9e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,13 +9,15 @@ Image ext4.img test-tools.img rootfs.cpio +Image.bz2 +rootfs.cpio.bz2 +test-tools.img.bz2 prebuilt.sha1 +.stamps/ +.semu-buildroot.lock # intermediate riscv-harts.dtsi -.smp_stamp -.dtb-config.stamp -.build-config.stamp # Build directories buildroot/ diff --git a/Makefile b/Makefile index cd3347a8..5c4bd4b3 100644 --- a/Makefile +++ b/Makefile @@ -11,24 +11,26 @@ DT_CFLAGS := -D CLOCK_FREQ=$(CLOCK_FREQ) CFLAGS += $(DT_CFLAGS) OBJS_EXTRA := -# command line option -OPTS := LDFLAGS := +# Caller-facing hooks used by scripts/rootfs_ext4.sh when the default host +# tools are unsuitable for generating ext4.img under fakeroot. +ROOTFS_CPIO ?= cpio +ROOTFS_CP ?= cp +ROOTFS_CHOWN ?= chown +ROOTFS_FAKEROOT_SHELL ?= /bin/sh + # external rootfs: boot from /dev/vda instead of unpacking initramfs. # Implies VIRTIOBLK and pulls the userland from rootfs.cpio into ext4.img. # Default-on. If fakeroot is missing or non-functional, fall back to the -# initramfs path so the build still succeeds without forcing a new -# dependency on the user. The probe must exec a real binary -- `fakeroot -# true` looks correct but `true` is a bash builtin, so the wrapper -# script never actually runs the DYLD_INSERT_LIBRARIES path. On macOS -# arm64 the brew fakeroot dylib is built for arm64 and refuses to load -# into arm64e system binaries (cpio, mkfs.ext4); using `/bin/sh -c :` -# forces an exec so we catch that failure here instead of mid-build. +# initramfs path so the build still succeeds without forcing a new dependency +# on the user. The probe exercises the same shell/cp/chown hooks used by the +# ext4 image path so hosts with partial fakeroot support fail before image +# generation starts. ENABLE_EXTERNAL_ROOT ?= 1 ifeq ($(call has, EXTERNAL_ROOT), 1) - ifneq (0,$(shell fakeroot /bin/sh -c : >/dev/null 2>&1; echo $$?)) + ifneq (0,$(shell ROOTFS_FAKEROOT_SHELL="$(ROOTFS_FAKEROOT_SHELL)" ROOTFS_CP="$(ROOTFS_CP)" ROOTFS_CHOWN="$(ROOTFS_CHOWN)" scripts/check-rootfs-fakeroot.sh >/dev/null 2>&1; echo $$?)) $(warning fakeroot not usable; falling back to initramfs boot.) $(warning Install a working fakeroot to enable /dev/vda boot.) override ENABLE_EXTERNAL_ROOT := 0 @@ -47,7 +49,6 @@ MKFS_EXT4 ?= mkfs.ext4 ifeq ($(call has, VIRTIOBLK), 1) OBJS_EXTRA += virtio-blk.o DISKIMG_FILE := ext4.img - OPTS += -d $(DISKIMG_FILE) MKFS_EXT4 := $(shell which $(MKFS_EXT4)) ifndef MKFS_EXT4 MKFS_EXT4 := $(shell which $$(brew --prefix e2fsprogs)/sbin/mkfs.ext4) @@ -70,7 +71,6 @@ $(call set-feature, VIRTIOFS) SHARED_DIRECTORY ?= ./shared ifeq ($(call has, VIRTIOFS), 1) OBJS_EXTRA += virtio-fs.o - OPTS += -s $(SHARED_DIRECTORY) endif NETDEV ?= tap @@ -234,6 +234,17 @@ OBJS := \ $(OBJS_EXTRA) deps := $(OBJS:%.o=.%.o.d) +STAMP_DIR := .stamps +BUILDROOT_OUTPUT_DIRS := buildroot/output buildroot/output-* +SOURCE_BUILD_DIRS := \ + buildroot \ + linux \ + DirectFB2 \ + DirectFB-examples \ + directfb \ + extra_packages +BUILD_CONFIG_STAMP := $(STAMP_DIR)/build-config.stamp +DTB_CONFIG_STAMP := $(STAMP_DIR)/dtb-config.stamp BUILD_CONFIG := CC=$(CC) $(strip $(CFLAGS)) GDBSTUB_LIB := mini-gdbstub/build/libgdbstub.a @@ -265,13 +276,14 @@ $(BIN): $(OBJS) $(VECHO) " LD\t$@\n" $(Q)$(CC) -o $@ $^ $(LDFLAGS) -.build-config.stamp: FORCE +$(BUILD_CONFIG_STAMP): FORCE + @mkdir -p $(@D) @if [ ! -f $@ ] || [ "$$(cat $@ 2>/dev/null)" != "$(BUILD_CONFIG)" ]; then \ printf '%s\n' "$(BUILD_CONFIG)" > $@; \ rm -f $(OBJS) $(deps); \ fi -%.o: %.c .build-config.stamp +%.o: %.c $(BUILD_CONFIG_STAMP) $(VECHO) " CC\t$@\n" $(Q)$(CC) -o $@ $(CFLAGS) -c -MMD -MF .$@.d $< @@ -304,17 +316,18 @@ CFLAGS += -D SEMU_BOOT_TARGET_TIME=10 SMP ?= 1 # Track DTB inputs so config changes regenerate both the hart DTSI and DTB. -.dtb-config.stamp: FORCE +$(DTB_CONFIG_STAMP): FORCE + @mkdir -p $(@D) @if [ ! -f $@ ] || [ "$$(cat $@ 2>/dev/null)" != "$(DTB_CONFIG)" ]; then \ printf '%s\n' "$(DTB_CONFIG)" > $@; \ rm -f riscv-harts.dtsi minimal.dtb; \ fi .PHONY: riscv-harts.dtsi -riscv-harts.dtsi: .dtb-config.stamp +riscv-harts.dtsi: $(DTB_CONFIG_STAMP) $(Q)python3 scripts/gen-hart-dts.py $@ $(SMP) $(CLOCK_FREQ) -minimal.dtb: minimal.dts riscv-harts.dtsi .dtb-config.stamp +minimal.dtb: minimal.dts riscv-harts.dtsi $(DTB_CONFIG_STAMP) $(VECHO) " DTC\t$@\n" $(Q)$(RM) $@ $(Q)$(CC) -nostdinc -E -P -x assembler-with-cpp -undef \ @@ -329,7 +342,10 @@ include mk/external.mk ifeq ($(call has, EXTERNAL_ROOT), 1) ext4.img: $(INITRD_DATA) scripts/rootfs_ext4.sh - $(Q)MKFS_EXT4="$(MKFS_EXT4)" scripts/rootfs_ext4.sh $(INITRD_DATA) $@ + $(Q)MKFS_EXT4="$(MKFS_EXT4)" ROOTFS_CPIO="$(ROOTFS_CPIO)" \ + ROOTFS_CP="$(ROOTFS_CP)" ROOTFS_CHOWN="$(ROOTFS_CHOWN)" \ + ROOTFS_FAKEROOT_SHELL="$(ROOTFS_FAKEROOT_SHELL)" \ + scripts/rootfs_ext4.sh $(INITRD_DATA) $@ else ext4.img: $(Q)dd if=/dev/zero of=$@ bs=4k count=600 @@ -345,28 +361,39 @@ $(SHARED_DIRECTORY): ifeq ($(call has, EXTERNAL_ROOT), 1) INITRD_DEP := -INITRD_OPT := else INITRD_DEP := $(INITRD_DATA) -INITRD_OPT := -i $(INITRD_DATA) endif +HEADLESS ?= 1 +KERNEL_OPT := -k $(KERNEL_DATA) +DTB_OPT := -b minimal.dtb +HEADLESS_OPT := $(if $(filter 0 false no,$(HEADLESS)),,-H) +INITRD_OPT := $(if $(INITRD_DEP),-i $(INITRD_DEP)) +DISKIMG_OPT := $(if $(DISKIMG_FILE),-d $(DISKIMG_FILE)) +VIRTIOFS_OPT := $(if $(filter 1,$(call has,VIRTIOFS)),-s $(SHARED_DIRECTORY)) +SMP_OPT := -c $(SMP) +NETDEV_OPT := $(if $(NETDEV),-n $(NETDEV)) + .PHONY: bench-login bench-login: $(BIN) minimal.dtb $(KERNEL_DATA) $(INITRD_DEP) $(DISKIMG_FILE) - $(Q)/usr/bin/time -p expect scripts/bench-login.expect \ - ./$(BIN) -k $(KERNEL_DATA) -b minimal.dtb -H $(INITRD_OPT) $(OPTS) + $(Q)/usr/bin/time -p expect scripts/bench-login.expect ./$(BIN) $(strip $(KERNEL_OPT) $(DTB_OPT) $(HEADLESS_OPT) $(INITRD_OPT) $(DISKIMG_OPT) $(VIRTIOFS_OPT)) check: $(BIN) minimal.dtb $(KERNEL_DATA) $(INITRD_DEP) $(DISKIMG_FILE) $(SHARED_DIRECTORY) @$(call notice, Ready to launch Linux kernel. Please be patient.) - $(Q)./$(BIN) -k $(KERNEL_DATA) -c $(SMP) -b minimal.dtb -H $(INITRD_OPT) $(if $(NETDEV),-n $(NETDEV)) $(OPTS) + $(Q)./$(BIN) $(strip $(KERNEL_OPT) $(DTB_OPT) $(HEADLESS_OPT) $(INITRD_OPT) $(DISKIMG_OPT) $(VIRTIOFS_OPT) $(SMP_OPT) $(NETDEV_OPT)) -BUILD_IMAGE_ARGS ?= --all -build-image: - scripts/build-image.sh $(BUILD_IMAGE_ARGS) +ARTIFACTS ?= all +.PHONY: build-artifacts +build-artifacts: + scripts/build-artifacts.sh $(ARTIFACTS) clean: $(Q)$(RM) $(BIN) $(OBJS) $(deps) - $(Q)$(MAKE) -C mini-gdbstub clean + $(Q)$(RM) -r $(BUILDROOT_OUTPUT_DIRS) + $(Q)if [ -f mini-gdbstub/Makefile ]; then \ + $(MAKE) -C mini-gdbstub clean; \ + fi $(Q)if [ -n "$(MINISLIRP_DIR)" ] && [ -d "$(MINISLIRP_DIR)/src" ]; then \ $(MAKE) -C $(MINISLIRP_DIR)/src clean; \ fi @@ -374,9 +401,11 @@ clean: distclean: clean $(Q)$(RM) riscv-harts.dtsi $(Q)$(RM) minimal.dtb - $(Q)$(RM) .dtb-config.stamp - $(Q)$(RM) .build-config.stamp $(Q)$(RM) Image rootfs.cpio prebuilt.sha1 + $(Q)$(RM) Image.bz2 rootfs.cpio.bz2 test-tools.img.bz2 $(Q)$(RM) ext4.img test-tools.img + $(Q)$(RM) -r $(SOURCE_BUILD_DIRS) + $(Q)$(RM) -r $(STAMP_DIR) + $(Q)$(RM) -r .semu-buildroot.lock -include $(deps) diff --git a/README.md b/README.md index e0ede423..3193a138 100644 --- a/README.md +++ b/README.md @@ -29,11 +29,13 @@ A minimalist RISC-V system emulator capable of running Linux the kernel and corr [Device Tree](https://www.kernel.org/doc/html/latest/devicetree/) compiler (dtc) is required. To install it on Debian/Ubuntu Linux, enter the following command: + ```shell $ sudo apt install device-tree-compiler ``` For macOS, use the following command: + ```shell $ brew install dtc ``` @@ -47,6 +49,7 @@ usage is the same as the mke2fs command. For most GNU/Linux distributions, `mkfs.ext4` command should be installed in advance. For macOS, use the following command: + ```shell $ brew install e2fsprogs ``` @@ -54,11 +57,13 @@ $ brew install e2fsprogs ## Build and Run Build the emulator: + ```shell $ make ``` Download the prebuilt guest artifacts and run the default check: + ```shell $ make check ``` @@ -70,12 +75,14 @@ and `ext4.img`, and boots `semu` headlessly with an equivalent command line: $ ./semu -k Image -c 1 -b minimal.dtb -H -d ext4.img ``` -If `ENABLE_EXTERNAL_ROOT=0` is used, `make check` switches to the legacy -initramfs path and passes `-i rootfs.cpio` instead of `-d ext4.img`. +If `ENABLE_EXTERNAL_ROOT=0` is used, `make check` adds the legacy +initramfs payload with `-i rootfs.cpio`. The default disk remains `ext4.img`; +set `DISKIMG_FILE=` when you want to boot the initramfs path without `-d ext4.img`. Please be patient while `semu` is running. Reference output: + ``` Starting syslogd: OK Starting klogd: OK @@ -90,17 +97,19 @@ Enter `root` to access shell. You can exit the emulator using: \. (press Ctrl+A, leave it, afterwards press X) -To test virtio-gpu with a visible SDL window, run `semu` manually without `-H`. -Make sure `sdl2-config` is in `PATH`, then build the emulator, DTB, kernel, and -test tools disk. Press `Ctrl+Alt+G` to release the mouse cursor from the SDL -window: +`make check` accepts runtime flags for common local test variants. Set +`HEADLESS=0` to show the SDL window, and set `DISKIMG_FILE` to choose the ext4 +image exposed as `/dev/vda`: ```shell -$ sdl2-config --version -$ make semu minimal.dtb Image test-tools.img -$ ./semu -k Image -c 1 -b minimal.dtb -d test-tools.img +$ make check HEADLESS=0 +$ make check HEADLESS=0 DISKIMG_FILE=test-tools.img ``` +The default disk is `ext4.img`. `test-tools.img` is an optional replacement disk +image for tests that need larger third-party tools, such as DirectFB2. Press +`Ctrl+Alt+G` to release the mouse cursor from the SDL window. + Log in as `root`, source the test-tools image environment, and run one of the DirectFB2 examples: @@ -127,7 +136,9 @@ project and can be listed in the guest with: * `disk-image` is the ext4 image exposed as `/dev/vda` to the guest. The default boot path mounts this as the root filesystem; `make` builds it from `rootfs.cpio` via `scripts/rootfs_ext4.sh`. -* `shared-directory` is optional, as it specifies the path of a directory on the host that will be shared with the guest operating system through virtio-fs, enabling file access from the guest via a virtual filesystem mount. +* `shared-directory` is optional, as it specifies the path of a directory on the host that + will be shared with the guest operating system through virtio-fs, enabling file access + from the guest via a virtual filesystem mount. * `-H` (or `--headless`) skips SDL window creation; useful for CI and `make check`. * `initrd-image` is optional and only used on the *legacy* boot path. The default `minimal.dtb` built with `ENABLE_EXTERNAL_ROOT=1` does not @@ -142,15 +153,18 @@ runs `/sbin/init` from the ext4 root, skipping the initramfs unpack step entirely. This is faster, avoids the RCU-stall the kernel hits when unpacking a large cpio, and matches how real systems deploy. The `ext4.img` is built from `rootfs.cpio` via `scripts/rootfs_ext4.sh`, -which requires `fakeroot` and `mkfs.ext4`. +which requires `fakeroot`, `mkfs.ext4`, and a fakeroot-compatible rootfs +copy/ownership toolchain. -The rolling `prebuilt` release provides an optional `test-tools.img.bz2` for -larger test/user tools that should not inflate `rootfs.cpio` or the default -`ext4.img`. Use `make test-tools.img` to download it. +`test-tools.img` is an optional replacement disk for larger test/user tools that +should not inflate `rootfs.cpio` or the default `ext4.img`. Use +`make test-tools.img` to download the rolling prebuilt disk, or +`make build-artifacts ARTIFACTS=test-tools` to rebuild it locally from source. -If `fakeroot` is missing, the build falls back to the legacy initramfs -path (`-i rootfs.cpio`) automatically and prints a one-line warning. To -force the legacy path explicitly: +If `fakeroot` or the selected rootfs toolchain cannot run under fakeroot, +the build falls back to the legacy initramfs path (`-i rootfs.cpio`) +automatically and prints a one-line warning. To force the legacy path +explicitly: ```shell $ make ENABLE_EXTERNAL_ROOT=0 @@ -196,87 +210,104 @@ $ umount [shared-directory] * `shared-directory` is the path of a directory you want to unmount in semu. +## Build guest artifacts from source -## Build Linux kernel image and root file system - -An automated build script is provided to compile the RISC-V cross-compiler, Busybox, and Linux kernel from source. +Fresh local `make` invocations download missing guest artifacts from the rolling +`prebuilt` release by default. An explicit local source-build path is provided +to compile the RISC-V cross-compiler, Busybox, and Linux kernel from source. Please note that it only supports the Linux host environment. -To build everything, simply run: +Use the Make target as the public entry point for local guest artifact source +builds: ```shell -$ make build-image +$ make build-artifacts ``` -This command invokes the underlying script: `scripts/build-image.sh --all`, which also offers more flexible usage options. - -### Script Usage +`ARTIFACTS` is passed to the lower-level build script. To build +selected artifact classes, set it explicitly: +```shell +$ make build-artifacts ARTIFACTS=image +$ make build-artifacts ARTIFACTS=rootfs +$ make build-artifacts ARTIFACTS="rootfs --no-ext4" +$ make build-artifacts ARTIFACTS=test-tools +$ make build-artifacts ARTIFACTS="all --clean-build" +$ make build-artifacts ARTIFACTS="all --full-rebuild" ``` -./scripts/build-image.sh [--buildroot] [--linux] [--directfb2-test] [--all] [--no-ext4] [--clean-build] [--help] -Options: - --buildroot Build Buildroot userland (produces rootfs.cpio and, - unless --no-ext4 is given, ext4.img for vda boot) - --directfb2-test Build test-tools.img with the DirectFB2 test payload - --linux Build the Linux kernel - --all Build both Buildroot and Linux - --no-ext4 Skip ext4.img generation; produce only rootfs.cpio - (matches the legacy ENABLE_EXTERNAL_ROOT=0 path) - --clean-build Remove buildroot/ and/or linux/ before building; - with --directfb2-test, also remove DirectFB2 build outputs - --help Show this message -``` +`make build-artifacts` is only a Make wrapper around `scripts/build-artifacts.sh`; +pass class targets and class-specific flags through `ARTIFACTS` so +the script remains the single local artifact-build CLI. -### Examples +Local `make` treats `Image`, `rootfs.cpio`, and `test-tools.img` as ordinary +file targets. If one is missing, Make downloads it from the rolling prebuilt +release; if one already exists, Make will use it. These local downloads verify +the release archive checksum recorded in `prebuilt.sha1`, but they are not +checked against the current working tree or CI recipe keys. -Build the Linux kernel only: +Run `make build-artifacts` explicitly after changing guest configs when you want the +build system to update those artifacts from source. CI prebuilt cache and +recipe-key stamps are handled separately under `.ci/prebuilt/` and are not part +of the local build interface. -``` -$ scripts/build-image.sh --linux -``` +The source builder keeps the default rootfs and optional test-tools rootfs in +separate Buildroot output directories, so test-tools packages do not leak +back into the default `rootfs.cpio` or `ext4.img`. Use `--clean-build` to +remove selected raw artifact outputs and those Buildroot output directories +while preserving source checkouts and download cache; use `--full-rebuild` to +remove selected raw artifact outputs and external source/build trees before +rebuilding from pinned revisions. -Build Buildroot (produces both `rootfs.cpio` and `ext4.img`): +Targets: ``` -$ scripts/build-image.sh --buildroot +image Build the Linux Image. This prepares the Buildroot + toolchain if needed but does not publish rootfs.cpio or + test-tools.img as final outputs. +rootfs Build Buildroot rootfs.cpio and, unless --no-ext4 is + given, derive ext4.img from it. +test-tools Build test-tools.img. By default this uses the canonical + X11 + DirectFB2 test payload recipe. +all Build image, rootfs, and canonical test-tools. ``` -Build Buildroot for the legacy initramfs-only path (no ext4): +Options: ``` -$ scripts/build-image.sh --buildroot --no-ext4 +--x11 Select the X11/C++ runtime recipe entry for test-tools. When + any test-tools recipe flag is given, only selected + entries are included. +--directfb2-test Select the DirectFB2 test payload recipe entry for test-tools. +--no-ext4 With rootfs, skip ext4.img generation and produce only + rootfs.cpio (matches the legacy ENABLE_EXTERNAL_ROOT=0 path). +--clean-build Remove selected raw artifact outputs and Buildroot + output directories while preserving source checkouts and + download cache. +--full-rebuild Remove selected raw artifact outputs and external + source/build trees before building so they are cloned and + built again from pinned revisions. +--help Show this message. ``` -`test-tools.img` is the shared optional disk for test payloads that should -not live in the default `rootfs.cpio` or `ext4.img`. This keeps the default -guest image small while still allowing larger tools to be collected in one -place. +`test-tools.img` is the shared optional disk for test payloads selected by +recipe entries that should not live in the default `rootfs.cpio` or +`ext4.img`. This keeps the default guest image small while still allowing +larger tools to be collected in one place. -Build Buildroot and the test tools image with the DirectFB2 test payload. Add -`--x11` when the test tools image should use an X11-enabled rootfs: +The selected test-tools recipe is part of the CI `test-tools.img` recipe key, so +workflow caches and release artifacts distinguish the canonical `x11,directfb2` +image from `x11` or `directfb2` local variants. -``` -$ scripts/build-image.sh --x11 --directfb2-test -``` - -To add a new test tool, extend the `test-tools.img` build path in -`scripts/build-image.sh` so the tool is staged into `extra_packages`, then -update `target/local-env.sh` if the tool needs an additional binary or library -search path. +To add a new test tool, extend the test-tools build path in +`scripts/prebuilt/artifact-recipes.sh` so the tool is staged into +`extra_packages`, then update `target/local-env.sh` if the tool needs an +additional binary or library search path. The build script copies `target/local-env.sh` to `/root/local-env.sh` in the -test tools image. After booting the VM, source it once to pick up paths such -as `/usr/local/bin` and `/usr/local/lib`, instead of running overlaid tools -through full paths like `/usr/local/bin/df_*`. - -Force a clean build: - -``` -$ scripts/build-image.sh --all --clean-build -$ scripts/build-image.sh --linux --clean-build -$ scripts/build-image.sh --buildroot --clean-build -``` +test tools image. After booting the VM, source it once to pick up paths such as +`/usr/local/bin` and `/usr/local/lib`, instead of running overlaid tools through +full paths like `/usr/local/bin/df_*`. ## License diff --git a/configs/meson-riscv-cross-file b/configs/meson-riscv-cross-file new file mode 100644 index 00000000..d4f1c129 --- /dev/null +++ b/configs/meson-riscv-cross-file @@ -0,0 +1,21 @@ +# scripts/prebuilt/artifact-recipes.sh replaces @BUILDROOT_OUTPUT@ with the +# Buildroot output directory for the artifact profile being built. + +[binaries] + c = 'riscv32-buildroot-linux-gnu-gcc' + strip = 'riscv32-buildroot-linux-gnu-strip' + pkgconfig = 'pkg-config' + python = '/usr/bin/python3' + +[properties] + pkg_config_libdir = ['@GLOBAL_SOURCE_ROOT@' / '../@BUILDROOT_OUTPUT@/host/riscv32-buildroot-linux-gnu/sysroot/usr/local/lib/pkgconfig', + '@GLOBAL_SOURCE_ROOT@' / '../@BUILDROOT_OUTPUT@/host/riscv32-buildroot-linux-gnu/sysroot/usr/share/pkgconfig/', + '@GLOBAL_SOURCE_ROOT@' / '../@BUILDROOT_OUTPUT@/host/riscv32-buildroot-linux-gnu/sysroot/usr/lib/pkgconfig/' + ] + sys_root = '@GLOBAL_SOURCE_ROOT@' / '../@BUILDROOT_OUTPUT@/host/riscv32-buildroot-linux-gnu/sysroot' + +[host_machine] + system = 'linux' + cpu_family = 'riscv32' + cpu = 'riscv32-ima' + endian = 'little' diff --git a/configs/riscv-cross-file b/configs/riscv-cross-file deleted file mode 100644 index 668f91f5..00000000 --- a/configs/riscv-cross-file +++ /dev/null @@ -1,18 +0,0 @@ -[binaries] - c = 'riscv32-buildroot-linux-gnu-gcc' - strip = 'riscv32-buildroot-linux-gnu-strip' - pkgconfig = 'pkg-config' - python = '/usr/bin/python3' - -[properties] - pkg_config_libdir = ['@GLOBAL_SOURCE_ROOT@' / '../buildroot/output/host/riscv32-buildroot-linux-gnu/sysroot/usr/local/lib/pkgconfig', - '@GLOBAL_SOURCE_ROOT@' / '../buildroot/output/host/riscv32-buildroot-linux-gnu/sysroot/usr/share/pkgconfig/', - '@GLOBAL_SOURCE_ROOT@' / '../buildroot/output/host/riscv32-buildroot-linux-gnu/sysroot/usr/lib/pkgconfig/' - ] - sys_root = '@GLOBAL_SOURCE_ROOT@' / '../buildroot/output/host/riscv32-buildroot-linux-gnu/sysroot' - -[host_machine] - system = 'linux' - cpu_family = 'riscv32' - cpu = 'riscv32-ima' - endian = 'little' diff --git a/docs/build-system-and-ci.md b/docs/build-system-and-ci.md new file mode 100644 index 00000000..9d7f1dfb --- /dev/null +++ b/docs/build-system-and-ci.md @@ -0,0 +1,690 @@ +# semu Build System and CI/CD Pipeline + +## Big Picture + +This document explains the build system and CI/CD pipeline in semu. The overall flow can be read as four related contexts: local development, CI artifact preparation, test consumption, and publishing. + +The main difference between those contexts is who decides whether to use a release artifact or rebuild from source. The local path stays close to the original Make file-target model, where the developer explicitly decides when to rebuild from source. CI automatically compares the current checkout, the rolling release, and the workflow-local cache so that guest artifacts do not need to be rebuilt from scratch on every push. + +For local builds, Make treats `Image`, `rootfs.cpio`, and `test-tools.img` as regular file targets. When a Make target depends on one of those files, or when the user explicitly requests one of those filenames, Make downloads the corresponding `.bz2` from the rolling `prebuilt` release if the raw artifact is missing. + +By default, `make check` requires `Image` and `rootfs.cpio`, then derives the default `ext4.img` from `rootfs.cpio`. `test-tools.img` is obtained through `make test-tools.img` or through the source-build `test-tools` class. + +When a developer wants to rebuild from the current working tree, they explicitly run `make build-artifacts`, letting the build system decide which parts need to be rebuilt. Source builds use separate Buildroot output directories for the default rootfs and the test-tools rootfs, so files added for test-tools do not affect the default `rootfs.cpio` or `ext4.img`. + +```text +local developer + │ + └─ make / make check + │ + └─ mk/external.mk + │ + ├─ artifact exists -> use existing local file + ├─ artifact missing -> download and verify release .bz2 + └─ explicit source build -> make build-artifacts +``` + +In CI, `guest-artifact-build` first determines the guest artifact set required by the current checkout, then compares it with the rolling release. If the release already matches the current requirements, this job skips materialization, packaging, and upload, and later test jobs download artifacts directly from the release. + +If the release does not match the current requirements, CI restores the workflow-local raw cache, then checks whether the restored cache contains usable artifacts. When neither the release nor the workflow-local raw cache matches the current requirements, CI builds from source. The workflow-local raw cache is CI's main acceleration layer. + +```text +Actions + │ + └─ guest-artifact-build + │ + ├─ resolve current checkout against release manifest + │ + ├─ release current? + │ │ + │ ├─ yes -> skip materialize/package/upload in this job + │ └─ no -> restore workflow-local raw cache + │ + ├─ resolve again after cache restore + │ + ├─ materialize supplied plan + │ │ + │ ├─ use-action-cache restored raw artifacts + │ ├─ download matching release artifacts + │ └─ build changed artifacts from source + │ + └─ package and upload artifacts for later jobs +``` + +The test and publish stages reuse the artifact content decided earlier. Test jobs obtain artifacts through `.github/actions/provide-guest-artifacts`, then run `make`, `make check`, or `.ci/autorun.sh`. When the release is determined to need an update in the current workflow run, test jobs use the raw artifacts produced by that workflow run. + +On a master push, if artifacts need an update and all required tests pass, `publish-prebuilt` verifies the packaged artifacts and updates the rolling `prebuilt` release. + +```text +Test jobs + │ + ├─ provide-guest-artifacts + │ │ + │ ├─ release stale -> use this workflow run raw bundle + │ └─ release current -> download release artifacts + │ + └─ make / make check / .ci/autorun.sh + +publish-prebuilt + │ + ├─ master push + release_needs_update + all required tests passed + ├─ verify packaged bundle + └─ update rolling prebuilt release +``` + +## File Responsibilities + +The files below are ordered roughly by the main call flow. Within the same layer, helpers that are sourced are listed after their callers. Local source builds and CI builds share the recipes under `scripts/prebuilt/`. + +### Local Source Build Path + +The local source-build path starts from a Make target, then enters the user CLI and the shared recipes. + +- `Makefile`: provides local entry points such as `build-artifacts`, `check`, and `bench-login`, and includes `mk/external.mk`. +- `mk/external.mk`: defines local raw artifact download targets for `Image`, `rootfs.cpio`, and `test-tools.img`. +- `scripts/build-artifacts.sh`: user CLI called by `make build-artifacts`; it parses build classes, local flags, Buildroot cleanup policy, and the local `ext4.img` derived output. +- `scripts/prebuilt/artifact-classes.sh`: shared source-build artifact contract; it defines class names and raw output filenames. +- `scripts/prebuilt/artifact-recipes.sh`: shared source-build recipe library; it actually builds the Linux, Buildroot rootfs, and test-tools raw artifacts. +- `scripts/prebuilt/artifact-recipe.env`: external revisions and size constants loaded by `artifact-recipes.sh`. +- `scripts/prebuilt/test-tools-recipe.sh`: test-tools recipe selection logic shared by `artifact-recipes.sh` and the CI recipe key. +- `scripts/prebuilt/download-release-artifact.sh`: release transport helper shared by local Make downloads and the CI materializer. + +### CI Prebuilt Layer + +The CI prebuilt layer starts from the workflow, decides artifact sources, then materializes, stamps, and packages artifacts. + +- `.github/workflows/main.yml`: starts `guest-artifact-build` and connects cache, workflow artifacts, the publish gate, and job outputs. +- `.ci/prebuilt/artifact-inputs.sh`: sourced by the resolver, materializer, and package helpers; it connects the shared artifact contract to CI recipe-key helpers and plan-key helpers. +- `.ci/prebuilt/resolve-artifacts.sh`: produces a materialization plan from the current checkout, the release manifest, and the restored raw cache. +- `.ci/prebuilt/materialize-artifacts.sh`: executes the plan produced by the resolver; the action may be `use-action-cache`, `download-release`, or `build`. +- `.ci/prebuilt/build-plan-artifacts.sh`: internal CI build wrapper called by the materializer when source builds are needed; it sources `scripts/prebuilt/artifact-recipes.sh`. +- `.ci/prebuilt/stamp-artifacts.sh`: writes `.stamps/prebuilt-local/*.recipe-key` stamps after successful materialization. +- `.ci/prebuilt/package.sh`: compresses raw artifacts and writes the release manifest. +- `.ci/prebuilt/verify-package.sh`: checks packaged artifacts and manifest shape before publish. + +### Test And GitHub Helpers + +After the artifact decision is complete, test jobs obtain the environment and guest artifacts through composite actions. GitHub helpers provide common workflow-output and PR-suggestion operations. + +- `.github/actions/setup-semu/action.yml`: GitHub test dependency installation. +- `.github/actions/provide-guest-artifacts/action.yml`: composite action used by test jobs to obtain guest artifacts. +- `.ci/github/append-github-output.sh`: validated `$GITHUB_OUTPUT` writer. +- `.ci/github/suggest-format.sh`: reviewdog/GitHub PR formatting suggestions. + +## Local Build Path + +The local flow starts from Make file targets. When a developer runs `make`, the build produces the emulator binary and `minimal.dtb`. When they run `make check`, Make needs `Image`, `rootfs.cpio`, and, in the default external-rootfs mode, `ext4.img`. Make checks whether those files already exist. Existing files are reused; missing files are downloaded according to the rules in `mk/external.mk` from the rolling `prebuilt` release. + +The Makefile includes `mk/external.mk` before the artifact-consuming targets, making guest artifact names such as `KERNEL_DATA`, `INITRD_DATA`, and `TEST_TOOLS_DATA` regular file targets. `check` connects `KERNEL_DATA`, the boot-mode-dependent `INITRD_DEP`, and `DISKIMG_FILE` to the emulator launch arguments. The relationship can be simplified as follows: + +```make +... +include mk/external.mk + +... + +check: $(BIN) minimal.dtb $(KERNEL_DATA) $(INITRD_DEP) $(DISKIMG_FILE) $(SHARED_DIRECTORY) + @$(call notice, Ready to launch Linux kernel. Please be patient.) + $(Q)./$(BIN) $(strip $(KERNEL_OPT) $(DTB_OPT) $(HEADLESS_OPT) \ + $(INITRD_OPT) $(DISKIMG_OPT) $(VIRTIOFS_OPT) \ + $(SMP_OPT) $(NETDEV_OPT)) +``` + +The download rule in `mk/external.mk` is intentionally thin and only downloads missing files. After the download completes, the helper decompresses the archive into the raw artifact and removes the `.bz2`, so the workspace usually contains raw artifacts rather than compressed archives. + +This local download path verifies the release archive checksum against `prebuilt.sha1`, ensuring that the downloaded `.bz2` bytes match the release manifest before decompression. Local `make check` uses existing or downloaded raw artifacts. Therefore, when a developer changes a guest config or build recipe, they explicitly run `make build-artifacts` to test artifacts produced from the current source. + +```make +define download_prebuilt_artifact +$(1): + $$(Q)printf ' GET\t$$@\n' + $$(Q)PREBUILT_URL="$$(PREBUILT_URL)" scripts/prebuilt/download-release-artifact.sh "$$@" +endef + +$(foreach artifact,$(PREBUILT_ARTIFACTS),$(eval $(call download_prebuilt_artifact,$(artifact)))) +``` + +`ext4.img` is produced differently. When the default external rootfs is enabled, `ext4.img` is a local derived file generated from `rootfs.cpio`. A fresh `make check` usually ensures that `Image` and `rootfs.cpio` exist first. If `ext4.img` is missing, it then runs `scripts/rootfs_ext4.sh rootfs.cpio ext4.img`. The rootfs helper exposes tool hooks so CI or developers can provide a coherent fakeroot-compatible toolchain without adding platform policy to the helper itself. + +```make +ext4.img: $(INITRD_DATA) scripts/rootfs_ext4.sh + $(Q)MKFS_EXT4="$(MKFS_EXT4)" ROOTFS_CPIO="$(ROOTFS_CPIO)" \ + ROOTFS_CP="$(ROOTFS_CP)" ROOTFS_CHOWN="$(ROOTFS_CHOWN)" \ + ROOTFS_FAKEROOT_SHELL="$(ROOTFS_FAKEROOT_SHELL)" \ + scripts/rootfs_ext4.sh $(INITRD_DATA) $@ +``` + +To regenerate guest artifacts from the current working tree, developers use `make build-artifacts`. The Makefile is a thin wrapper here; CLI parsing lives in `scripts/build-artifacts.sh`. + +```make +ARTIFACTS ?= all +.PHONY: build-artifacts +build-artifacts: + scripts/build-artifacts.sh $(ARTIFACTS) +``` + +`build-artifacts.sh` first maps the user-supplied class to three internal boolean values. `image` builds the Linux kernel artifact, `rootfs` builds `rootfs.cpio` and the default `ext4.img`, `test-tools` builds the optional disk, and `all` enables all three classes. `--x11` and `--directfb2-test` are test-tools recipe entries. When the user provides a recipe flag explicitly, the script builds the selected entry. + +```bash +... +case "$1" in + image) BUILD_LINUX=1 ;; + rootfs) BUILD_ROOTFS=1 ;; + test-tools) BUILD_TEST_TOOLS=1 ;; + all) + BUILD_LINUX=1 + BUILD_ROOTFS=1 + BUILD_TEST_TOOLS=1 + ;; + --x11) add_test_tools_recipe_entry x11 ;; + --directfb2-test) add_test_tools_recipe_entry directfb2 ;; +esac +... +``` + +Before the actual build, `--clean-build` and `--full-rebuild` process old outputs. Both options remove the selected raw artifact outputs so that a failed source build does not leave stale `Image`, `rootfs.cpio`, `ext4.img`, or `test-tools.img` in the workspace. + +`--clean-build` removes Buildroot output directories while preserving the `buildroot/` checkout and download cache. `--full-rebuild` removes external source/build trees so the next step reclones the pinned revisions. + +```bash +... +if [[ $CLEAN_BUILD -eq 1 || $FULL_REBUILD -eq 1 ]]; then + remove_selected_artifact_outputs +fi + +if [[ $FULL_REBUILD -eq 1 ]]; then + ... + remove_existing_paths "Buildroot source tree for full rebuild" buildroot + remove_existing_paths "Linux source tree for full rebuild" linux +elif [[ $CLEAN_BUILD -eq 1 ]]; then + remove_existing_paths "default Buildroot output for clean build" \ + "$BUILDROOT_DEFAULT_OUTPUT" buildroot/output + remove_existing_paths "test-tools Buildroot output for clean build" \ + "$BUILDROOT_TEST_TOOLS_OUTPUT" +fi +... +``` + +The actual build logic is centralized in `scripts/prebuilt/artifact-recipes.sh`. This file is used by both the local CLI and the CI internal builder. It connects source trees, Buildroot outputs, and final raw artifacts. The default rootfs and test-tools rootfs use separate Buildroot outputs so that test-tools recipe changes remain isolated from the default rootfs. + +```bash +BUILDROOT_DEFAULT_OUTPUT=${BUILDROOT_DEFAULT_OUTPUT:-buildroot/output-default} +BUILDROOT_TEST_TOOLS_OUTPUT=${BUILDROOT_TEST_TOOLS_OUTPUT:-buildroot/output-test-tools} + +buildroot_make() { + local output_dir=$1 + shift + + ASSERT mkdir -p "$output_dir" + ASSERT make -C buildroot O="$PWD/$output_dir" "$@" +} +``` + +The `image`, `rootfs`, and `test-tools` recipe entrypoints all run under `with_buildroot_lock` because they share the same `buildroot/` checkout, configuration, host toolchain, and output directories. The lock uses an atomic symlink at `.semu-buildroot.lock` by default, or at `BUILDROOT_LOCK_PATH` when that variable is set. Creating the symlink both acquires the lock and publishes the owner token, so waiters never observe a successfully acquired lock before its owner is recorded. After `ln -s` succeeds, the recipe verifies that the configured lock path itself points at the current owner token, which also catches stale directory locks left by older versions. + +```bash +BUILDROOT_LOCK_PATH=${BUILDROOT_LOCK_PATH:-.semu-buildroot.lock} + +buildroot_acquire_lock() { + ... + BUILDROOT_LOCK_TOKEN=$(buildroot_new_lock_token) + while ! ln -s "$BUILDROOT_LOCK_TOKEN" "$BUILDROOT_LOCK_PATH" 2>/dev/null; do + ... + done + ... +} +``` + +The recipe does not automatically remove stale source-build locks. A stale lock should only be left behind when an artifact source build is interrupted after acquiring the lock, such as when the user kills the build or a CI runner is terminated. Portable shell/filesystem operations do not provide an atomic "remove this lock only if it still has this owner token" operation, so a read/compare/remove cleanup can race with another process that already acquired a fresh lock. When a stale or unrecognized lock is reported, first confirm that no semu artifact source build is still running, then remove the reported lock path manually. The default reported path is `.semu-buildroot.lock`, and `make distclean` also removes that default path. + +When building the default rootfs, the shared recipe uses the base Buildroot config to produce `buildroot/output-default/images/rootfs.cpio`, then publishes it as `rootfs.cpio` in the repository root. If the local CLI does not receive `--no-ext4`, it also derives `ext4.img` after `rootfs.cpio` is produced. + +```bash +do_rootfs_unlocked() { + build_buildroot_rootfs default "$BUILDROOT_DEFAULT_OUTPUT" + + echo "Publishing rootfs.cpio" + ASSERT cp -f "$BUILDROOT_DEFAULT_OUTPUT/images/rootfs.cpio" ./rootfs.cpio +} + +derive_local_ext4_from_rootfs() { + if [[ $NO_EXT4 -eq 1 ]]; then + echo "Skipping ext4.img build (--no-ext4)" + return + fi + + ASSERT ./scripts/rootfs_ext4.sh ./rootfs.cpio ./ext4.img +} +``` + +When building test-tools, the recipe uses `buildroot/output-test-tools`. If the recipe includes `x11`, the Buildroot config is layered with `configs/x11.config`, and the C++ runtime needed at runtime is added to the `test-tools.img` overlay. + +If the recipe includes `directfb2`, `do_extra_packages` first installs DirectFB2 and DirectFB examples into `directfb/`, copies them into `extra_packages/`, and finally overlays `extra_packages/` into `test-tools.img`. X11 and DirectFB2 payloads enter the test-tools output, while the default `rootfs.cpio` and `ext4.img` keep using the default rootfs recipe. + +```bash +do_test_tools_unlocked() { + local buildroot_mode + local buildroot_output=$BUILDROOT_TEST_TOOLS_OUTPUT + local test_tools_rootfs + + configure_test_tools_recipe_flags + + if [[ $BUILD_X11 -eq 1 ]]; then + buildroot_mode=x11 + else + buildroot_mode=default + fi + build_buildroot_rootfs "$buildroot_mode" "$buildroot_output" + test_tools_rootfs=./$buildroot_output/images/rootfs.cpio + + if [[ $BUILD_DIRECTFB_TEST -eq 1 ]]; then + do_extra_packages "$buildroot_output" + if [[ $BUILD_X11 -eq 1 ]]; then + stage_cxx_runtime "$buildroot_output" + fi + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" ./extra_packages + elif [[ $BUILD_X11 -eq 1 ]]; then + ASSERT rm -rf extra_packages + stage_cxx_runtime "$buildroot_output" + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" ./extra_packages + else + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" + fi +} +``` + +## CI Artifact Contract + +The contract between CI and the rolling release defines fixed raw artifact filenames and a sha1sum-format manifest. The release contains `Image.bz2`, `rootfs.cpio.bz2`, `test-tools.img.bz2`, and `prebuilt.sha1`. `prebuilt.sha1` records both the byte checksum of each `.bz2` archive and the recipe key for each artifact class. + +The class registry for this contract is defined by `scripts/prebuilt/artifact-classes.sh`, because class names and raw output filenames belong to the source-build contract. CI policy lives under `.ci/prebuilt/`. + +Each registry row contains a class name and the raw artifact filename for that class. + +`.ci/prebuilt/artifact-inputs.sh` sources this contract, then adds CI recipe-key helpers and plan-key helpers. + +Shell-safe names used by plan variables are derived from class names. For example, `test-tools` becomes `test_tools`. + +```bash +source_artifact_class_registry() { + printf '%s\t%s\n' image Image + printf '%s\t%s\n' rootfs rootfs.cpio + printf '%s\t%s\n' test-tools test-tools.img +} +``` + +A recipe key has two kinds of inputs. The first kind is real files, such as the Buildroot config, Linux config, BusyBox config, `target/init`, `scripts/rootfs_ext4.sh`, and test-tools-specific files such as `configs/x11.config` or `configs/meson-riscv-cross-file`. The second kind is recipe variables that do not directly appear as files but affect the output content, such as `BUILDROOT_REV`, `LINUX_REV`, `TEST_TOOLS_SIZE_MB`, and the normalized `PREBUILT_TEST_TOOLS_RECIPE` selection. + +`prebuilt_class_recipe_manifest` expands both groups into a sha1sum-style manifest. `prebuilt_class_recipe_key` then folds that full manifest into a single SHA-1. Every later CI decision about whether an artifact matches the current checkout uses this class recipe key. + +```bash +prebuilt_class_recipe_manifest() { + local class=$1 + local -a inputs=() + local input + + while IFS= read -r input; do + inputs+=("$input") + done < <(prebuilt_class_inputs "$class") + + if [ "${#inputs[@]}" -eq 0 ]; then + echo "[!] Prebuilt input class $class has no inputs" >&2 + return 1 + fi + + for input in "${inputs[@]}"; do + if [ ! -f "$input" ]; then + echo "[!] Missing prebuilt input $input" >&2 + return 1 + fi + done + + prebuilt_sha1sum "${inputs[@]}" + prebuilt_class_recipe_virtual_entries "$class" +} + +prebuilt_class_recipe_key() { + local manifest + + manifest=$(prebuilt_class_recipe_manifest "$1") || return 1 + printf '%s\n' "$manifest" | prebuilt_sha1sum | awk '{print $1}' +} +``` + +The package stage first confirms that raw artifacts and recipe inputs exist, then compresses each raw artifact into `.bz2`. When writing `prebuilt.sha1`, the first part contains checksums for each `.bz2` archive, and the second part contains virtual `*.recipe-key` entries generated from class recipe keys. For example, the recipe key for the `image` class is written as `Image.recipe-key`, the `rootfs` class as `rootfs.cpio.recipe-key`, and the `test-tools` class as `test-tools.img.recipe-key`. + +```bash +write_archive_checksum_entries_for_class() { + local class=$1 + local artifact + + while IFS= read -r artifact; do + prebuilt_sha1sum "${artifact}.bz2" + done < <(source_artifact_class_outputs "$class") +} + +write_recipe_key_entries_for_class() { + local class=$1 + local recipe_key=$2 + local entry + + while IFS= read -r entry; do + printf '%s %s\n' "$recipe_key" "$entry" + done < <(prebuilt_class_recipe_entries "$class") +} +``` + +This design separates the question of whether the release matches the current recipe from the question of whether downloaded archive bytes match the manifest. The resolver uses recipe keys to decide release freshness and requires the manifest to contain the corresponding `.bz2` checksum entries. + +When the materializer downloads a release artifact, it first verifies the `.bz2` checksum against `prebuilt.sha1` and confirms that the `*.recipe-key` in the manifest still matches the resolver plan. It then decompresses the archive and writes stamps. If the release asset itself is stale or corrupt, or if the manifest and asset are inconsistent, CI fails and leaves the rolling release state issue for the maintainer to inspect. + +The CI local cache contract is also the recipe key. After the materializer successfully downloads or builds a class, it calls `prebuilt_stamp_artifacts_for_class` and writes the same class recipe key to `.stamps/prebuilt-local/.recipe-key`. After a later workflow restores the raw artifact cache, the resolver uses those stamps to decide whether cached raw artifacts are trustworthy. + +```bash +prebuilt_local_stamp_dir=.stamps/prebuilt-local + +prebuilt_artifact_recipe_stamp() { + printf '%s/%s.recipe-key\n' "$prebuilt_local_stamp_dir" "$1" +} + +prebuilt_stamp_artifacts_for_class() { + local class=$1 + local recipe_key=$2 + local artifact + + if [ -z "$recipe_key" ]; then + echo "[!] Cannot stamp $class with an empty recipe key" >&2 + return 1 + fi + + mkdir -p "$prebuilt_local_stamp_dir" || return 1 + while IFS= read -r artifact; do + if [ ! -f "$artifact" ]; then + echo "[!] Cannot stamp missing artifact: $artifact" >&2 + return 1 + fi + printf '%s\n' "$recipe_key" > "$(prebuilt_artifact_recipe_stamp "$artifact")" + done < <(source_artifact_class_outputs "$class") +} +``` + +It is important to distinguish `.stamps/prebuilt-local` from local Make. `.stamps/prebuilt-local` belongs to the CI raw cache and records which recipe key produced a raw artifact restored from cache. Local `make check` uses Make file targets and local filesystem state. Developers explicitly decide to rebuild artifacts through `make build-artifacts`. + +## CI Decision Model + +CI decision-making starts in the `guest-artifact-build` job in `.github/workflows/main.yml`. After checking out the source tree, this job runs `.ci/prebuilt/resolve-artifacts.sh`, which compares the current checkout, the rolling release manifest, and any local stamps already present in the workspace into a plan. + +Cache restore and builds happen after this plan. The plan is a plain `KEY=VALUE` file. Later dependency installation, materialization, and publish-gate steps read it. + +The resolver first sources `.ci/prebuilt/artifact-inputs.sh` to load the shared artifact contract, CI recipe-key helpers, and plan-variable naming helpers. It then computes `current_*_recipe_key` for every registered class. This key describes what artifact the current checkout needs. + +```bash +while IFS= read -r class; do + prebuilt_plan_set_class_var "$class" current_recipe_key \ + "$(prebuilt_class_recipe_key "$class")" + resolve_local_class "$class" +done < <(source_artifact_classes) +``` + +In the same pass, the resolver checks whether the workflow-local raw cache has been restored into the workspace. Here, local means the raw artifact cache inside the GitHub Actions workspace. The resolver requires every artifact in a class to exist, every artifact to have a `.stamps/prebuilt-local/*.recipe-key` stamp, and all stamps in the same class to have the same value. If any artifact or stamp is missing, or stamps in the same class disagree, the cache for that class is marked invalid. + +```bash +class_local_recipe_key() { + local class=$1 + local artifact + local recipe_stamp + local recipe_key= + local next= + local have_key=false + + while IFS= read -r artifact; do + recipe_stamp=$(prebuilt_artifact_recipe_stamp "$artifact") + if [ ! -f "$artifact" ] || [ ! -f "$recipe_stamp" ]; then + return 1 + fi + IFS= read -r next < "$recipe_stamp" || return 1 + if [ -z "$next" ]; then + return 1 + fi + if [ "$have_key" = false ]; then + recipe_key=$next + have_key=true + elif [ "$recipe_key" != "$next" ]; then + return 1 + fi + done < <(source_artifact_class_outputs "$class") + + if [ "$have_key" = false ]; then + return 1 + fi + + printf '%s\n' "$recipe_key" +} +``` + +Next, the resolver downloads the rolling release `prebuilt.sha1` into a temporary directory. `manifest_class_recipe_key` first confirms that every artifact has a corresponding `.bz2` checksum entry, then reads virtual entries such as `Image.recipe-key`, `rootfs.cpio.recipe-key`, or `test-tools.img.recipe-key`. If a class is missing an archive checksum entry, is missing a recipe-key entry, or has multiple artifact entries in the same class pointing to different keys, that class release key is treated as empty, and later action selection falls back to another artifact source. + +```bash +manifest_entry_value() { + local entry=$1 + + awk -v f="$entry" '$2 == f {print $1; found = 1; exit} END {exit !found}' "$manifest" +} + +manifest_class_recipe_key() { + local class=$1 + local artifact + local next + local recipe_key= + local have_key=false + + while IFS= read -r artifact; do + if ! manifest_entry_value "${artifact}.bz2" >/dev/null; then + printf '\n' + return + fi + if ! next=$(manifest_entry_value "${artifact}.recipe-key"); then + printf '\n' + return + fi + if [ "$have_key" = false ]; then + recipe_key=$next + have_key=true + elif [ "$recipe_key" != "$next" ]; then + printf '\n' + return + fi + done < <(source_artifact_class_outputs "$class") + + printf '%s\n' "$recipe_key" +} +``` + +After the resolver knows the current recipe key, the workflow-local raw cache recipe key and status, and the rolling release recipe key, it decides the real action. Release and local keys are internal resolver decision state. The materializer plan outputs each class action and current recipe key. + +The priority order is workflow-local raw cache, rolling release, then source build. This order matters because the workflow-local raw cache is a raw artifact bundle from the same workflow or nearby workflows. When a stamp matches the current checkout, CI uses the restored cache. When the release matches, CI downloads release artifacts. When both sources miss, CI builds from source. + +```bash +decide_action() { + local current_recipe_key=$1 + local release_recipe_key=$2 + local local_recipe_key=$3 + local local_status=$4 + + if [ "$local_status" = valid ] && [ "$local_recipe_key" = "$current_recipe_key" ]; then + printf '%s\n' use-action-cache + return + fi + + if [ -n "$release_recipe_key" ] && [ "$release_recipe_key" = "$current_recipe_key" ]; then + printf '%s\n' download-release + else + printf '%s\n' build + fi +} +``` + +The final resolver plan keeps the fields required by later steps: each class action, the current recipe key, `requires_build`, `release_needs_update`, `platform_action_cache_tag`, and `ci_cache_schema_tag`. `platform_action_cache_tag` folds the current recipe keys of the three classes into one cache namespace for GitHub Actions cache naming. Artifact correctness is determined by each class action and recipe key. + +```bash +printf 'plan_version=1\n' +prebuilt_plan_print_class_values action +prebuilt_plan_print_class_values current_recipe_key +printf 'platform_action_cache_tag=%s\n' "$platform_action_cache_tag" +printf 'ci_cache_schema_tag=%s\n' "$ci_cache_schema_tag" +printf 'requires_build=%s\n' "$requires_build" +printf 'release_needs_update=%s\n' "$release_needs_update" +``` + +The first workflow resolve is the publish gate. If `release_needs_update=false`, the rolling release matches the current checkout for all classes, so `guest-artifact-build` skips cache restore, materialization, packaging, and upload. + +Later test jobs materialize raw artifacts directly from the release through `.github/actions/provide-guest-artifacts`. + +If `release_needs_update=true`, the workflow restores the workflow-local raw cache, then runs the resolver again after cache restore to produce the post-cache plan. + +```yaml +... +- name: resolve release update plan + id: resolve + shell: bash + env: + PREBUILT_BOOTSTRAP_ON_404: ${{ github.repository != 'sysprog21/semu' }} + run: | + .ci/prebuilt/resolve-artifacts.sh > "$RUNNER_TEMP/prebuilt-plan-main.env" + .ci/github/append-github-output.sh "$RUNNER_TEMP/prebuilt-plan-main.env" \ + release_needs_update \ + platform_action_cache_tag \ + ci_cache_schema_tag + +... + +- name: cache workflow-local raw guest artifacts + if: steps.resolve.outputs.release_needs_update == 'true' + uses: actions/cache@v5 + ... + +- name: materialization plan after cache restore + id: post_cache_resolve + if: steps.resolve.outputs.release_needs_update == 'true' + shell: bash + env: + PREBUILT_BOOTSTRAP_ON_404: ${{ github.repository != 'sysprog21/semu' }} + run: | + .ci/prebuilt/resolve-artifacts.sh > "$RUNNER_TEMP/prebuilt-plan-post-cache.env" + .ci/github/append-github-output.sh "$RUNNER_TEMP/prebuilt-plan-post-cache.env" \ + requires_build +... +``` + +The materializer executes the plan already decided by the resolver. The caller points it at the plan file with `PREBUILT_PLAN_FILE`. `parse_resolver_output` consumes `plan_version`, `*_action`, and `current_*_recipe_key`, and skips scalar fields used by the workflow, such as `requires_build`, `platform_action_cache_tag`, `ci_cache_schema_tag`, and `release_needs_update`. Any other unknown key fails fast so an old materializer cannot silently misunderstand a new plan schema. + +```bash +... +if [ -z "${PREBUILT_PLAN_FILE:-}" ]; then + echo "[!] PREBUILT_PLAN_FILE is required; create a CI resolver plan before materializing." >&2 + exit 1 +fi +if [ ! -f "$PREBUILT_PLAN_FILE" ]; then + echo "[!] PREBUILT_PLAN_FILE does not exist: $PREBUILT_PLAN_FILE" >&2 + exit 1 +fi +resolver_output=$(cat "$PREBUILT_PLAN_FILE") + +... + +parse_resolver_output <<< "$resolver_output" + +if [ "$plan_version" != 1 ]; then + echo "[!] Unsupported prebuilt plan version: ${plan_version:-missing}" >&2 + exit 1 +fi +... +``` + +Before doing any download or build, the materializer verifies that every class has a valid action. If a test job sets `PREBUILT_FORBID_BUILD=1` while the plan still contains a `build` action, the materializer fails immediately. This guard lets test jobs consume workflow artifacts or the rolling release that have already been selected. Publish-state issues encountered during testing are surfaced directly. + +```bash +if [ "$requires_build" = true ] && [ "$forbid_build" = true ]; then + echo "[!] PREBUILT_FORBID_BUILD forbids source builds, but the plan contains:" >&2 + while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = build ]; then + printf '[!] %s=build\n' "$(prebuilt_plan_class_var_name "$class" action)" >&2 + fi + done < <(source_artifact_classes) + exit 1 +fi +``` + +When source builds are actually needed, the materializer calls `.ci/prebuilt/build-plan-artifacts.sh`. This internal builder sources `scripts/prebuilt/artifact-recipes.sh` and runs `build_prebuilt_artifact_classes`. The commit point for stamps remains in the materializer because the materializer knows whether the full plan action completed successfully. + +After a successful build, the materializer stamps the corresponding class with the current recipe key. + +```bash +if [ "$requires_build" = true ]; then + build_plan_classes + while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = build ]; then + prebuilt_stamp_artifacts_for_class "$class" \ + "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" + fi + done < <(source_artifact_classes) +fi +``` + +For each `download-release` class, the materializer downloads the corresponding `.bz2` through `scripts/prebuilt/download-release-artifact.sh` and verifies the archive checksum and expected recipe key against `prebuilt.sha1`. + +After verification succeeds, the materializer decompresses the archive into the raw artifact and writes `.stamps/prebuilt-local`. For each `use-action-cache` class, cache restore has already placed the raw artifact and stamp in the workspace. + +The final result of the CI decision model is that every class leaves a raw artifact in the workspace. If the artifact was managed by the CI materializer, it also leaves a matching recipe-key stamp for a later cache restore to evaluate again. + +```bash +while IFS= read -r class; do + if [ "$(prebuilt_plan_get_class_var "$class" action)" = download-release ]; then + download_release_class "$class" \ + "$(prebuilt_plan_get_class_var "$class" current_recipe_key)" + fi +done < <(source_artifact_classes) +``` + +The responsibilities of each layer are therefore: + +- The resolver makes decisions and leaves raw artifacts unchanged. +- The materializer executes the plan. +- The package step compresses materialized raw artifacts and writes the archive checksum and recipe-key manifest. +- Test jobs obtain artifacts and run tests. +- The publish job updates the rolling `prebuilt` release after a master push and successful tests for the same artifact set. + +## GitHub Workflow + +`.github/workflows/main.yml` runs on `push` and `pull_request`. The main artifact job is `guest-artifact-build`. + +`guest-artifact-build` runs in this order: + +1. Check out the source tree. +2. Run `.ci/prebuilt/resolve-artifacts.sh` against the rolling release manifest. +3. Publish `release_needs_update`, `platform_action_cache_tag`, and `ci_cache_schema_tag` as step outputs. +4. If the rolling release already matches the current recipe keys, skip materialization, packaging, and upload in this job. +5. If the release is stale, restore the workflow-local raw artifact cache keyed by `platform_action_cache_tag` and `ci_cache_schema_tag`. +6. Run the resolver again after cache restore. +7. If the post-cache plan still contains a `build` action, install Buildroot/Linux build dependencies. +8. Run `.ci/prebuilt/materialize-artifacts.sh` with the post-cache plan. +9. Run `.ci/prebuilt/package.sh`. +10. Upload the raw artifacts for test jobs and the packaged artifacts for publish. + +The workflow-local raw artifact cache contains: + +- `Image` +- `rootfs.cpio` +- `test-tools.img` +- `.stamps/prebuilt-local/` + +Master pushes also restore this cache. The publish decision is separate: master can publish after all required tests pass on the same materialized artifacts. + +Here, `release_needs_update` represents artifact recipe-key drift and release-contract drift explicitly checked by the resolver, such as the requirement that `prebuilt.sha1` contain archive checksum entries. Changes to `.ci/prebuilt/package.sh`, `.ci/prebuilt/verify-package.sh`, release transport, or release layout need to make the resolver observe stale state from the current release manifest, or explicitly add schema to the gate, before they can automatically trigger a republish. + +Forks and mirrors may not have their own rolling `prebuilt` release yet. The GitHub workflow enables `PREBUILT_BOOTSTRAP_ON_404` for non-canonical repositories, so an HTTP 404 for `prebuilt.sha1` becomes a first-run source build. In the canonical `sysprog21/semu` repository, HTTP 404 is a failure and asks the maintainer to inspect external release state. + +This bootstrap exception applies to HTTP 404. DNS failures, connection timeouts, HTTP 5xx, and other non-404 errors are external release-state or network-state problems. The resolver fails directly. diff --git a/docs/networking.md b/docs/networking.md index e0966ac8..35ca0d5c 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -405,7 +405,7 @@ sudo make check Automated tests: ```shell # Requires sudo on all platforms -sudo .ci/test-netdev.sh +sudo .ci/device-smoke/test-netdev.sh ``` ## References diff --git a/main.c b/main.c index a0b53334..c09c2b57 100644 --- a/main.c +++ b/main.c @@ -825,7 +825,6 @@ static void handle_options(int argc, *dtb_file = "minimal.dtb"; } -#if SEMU_HAS(EXTERNAL_ROOT) static bool uses_default_minimal_dtb(const char *dtb_file) { const char *name; @@ -841,8 +840,53 @@ static bool uses_default_minimal_dtb(const char *dtb_file) return strcmp(name, "minimal.dtb") == 0; } + +static int validate_default_dtb_boot_options(const char *dtb_file, + const char *initrd_file, + const char *disk_file) +{ + if (!uses_default_minimal_dtb(dtb_file)) { + fprintf(stderr, + "note: skipping default minimal.dtb boot option checks " + "for custom DTB '%s'.\n", + dtb_file ? dtb_file : "(null)"); + return 0; + } + +#if SEMU_HAS(EXTERNAL_ROOT) + if (initrd_file) { + fprintf(stderr, + "-i requires a DTB with linux,initrd-start/end. " + "Rebuild with ENABLE_EXTERNAL_ROOT=0 or pass -b " + ".\n"); + return 2; + } + + if (!disk_file) { + fprintf(stderr, + "warning: EXTERNAL_ROOT build expects -d ; " + "without it the kernel will hang in rootwait.\n"); + } +#else + if (!initrd_file) { + fprintf(stderr, + "This semu build uses legacy initramfs boot " + "(ENABLE_EXTERNAL_ROOT=0).\n" + "Run with -i rootfs.cpio, use make check, or rebuild " + "with ENABLE_EXTERNAL_ROOT=1 for /dev/vda root boot.\n"); + if (disk_file) { + fprintf(stderr, + "-d only attaches a virtio-blk disk; it does not " + "select the root filesystem without root=/dev/vda " + "in DTB bootargs.\n"); + } + return 2; + } #endif + return 0; +} + #define INIT_HART(hart, emu, id) \ do { \ hart->priv = emu; \ @@ -880,21 +924,10 @@ static int semu_init(emu_state_t *emu, int argc, char **argv) (void) headless; #endif -#if SEMU_HAS(EXTERNAL_ROOT) - if (initrd_file && uses_default_minimal_dtb(dtb_file)) { - fprintf(stderr, - "-i requires a DTB with linux,initrd-start/end. " - "Rebuild with ENABLE_EXTERNAL_ROOT=0 or pass -b " - ".\n"); - return 2; - } - - if (!disk_file && uses_default_minimal_dtb(dtb_file)) { - fprintf(stderr, - "warning: EXTERNAL_ROOT build expects -d ; " - "without it the kernel will hang in rootwait.\n"); - } -#endif + int boot_option_status = + validate_default_dtb_boot_options(dtb_file, initrd_file, disk_file); + if (boot_option_status) + return boot_option_status; /* Initialize the emulator */ memset(emu, 0, sizeof(*emu)); diff --git a/mk/common.mk b/mk/common.mk index 6c597137..cf17a128 100644 --- a/mk/common.mk +++ b/mk/common.mk @@ -32,15 +32,3 @@ PASS_COLOR = \e[32;01m NO_COLOR = \e[0m notice = $(PRINTF) "$(PASS_COLOR)$(strip $1)$(NO_COLOR)\n" - -# File utilities -SHA1SUM = sha1sum -SHA1SUM := $(shell which $(SHA1SUM)) -ifndef SHA1SUM - SHA1SUM = shasum - SHA1SUM := $(shell which $(SHA1SUM)) - ifndef SHA1SUM - $(warning No shasum found. Disable checksums) - SHA1SUM := echo - endif -endif diff --git a/mk/external.mk b/mk/external.mk index 011eea23..4d98b8d7 100644 --- a/mk/external.mk +++ b/mk/external.mk @@ -1,122 +1,26 @@ -# For each external target, the following must be defined in advance: -# _DATA_URL : the hyperlink which points to archive. -# _DATA : the file to be read by specific executable. -# -# Artifacts are published as assets on the fixed-tag prebuilt GitHub -# prerelease by .github/workflows/prebuilt.yml. The expected SHA-1 of -# each archive is read from the prebuilt.sha1 manifest published -# alongside the archives, so checksum updates require no edit here. - -COMMON_URL = https://github.com/sysprog21/semu/releases/download/prebuilt - -PREBUILT_MANIFEST = prebuilt.sha1 -PREBUILT_MANIFEST_URL = $(COMMON_URL)/$(PREBUILT_MANIFEST) - -# kernel -KERNEL_DATA_URL = $(COMMON_URL)/Image.bz2 -KERNEL_DATA = Image - -# initrd -INITRD_DATA_URL = $(COMMON_URL)/rootfs.cpio.bz2 -INITRD_DATA = rootfs.cpio - -$(PREBUILT_MANIFEST): FORCE - $(VECHO) " GET\t$@\n" - $(Q)if curl --fail --retry 3 --retry-delay 1 --progress-bar \ - -L -o "$@.part" "$(PREBUILT_MANIFEST_URL)"; then \ - if [ -f "$@" ] && cmp -s "$@" "$@.part"; then \ - rm -f "$@.part"; \ - else \ - mv "$@.part" "$@"; \ - fi; \ - else \ - rm -f "$@.part"; \ - if [ -f "$@" ]; then \ - $(PRINTF) " KEEP\t$@ (offline; using cached manifest)\n"; \ - else \ - echo "manifest fetch failed and no cached manifest; cannot proceed" >&2; \ - exit 1; \ - fi; \ - fi - -# optional test tools disk -TEST_TOOLS_DATA_URL = $(COMMON_URL)/test-tools.img.bz2 -TEST_TOOLS_DATA = test-tools.img - -define download -# Download to a .part file so an interrupted curl never lands a -# corrupt or incomplete .bz2 that a later run mistakes for valid input. -# Curl resume (-C -) is intentionally NOT used: a fully-downloaded .part -# left over from a previous run, e.g. interrupted before sha1 verify, -# would make curl request a byte range past EOF, the server replies -# HTTP 416, and curl exits non-zero, a permanent self-inflicted -# deadlock. These files are small enough that a fresh GET is cheap. -# -# Look up the expected SHA-1 by archive basename in the release -# manifest, then verify the .part against it. Decompress to a .tmp -# file and rename only on success, so an interrupted bunzip2 cannot -# leave a half-decompressed artifact that make would treat as a valid -# up-to-date target on the next invocation. -$($(T)_DATA): $(PREBUILT_MANIFEST) | prebuilt-check - $(VECHO) " GET\t$$@\n" - $(Q)curl --fail --retry 3 --retry-delay 1 --progress-bar \ - -L -o "$$@.bz2.part" "$(strip $($(T)_DATA_URL))" \ - || { rm -f "$$@.bz2.part"; exit 1; } - $(Q)expected=$$$$(awk -v f="$(notdir $($(T)_DATA_URL))" '$$$$2==f{print $$$$1}' $(PREBUILT_MANIFEST)); \ - [ -n "$$$$expected" ] || { echo "verify: no $(notdir $($(T)_DATA_URL)) entry in $(PREBUILT_MANIFEST)" >&2; rm -f "$$@.bz2.part"; exit 1; }; \ - echo "$$$$expected $$@.bz2.part" | $(SHA1SUM) -c - \ - || { rm -f "$$@.bz2.part"; exit 1; } - $(Q)mv "$$@.bz2.part" "$$@.bz2" - $(Q)bunzip2 -c "$$@.bz2" > "$$@.tmp" \ - || { rm -f "$$@.tmp"; exit 1; } - $(Q)mv "$$@.tmp" "$$@" - $(Q)rm -f "$$@.bz2" +# Keep local file targets aligned with the shared source-build artifact +# contract instead of spelling raw artifact filenames here. Query by class name +# so registry order changes cannot silently swap Make's semantic variables. +prebuilt_artifact_output = $(strip $(shell scripts/prebuilt/artifact-classes.sh output $(1))) +KERNEL_DATA := $(call prebuilt_artifact_output,image) +INITRD_DATA := $(call prebuilt_artifact_output,rootfs) +TEST_TOOLS_DATA := $(call prebuilt_artifact_output,test-tools) + +$(if $(KERNEL_DATA),,$(error Failed to resolve prebuilt artifact output for image)) +$(if $(INITRD_DATA),,$(error Failed to resolve prebuilt artifact output for rootfs)) +$(if $(TEST_TOOLS_DATA),,$(error Failed to resolve prebuilt artifact output for test-tools)) + +PREBUILT_ARTIFACTS := $(KERNEL_DATA) $(INITRD_DATA) $(TEST_TOOLS_DATA) + +PREBUILT_URL ?= https://github.com/sysprog21/semu/releases/download/prebuilt + +# Local Make defaults to consuming the rolling release for missing guest +# artifacts. Source builds stay explicit through `make build-artifacts`; CI owns +# recipe-key/cache decisions separately under .ci/prebuilt. +define download_prebuilt_artifact +$(1): + $$(Q)printf ' GET\t$$@\n' + $$(Q)PREBUILT_URL="$$(PREBUILT_URL)" scripts/prebuilt/download-release-artifact.sh "$$@" endef -EXTERNAL_DATA = KERNEL INITRD TEST_TOOLS -$(foreach T,$(EXTERNAL_DATA),$(eval $(download))) - -# --- Stale-prebuilt detection ------------------------------------------- -# -# The prebuilt Image, rootfs.cpio, and test-tools.img above are baked from a -# fixed set of input files (kernel/buildroot/busybox configs, the build script, -# and the init stub). When any of those change locally the prebuilt may no -# longer reflect the user's intent, so we compute the SHA1 of those -# inputs and compare against the publisher's recorded inputs hash -- -# the line of prebuilt.sha1 written by .ci/publish-prebuilt.sh under -# the virtual name 'inputs'. -# -# Mismatch -> warn but do not auto-rebuild: a buildroot run takes the -# better part of an hour, so we let the user opt in via make build-image. -# Keep this list in sync with the INPUTS array in .ci/publish-prebuilt.sh -# and the paths filter in .github/workflows/prebuilt.yml. -PREBUILT_INPUTS := \ - configs/linux.config \ - configs/busybox.config \ - configs/buildroot.config \ - configs/x11.config \ - configs/riscv-cross-file \ - scripts/build-image.sh \ - scripts/rootfs_ext4.sh \ - target/init \ - target/local-env.sh - -# Read the publisher's inputs hash from the downloaded manifest at -# recipe time, after the manifest refresh above has had a chance to run. -.PHONY: prebuilt-check -prebuilt-check: $(PREBUILT_MANIFEST) - $(Q)manifest_sha1=$$(awk '$$2 == "inputs" {print $$1}' $(PREBUILT_MANIFEST)); \ - if [ -n "$$manifest_sha1" ]; then \ - expected=$(words $(PREBUILT_INPUTS)); \ - found=0; \ - for f in $(PREBUILT_INPUTS); do [ -f "$$f" ] && found=$$((found + 1)); done; \ - if [ "$$found" -eq "$$expected" ]; then \ - live_sha1=$$(cat $(PREBUILT_INPUTS) | $(SHA1SUM) | awk '{print $$1}'); \ - if [ "$$live_sha1" != "$$manifest_sha1" ]; then \ - echo "warning: Local prebuilt guest inputs ($$live_sha1) differ from" >&2; \ - echo "warning: the prebuilt's recorded inputs ($$manifest_sha1)." >&2; \ - echo "warning: The downloaded guest artifacts do not reflect your local configs." >&2; \ - echo "warning: Run \`make build-image\` to rebuild from source." >&2; \ - fi; \ - fi; \ - fi +$(foreach artifact,$(PREBUILT_ARTIFACTS),$(eval $(call download_prebuilt_artifact,$(artifact)))) diff --git a/scripts/build-artifacts.sh b/scripts/build-artifacts.sh new file mode 100755 index 00000000..4baf7587 --- /dev/null +++ b/scripts/build-artifacts.sh @@ -0,0 +1,254 @@ +#!/usr/bin/env bash + +# build-artifacts.sh is the user-facing source-build entrypoint. Normal Make +# flows such as `make check` do not call this script for missing guest artifacts; +# they use mk/external.mk to download the rolling prebuilt release instead. +# CI cache/stamp decisions live under .ci/prebuilt and are intentionally outside +# this local CLI. +set -euo pipefail + +SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH='' cd -- "$SCRIPT_DIR/.." && pwd) + +cd "$REPO_ROOT" +. "$REPO_ROOT/scripts/prebuilt/artifact-recipes.sh" + +TEST_TOOLS_RECIPE_EXPLICIT=0 +TEST_TOOLS_RECIPE= + +add_test_tools_recipe_entry() { + local entry=$1 + + if [[ $TEST_TOOLS_RECIPE_EXPLICIT -eq 0 ]]; then + TEST_TOOLS_RECIPE= + TEST_TOOLS_RECIPE_EXPLICIT=1 + fi + + case ",$TEST_TOOLS_RECIPE," in + *,"$entry",*) + ;; + ,,) + TEST_TOOLS_RECIPE=$entry + ;; + *) + TEST_TOOLS_RECIPE=$TEST_TOOLS_RECIPE,$entry + ;; + esac +} + +show_help() { + local exit_code=${1:-0} + local prog + + prog=$(basename "$0") + cat << HELP +Usage: $prog ... [--x11] [--directfb2-test] [--no-ext4] [--clean-build] [--full-rebuild] [--help] + +Targets: + image Build the Linux Image. This prepares the Buildroot + toolchain if needed but does not publish rootfs.cpio or + test-tools.img as final outputs. + rootfs Build Buildroot rootfs.cpio and, unless --no-ext4 is + given, derive ext4.img from it. + test-tools Build test-tools.img. By default this uses the canonical + X11 + DirectFB2 test payload recipe. + all Build image, rootfs, and canonical test-tools. + +Options: + --x11 Select the X11/C++ runtime recipe entry for test-tools. When + any test-tools recipe flag is given, only selected + entries are included. + --directfb2-test Select the DirectFB2 test payload recipe entry for test-tools. + --no-ext4 With rootfs, skip ext4.img generation and produce only + rootfs.cpio (matches the legacy ENABLE_EXTERNAL_ROOT=0 path). + --clean-build Remove selected raw artifact outputs and Buildroot + output directories while preserving source checkouts and + download cache. + --full-rebuild Remove selected raw artifact outputs and external + source/build trees before building so they are cloned and + built again from pinned revisions. + --help Show this message. +HELP + exit "$exit_code" +} + +BUILD_LINUX=0 +BUILD_ROOTFS=0 +BUILD_TEST_TOOLS=0 +NO_EXT4=0 +CLEAN_BUILD=0 +FULL_REBUILD=0 +TARGET_SPECIFIED=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + image) + BUILD_LINUX=1 + TARGET_SPECIFIED=1 + ;; + rootfs) + BUILD_ROOTFS=1 + TARGET_SPECIFIED=1 + ;; + test-tools) + BUILD_TEST_TOOLS=1 + TARGET_SPECIFIED=1 + ;; + all) + BUILD_LINUX=1 + BUILD_ROOTFS=1 + BUILD_TEST_TOOLS=1 + TARGET_SPECIFIED=1 + ;; + --x11) + add_test_tools_recipe_entry x11 + ;; + --directfb2-test) + add_test_tools_recipe_entry directfb2 + ;; + --no-ext4) + NO_EXT4=1 + ;; + --clean-build) + CLEAN_BUILD=1 + ;; + --full-rebuild) + FULL_REBUILD=1 + ;; + --help|-h) + show_help 0 + ;; + *) + echo "Unknown option or target: $1" + show_help 1 + ;; + esac + shift +done + +if [[ $TARGET_SPECIFIED -eq 0 ]]; then + echo "Error: No build target specified. Use image, rootfs, test-tools, or all." + show_help 1 +fi + +if [[ $TEST_TOOLS_RECIPE_EXPLICIT -eq 1 && $BUILD_TEST_TOOLS -eq 0 ]]; then + echo "Error: test-tools recipe options require the test-tools or all target." + show_help 1 +fi + +if [[ $NO_EXT4 -eq 1 && $BUILD_ROOTFS -eq 0 ]]; then + echo "Error: --no-ext4 requires the rootfs or all target." + show_help 1 +fi + +if [[ $BUILD_TEST_TOOLS -eq 1 ]]; then + if [[ $TEST_TOOLS_RECIPE_EXPLICIT -eq 1 ]]; then + export PREBUILT_TEST_TOOLS_RECIPE=$TEST_TOOLS_RECIPE + fi + prebuilt_test_tools_recipe_key >/dev/null +fi + +remove_existing_paths() { + local label=$1 + shift + local paths=() + local path + + for path in "$@"; do + if [[ -e $path ]]; then + paths+=("$path") + fi + done + + if [[ ${#paths[@]} -eq 0 ]]; then + return + fi + + echo "Removing $label:" + printf ' %s\n' "${paths[@]}" + rm -rf -- "${paths[@]}" +} + +append_class_outputs() { + local class=$1 + local output + + while IFS= read -r output; do + outputs+=("$output") + done < <(source_artifact_class_outputs "$class") +} + +remove_selected_artifact_outputs() { + local outputs=() + + if [[ $BUILD_LINUX -eq 1 ]]; then + append_class_outputs image + fi + + if [[ $BUILD_ROOTFS -eq 1 ]]; then + append_class_outputs rootfs + if [[ $NO_EXT4 -eq 0 ]]; then + outputs+=(ext4.img) + fi + fi + + if [[ $BUILD_TEST_TOOLS -eq 1 ]]; then + append_class_outputs test-tools + fi + + remove_existing_paths "selected raw artifact outputs" "${outputs[@]}" +} + +derive_local_ext4_from_rootfs() { + if [[ $NO_EXT4 -eq 1 ]]; then + echo "Skipping ext4.img build (--no-ext4)" + return + fi + + ASSERT ./scripts/rootfs_ext4.sh ./rootfs.cpio ./ext4.img +} + +if [[ $CLEAN_BUILD -eq 1 || $FULL_REBUILD -eq 1 ]]; then + remove_selected_artifact_outputs +fi + +if [[ $FULL_REBUILD -eq 1 ]]; then + if [[ $BUILD_LINUX -eq 1 || $BUILD_ROOTFS -eq 1 || $BUILD_TEST_TOOLS -eq 1 ]]; then + remove_existing_paths "Buildroot source tree for full rebuild" buildroot + fi + + if [[ $BUILD_LINUX -eq 1 ]]; then + remove_existing_paths "Linux source tree for full rebuild" linux + fi + + if [[ $BUILD_TEST_TOOLS -eq 1 ]]; then + remove_existing_paths "test-tools source/build trees for full rebuild" \ + DirectFB2 DirectFB-examples directfb extra_packages + fi +elif [[ $CLEAN_BUILD -eq 1 ]]; then + if [[ $BUILD_LINUX -eq 1 || $BUILD_ROOTFS -eq 1 ]]; then + remove_existing_paths "default Buildroot output for clean build" \ + "$BUILDROOT_DEFAULT_OUTPUT" buildroot/output + fi + + if [[ $BUILD_TEST_TOOLS -eq 1 ]]; then + remove_existing_paths "test-tools Buildroot output for clean build" \ + "$BUILDROOT_TEST_TOOLS_OUTPUT" + fi +fi + +if [[ $BUILD_LINUX -eq 1 ]]; then + build_source_artifact_class image + OK +fi + +if [[ $BUILD_ROOTFS -eq 1 ]]; then + build_source_artifact_class rootfs + derive_local_ext4_from_rootfs + OK +fi + +if [[ $BUILD_TEST_TOOLS -eq 1 ]]; then + build_source_artifact_class test-tools + OK +fi diff --git a/scripts/build-image.sh b/scripts/build-image.sh deleted file mode 100755 index f3543358..00000000 --- a/scripts/build-image.sh +++ /dev/null @@ -1,351 +0,0 @@ -#!/usr/bin/env bash - -function ASSERT -{ - $* - RES=$? - if [ $RES -ne 0 ]; then - echo 'Assert failed: "' $* '"' - exit $RES - fi -} - -PASS_COLOR='\e[32;01m' -NO_COLOR='\e[0m' -function OK -{ - printf " [ ${PASS_COLOR} OK ${NO_COLOR} ]\n" -} - -PARALLEL="-j$(nproc)" - -DIRECTFB2_REPO="https://github.com/directfb2/DirectFB2" -DIRECTFB2_REV="7d4682d0cc092ed2f28c903175d1a0c104e9e9a8" -DIRECTFB_EXAMPLES_REPO="https://github.com/directfb2/DirectFB-examples" -DIRECTFB_EXAMPLES_REV="eecf1019b29933a45578e62aea5f08a884d30fbc" -TEST_TOOLS_SIZE_MB=192 - -function safe_copy { - local src="$1" - local dst="$2" - - if [ ! -f "$dst" ]; then - echo "Copying $src -> $dst" - cp -f "$src" "$dst" - else - echo "$dst already exists, skipping copy" - fi -} - -function checkout_repo_rev -{ - local dir="$1" - local repo="$2" - local rev="$3" - - if [ ! -d "$dir/.git" ]; then - echo "Cloning $dir..." - ASSERT git clone "$repo" "$dir" - else - echo "$dir already exists, reusing clone..." - fi - - pushd "$dir" - if ! git cat-file -e "$rev^{commit}" 2>/dev/null; then - ASSERT git fetch origin - fi - ASSERT git checkout --detach "$rev" - popd -} - -function meson_setup_or_reconfigure -{ - local build_dir="$1" - shift - - if [ -f "$build_dir/build.ninja" ]; then - if ! meson setup --reconfigure "$@" "$build_dir"; then - echo "Recreating stale Meson build directory: $build_dir" - rm -rf "$build_dir" - ASSERT meson setup "$@" "$build_dir" - fi - else - ASSERT meson setup "$@" "$build_dir" - fi -} - -function configure_buildroot -{ - local mode="${1:-default}" - local buildroot_config="configs/buildroot.config" - local x11_config="configs/x11.config" - local merge_tool="buildroot/support/kconfig/merge_config.sh" - - if [[ "$mode" == "x11" ]]; then - echo "Preparing Buildroot config with X11 fragment..." - ASSERT "$merge_tool" -m -r -O buildroot "$buildroot_config" "$x11_config" - else - echo "Preparing default Buildroot config..." - cp -f "$buildroot_config" buildroot/.config - fi -} - -function build_buildroot_rootfs -{ - local mode="${1:-default}" - - configure_buildroot "$mode" - safe_copy configs/busybox.config buildroot/busybox.config - cp -f target/init buildroot/fs/cpio/init - - # Otherwise, the error below raises: - # You seem to have the current working directory in your - # LD_LIBRARY_PATH environment variable. This doesn't work. - unset LD_LIBRARY_PATH - pushd buildroot - ASSERT make olddefconfig - if [[ "$mode" == "x11" && \ - ! -x output/host/bin/riscv32-buildroot-linux-gnu-g++ ]]; then - echo "Rebuilding Buildroot final GCC with C++ support..." - ASSERT make host-gcc-final-dirclean - fi - ASSERT make $PARALLEL - popd -} - -function do_buildroot -{ - if [ ! -d buildroot ]; then - echo "Cloning Buildroot..." - ASSERT git clone https://github.com/buildroot/buildroot -b 2025.02.x --depth=1 - else - echo "buildroot/ already exists, skipping clone" - fi - - build_buildroot_rootfs default - - # Always publish the cpio. It is the canonical buildroot output and - # serves both as the source for the ext4 image and as the legacy - # initramfs payload (when ENABLE_EXTERNAL_ROOT=0). - echo "Publishing rootfs.cpio" - cp -f buildroot/output/images/rootfs.cpio ./rootfs.cpio - - # Build ext4.img unless --no-ext4 was passed. The make default - # (ENABLE_EXTERNAL_ROOT=1) boots from /dev/vda and needs this image. - # --no-ext4 is the escape hatch for users who only want the legacy - # initramfs path or do not have fakeroot/mkfs.ext4 installed. - if [[ $NO_EXT4 -eq 1 ]]; then - echo "Skipping ext4.img build (--no-ext4)" - else - ASSERT ./scripts/rootfs_ext4.sh ./rootfs.cpio ./ext4.img - - local test_tools_rootfs=./rootfs.cpio - if [[ $BUILD_X11 -eq 1 ]]; then - build_buildroot_rootfs x11 - test_tools_rootfs=./buildroot/output/images/rootfs.cpio - fi - - if [[ $BUILD_DIRECTFB_TEST -eq 1 ]]; then - do_extra_packages - if [[ $BUILD_X11 -eq 1 ]]; then - stage_cxx_runtime - fi - ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ - "$TEST_TOOLS_SIZE_MB" ./extra_packages - elif [[ $BUILD_X11 -eq 1 ]]; then - rm -rf extra_packages - mkdir -p extra_packages - stage_cxx_runtime - ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ - "$TEST_TOOLS_SIZE_MB" ./extra_packages - fi - fi -} - -function do_linux -{ - if [ ! -d linux ]; then - echo "Cloning Linux kernel..." - ASSERT git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git -b linux-6.12.y --depth=1 - else - echo "linux/ already exists, skipping clone" - fi - - safe_copy configs/linux.config linux/.config - - export PATH="$PWD/buildroot/output/host/bin:$PATH" - export CROSS_COMPILE=riscv32-buildroot-linux-gnu- - export ARCH=riscv - pushd linux - ASSERT make olddefconfig - ASSERT make $PARALLEL - cp -f arch/riscv/boot/Image ../Image - popd -} - -function do_directfb -{ - export PATH="$PWD/buildroot/output/host/bin:$PATH" - export BUILDROOT_OUT=$PWD/buildroot/output/ - export DIRECTFB_STAGE=$PWD/directfb - mkdir -p directfb - - # Build DirectFB2 - checkout_repo_rev DirectFB2 "$DIRECTFB2_REPO" "$DIRECTFB2_REV" - pushd DirectFB2 - cp ../configs/riscv-cross-file . - meson_setup_or_reconfigure build/riscv -Ddrmkms=true --cross-file \ - riscv-cross-file - ASSERT meson compile -C build/riscv - ASSERT env DESTDIR=$BUILDROOT_OUT/host/riscv32-buildroot-linux-gnu/sysroot meson install -C build/riscv - ASSERT env DESTDIR=$DIRECTFB_STAGE meson install -C build/riscv - popd - - # Build DirectFB2 examples - checkout_repo_rev DirectFB-examples "$DIRECTFB_EXAMPLES_REPO" \ - "$DIRECTFB_EXAMPLES_REV" - pushd DirectFB-examples/ - cp ../configs/riscv-cross-file . - meson_setup_or_reconfigure build/riscv --cross-file riscv-cross-file - ASSERT meson compile -C build/riscv - ASSERT env DESTDIR=$DIRECTFB_STAGE meson install -C build/riscv - popd -} - -function do_extra_packages -{ - export PATH="$PWD/buildroot/output/host/bin:$PATH" - export CROSS_COMPILE=riscv32-buildroot-linux-gnu- - - rm -rf directfb extra_packages - mkdir -p directfb - mkdir -p extra_packages - mkdir -p extra_packages/root - - do_directfb && OK - - if ! find directfb -mindepth 1 -print -quit | grep -q .; then - echo "Error: DirectFB staging tree is empty." - exit 1 - fi - - ASSERT cp -r directfb/. extra_packages/ - ASSERT cp target/local-env.sh extra_packages/root/ -} - -function stage_cxx_runtime -{ - local toolchain_lib="buildroot/output/host/riscv32-buildroot-linux-gnu/lib" - local libstdcpp="$toolchain_lib/libstdc++.so.6" - local libstdcpp_real - - if [ ! -e "$libstdcpp" ]; then - echo "Error: libstdc++.so.6 not found in $toolchain_lib" - exit 1 - fi - - libstdcpp_real="$(readlink "$libstdcpp" || basename "$libstdcpp")" - if [[ "$libstdcpp_real" != /* ]]; then - libstdcpp_real="$toolchain_lib/$libstdcpp_real" - fi - mkdir -p extra_packages/lib - ASSERT cp -a "$toolchain_lib/libstdc++.so" "$libstdcpp" \ - "$libstdcpp_real" extra_packages/lib/ -} - -function show_help { - cat << EOF -Usage: $0 [--buildroot] [--x11] [--linux] [--directfb2-test] [--all] [--no-ext4] [--clean-build] [--help] - -Options: - --buildroot Build Buildroot userland (produces rootfs.cpio and, - unless --no-ext4 is given, ext4.img for vda boot) - --x11 Build test-tools.img from an X11-enabled rootfs - --directfb2-test Overlay the DirectFB2 test payload into test-tools.img - --linux Build the Linux kernel - --all Build both Buildroot and Linux - --no-ext4 Skip ext4.img generation; produce only rootfs.cpio - (matches the legacy ENABLE_EXTERNAL_ROOT=0 path) - --clean-build Remove buildroot/ and/or linux/ before building; - with --directfb2-test, also remove DirectFB2 sources - --help Show this message -EOF - exit 1 -} - -BUILD_BUILDROOT=0 -BUILD_X11=0 -BUILD_DIRECTFB_TEST=0 -BUILD_LINUX=0 -NO_EXT4=0 -CLEAN_BUILD=0 - -while [[ $# -gt 0 ]]; do - case "$1" in - --buildroot) - BUILD_BUILDROOT=1 - ;; - --x11) - BUILD_BUILDROOT=1 - BUILD_X11=1 - ;; - --directfb2-test) - BUILD_BUILDROOT=1 - BUILD_DIRECTFB_TEST=1 - ;; - --linux) - BUILD_LINUX=1 - ;; - --all) - BUILD_BUILDROOT=1 - BUILD_LINUX=1 - ;; - --no-ext4) - NO_EXT4=1 - ;; - --clean-build) - CLEAN_BUILD=1 - ;; - --help|-h) - show_help - ;; - *) - echo "Unknown option: $1" - show_help - ;; - esac - shift -done - -if [[ $BUILD_BUILDROOT -eq 0 && $BUILD_LINUX -eq 0 ]]; then - echo "Error: No build target specified. Use --buildroot, --linux, or --all." - show_help -fi - -if [[ ( $BUILD_DIRECTFB_TEST -eq 1 || $BUILD_X11 -eq 1 ) && $NO_EXT4 -eq 1 ]]; then - echo "Error: --x11/--directfb2-test requires an ext4 image; remove --no-ext4." - show_help -fi - -if [[ $CLEAN_BUILD -eq 1 && $BUILD_BUILDROOT -eq 1 && -d buildroot ]]; then - echo "Removing buildroot/ for clean build..." - rm -rf buildroot -fi - -if [[ $CLEAN_BUILD -eq 1 && $BUILD_LINUX -eq 1 && -d linux ]]; then - echo "Removing linux/ for clean build..." - rm -rf linux -fi - -if [[ $CLEAN_BUILD -eq 1 && $BUILD_DIRECTFB_TEST -eq 1 ]]; then - echo "Removing DirectFB2 sources for clean build..." - rm -rf DirectFB2 DirectFB-examples directfb extra_packages -fi - -if [[ $BUILD_BUILDROOT -eq 1 ]]; then - do_buildroot && OK -fi - -if [[ $BUILD_LINUX -eq 1 ]]; then - do_linux && OK -fi diff --git a/scripts/check-rootfs-fakeroot.sh b/scripts/check-rootfs-fakeroot.sh new file mode 100755 index 00000000..49f5c8bd --- /dev/null +++ b/scripts/check-rootfs-fakeroot.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +# Probe whether fakeroot can run the same external tools used by +# scripts/rootfs_ext4.sh. Some hosts can start fakeroot but fail only when it +# injects into a later child command, so a shell-only probe is not enough. + +set -euo pipefail + +ROOTFS_CP="${ROOTFS_CP:-cp}" +ROOTFS_CHOWN="${ROOTFS_CHOWN:-chown}" +ROOTFS_FAKEROOT_SHELL="${ROOTFS_FAKEROOT_SHELL:-/bin/sh}" + +check_tool() { + local tool=$1 + local name=$2 + + if ! [ -x "$tool" ] && ! command -v "$tool" >/dev/null 2>&1; then + echo "[!] $name is required for fakeroot rootfs image generation" >&2 + exit 1 + fi +} + +check_tool fakeroot fakeroot +check_tool "$ROOTFS_FAKEROOT_SHELL" "fakeroot shell" +check_tool "$ROOTFS_CP" cp +check_tool "$ROOTFS_CHOWN" chown + +TEST_DIR="$(mktemp -d "${TMPDIR:-/tmp}/semu-fakeroot-probe.XXXXXX")" +PROBE_SCRIPT="$(mktemp "${TMPDIR:-/tmp}/semu-fakeroot-probe-script.XXXXXX")" +trap 'rm -rf "$TEST_DIR" "$PROBE_SCRIPT"' EXIT + +mkdir -p "$TEST_DIR/src" "$TEST_DIR/dst" +printf 'probe\n' > "$TEST_DIR/src/file" + +cat >"$PROBE_SCRIPT" <<'EOF' +#!/bin/sh +set -e + +if [ -z "${SEMU_FAKEROOT_TEST_DIR:-}" ] || \ + [ -z "${SEMU_ROOTFS_CP:-}" ] || \ + [ -z "${SEMU_ROOTFS_CHOWN:-}" ]; then + echo "[!] Missing fakeroot probe environment" >&2 + exit 1 +fi + +cd "$SEMU_FAKEROOT_TEST_DIR" +"$SEMU_ROOTFS_CP" -a src/. dst/ +"$SEMU_ROOTFS_CHOWN" -R 0:0 dst +EOF +chmod +x "$PROBE_SCRIPT" + +SEMU_FAKEROOT_TEST_DIR="$TEST_DIR" \ +SEMU_ROOTFS_CP="$ROOTFS_CP" \ +SEMU_ROOTFS_CHOWN="$ROOTFS_CHOWN" \ +fakeroot "$ROOTFS_FAKEROOT_SHELL" "$PROBE_SCRIPT" diff --git a/scripts/prebuilt/artifact-classes.sh b/scripts/prebuilt/artifact-classes.sh new file mode 100755 index 00000000..17d023d6 --- /dev/null +++ b/scripts/prebuilt/artifact-classes.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# +# Shared source-build artifact contract. This file describes the artifact +# classes that the source builder can produce and the raw output files for each +# class. It does not describe CI recipe keys, stamps, resolver actions, or +# release publish policy. + +set -euo pipefail + +source_artifact_class_registry() { + printf '%s\t%s\n' image Image + printf '%s\t%s\n' rootfs rootfs.cpio + printf '%s\t%s\n' test-tools test-tools.img +} + +source_artifact_classes() { + local class + local _outputs + + while IFS=$'\t' read -r class _outputs; do + printf '%s\n' "$class" + done < <(source_artifact_class_registry) +} + +source_artifact_class_outputs() { + local want=$1 + local class + local outputs + + while IFS=$'\t' read -r class outputs; do + if [ "$class" = "$want" ]; then + printf '%s\n' "$outputs" | tr ' ' '\n' + return 0 + fi + done < <(source_artifact_class_registry) + + echo "[!] Unknown source artifact class: $want" >&2 + return 1 +} + +source_artifact_outputs() { + local class + + while IFS= read -r class; do + source_artifact_class_outputs "$class" + done < <(source_artifact_classes) +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + case "${1:-}" in + outputs) + source_artifact_outputs + ;; + output) + [ "$#" -eq 2 ] || { + echo "usage: $0 output " >&2 + exit 2 + } + source_artifact_class_outputs "$2" + ;; + *) + echo "usage: $0 {outputs|output }" >&2 + exit 2 + ;; + esac +fi diff --git a/scripts/prebuilt/artifact-recipe.env b/scripts/prebuilt/artifact-recipe.env new file mode 100644 index 00000000..99c3cfc8 --- /dev/null +++ b/scripts/prebuilt/artifact-recipe.env @@ -0,0 +1,13 @@ +BUILDROOT_REPO=https://github.com/buildroot/buildroot +BUILDROOT_REV=898251ee2b83a9cd5ae0ae5db57828035a5a6f85 + +LINUX_REPO=https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git +LINUX_REV=c4ffbe29c40ed851601bce640d5ead48eaaae08d + +DIRECTFB2_REPO=https://github.com/directfb2/DirectFB2 +DIRECTFB2_REV=7d4682d0cc092ed2f28c903175d1a0c104e9e9a8 + +DIRECTFB_EXAMPLES_REPO=https://github.com/directfb2/DirectFB-examples +DIRECTFB_EXAMPLES_REV=eecf1019b29933a45578e62aea5f08a884d30fbc + +TEST_TOOLS_SIZE_MB=192 diff --git a/scripts/prebuilt/artifact-recipes.sh b/scripts/prebuilt/artifact-recipes.sh new file mode 100755 index 00000000..3c9d9040 --- /dev/null +++ b/scripts/prebuilt/artifact-recipes.sh @@ -0,0 +1,562 @@ +#!/usr/bin/env bash + +# This recipe library is sourced by scripts/build-artifacts.sh and the internal +# prebuilt plan builder. It intentionally enables fail-fast shell options for +# the caller before any artifact work starts. +set -euo pipefail + +ARTIFACT_RECIPES_DIR=$(CDPATH='' cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) + +# Load build recipe pins and recipe selection helpers before defining build +# helpers. CI owns recipe-key/stamp metadata separately under .ci/prebuilt; this +# file keeps the build-facing source configuration in one place. +{ + . "$ARTIFACT_RECIPES_DIR/artifact-classes.sh" + . "$ARTIFACT_RECIPES_DIR/artifact-recipe.env" + . "$ARTIFACT_RECIPES_DIR/test-tools-recipe.sh" +} + +ASSERT() { + local res + + if "$@"; then + return 0 + else + res=$? + fi + + echo 'Assert failed: "' "$@" '"' + exit "$res" +} + +PASS_COLOR='\e[32;01m' +NO_COLOR='\e[0m' +OK() { + printf ' [ %b OK %b ]\n' "$PASS_COLOR" "$NO_COLOR" +} + +# Match Buildroot/Linux parallelism to the host by default. Keep the value in +# one variable so every underlying make invocation uses the same job policy. +if command -v nproc >/dev/null 2>&1; then + build_jobs=$(nproc) +else + build_jobs=$(getconf _NPROCESSORS_ONLN 2>/dev/null || printf '1') +fi +PARALLEL="-j$build_jobs" + +BUILDROOT_DEFAULT_OUTPUT=${BUILDROOT_DEFAULT_OUTPUT:-buildroot/output-default} +BUILDROOT_TEST_TOOLS_OUTPUT=${BUILDROOT_TEST_TOOLS_OUTPUT:-buildroot/output-test-tools} + +# buildroot/ is shared mutable state: checkout, config, host toolchain, and +# rootfs output all live there. Serialize separate shell processes that might +# race on that tree, while still letting Buildroot use its own internal -j +# parallelism inside one locked operation. +BUILDROOT_LOCK_PATH=${BUILDROOT_LOCK_PATH:-.semu-buildroot.lock} +BUILDROOT_LOCK_DEPTH=0 +BUILDROOT_LOCK_TOKEN= + +buildroot_lock_host() { + hostname 2>/dev/null || uname -n 2>/dev/null || printf 'unknown' +} + +buildroot_new_lock_token() { + printf 'semu-buildroot:%s:%s:%s:%s\n' \ + "$(buildroot_lock_host)" "$$" "$(date +%s)" "$RANDOM$RANDOM" +} + +buildroot_lock_owner() { + readlink "$BUILDROOT_LOCK_PATH" 2>/dev/null || true +} + +buildroot_lock_owner_field() { + local owner=$1 + local field=$2 + local prefix host pid started nonce + + IFS=: read -r prefix host pid started nonce <<< "$owner" + case "$prefix" in + semu-buildroot) ;; + *) return 1 ;; + esac + + case "$field" in + host) printf '%s\n' "$host" ;; + pid) printf '%s\n' "$pid" ;; + *) return 1 ;; + esac +} + +buildroot_report_unowned_lock() { + cat >&2 << EOF +Error: Buildroot source build lock path is not an owned semu lock: + $BUILDROOT_LOCK_PATH + +Check that no semu artifact source build is running, then remove the lock: + rm -rf "$BUILDROOT_LOCK_PATH" +EOF +} + +buildroot_report_stale_lock() { + local owner=$1 + + cat >&2 << EOF +Error: Buildroot source build lock appears stale: + $BUILDROOT_LOCK_PATH -> $owner + +Portable shell/filesystem operations cannot remove this lock with an atomic +owner check. Check that no semu artifact source build is running, then remove +the lock: + rm -f "$BUILDROOT_LOCK_PATH" +EOF +} + +buildroot_lock_owner_is_valid() { + local owner=$1 + local host + local pid + + host=$(buildroot_lock_owner_field "$owner" host) || return 1 + pid=$(buildroot_lock_owner_field "$owner" pid) || return 1 + + [ -n "$host" ] || return 1 + case "$pid" in + ''|*[!0-9]*) return 1 ;; + esac +} + +buildroot_lock_path_is_unowned() { + [ -e "$BUILDROOT_LOCK_PATH" ] && [ ! -L "$BUILDROOT_LOCK_PATH" ] +} + +buildroot_lock_is_known_stale() { + local owner=$1 + local host + local pid + + host=$(buildroot_lock_owner_field "$owner" host) || return 1 + pid=$(buildroot_lock_owner_field "$owner" pid) || return 1 + + [ "$host" = "$(buildroot_lock_host)" ] || return 1 + case "$pid" in + ''|*[!0-9]*) return 0 ;; + esac + + ! kill -0 "$pid" 2>/dev/null +} + +buildroot_force_release_lock() { + if [ "${BUILDROOT_LOCK_DEPTH:-0}" -gt 0 ]; then + if [ "$(buildroot_lock_owner)" = "$BUILDROOT_LOCK_TOKEN" ]; then + rm -f "$BUILDROOT_LOCK_PATH" + fi + BUILDROOT_LOCK_DEPTH=0 + BUILDROOT_LOCK_TOKEN= + fi +} + +buildroot_release_lock() { + if [ "${BUILDROOT_LOCK_DEPTH:-0}" -gt 1 ]; then + BUILDROOT_LOCK_DEPTH=$((BUILDROOT_LOCK_DEPTH - 1)) + return + fi + + buildroot_force_release_lock +} + +buildroot_acquire_lock() { + local waited=false + local owner + + if [ "${BUILDROOT_LOCK_DEPTH:-0}" -gt 0 ]; then + BUILDROOT_LOCK_DEPTH=$((BUILDROOT_LOCK_DEPTH + 1)) + return 0 + fi + + if buildroot_lock_path_is_unowned; then + buildroot_report_unowned_lock + return 1 + fi + + BUILDROOT_LOCK_TOKEN=$(buildroot_new_lock_token) + while ! ln -s "$BUILDROOT_LOCK_TOKEN" "$BUILDROOT_LOCK_PATH" 2>/dev/null; do + owner=$(buildroot_lock_owner) + if [ -z "$owner" ] || ! buildroot_lock_owner_is_valid "$owner"; then + buildroot_report_unowned_lock + return 1 + fi + if buildroot_lock_is_known_stale "$owner"; then + buildroot_report_stale_lock "$owner" + return 1 + fi + if [ "$waited" = false ]; then + echo "Waiting for Buildroot source build lock: $owner" + waited=true + fi + sleep 1 + done + + if [ "$(buildroot_lock_owner)" != "$BUILDROOT_LOCK_TOKEN" ]; then + buildroot_report_unowned_lock + return 1 + fi + + BUILDROOT_LOCK_DEPTH=1 + # The build recipe is sourced by simple drivers that do not install their + # own EXIT trap. While a lock is held, this library owns the caller EXIT + # trap so interrupted builds do not leave Buildroot locked forever. + trap buildroot_force_release_lock EXIT +} + +with_buildroot_lock() { + local status + + if ! buildroot_acquire_lock; then + exit 1 + fi + "$@" + status=$? + buildroot_release_lock + if [ "$status" -ne 0 ]; then + exit "$status" + fi +} + +# Keep external source trees pinned to artifact-recipe.env revisions. A shallow +# fetch is enough for release pins most of the time; fall back to a full fetch +# when the requested revision is not reachable from a shallow fetch. +checkout_repo_rev() { + local dir=$1 + local repo=$2 + local rev=$3 + + if [ ! -d "$dir/.git" ]; then + echo "Cloning $dir..." + rm -rf "$dir" + ASSERT git init "$dir" + pushd "$dir" + ASSERT git remote add origin "$repo" + if ! git fetch --depth=1 origin "$rev"; then + ASSERT git fetch origin "$rev" + fi + ASSERT git checkout --detach FETCH_HEAD + popd + return + fi + + echo "$dir already exists, verifying checkout..." + pushd "$dir" + if ! git cat-file -e "$rev^{commit}" 2>/dev/null; then + if ! git fetch --depth=1 origin "$rev"; then + ASSERT git fetch origin "$rev" + fi + fi + ASSERT git checkout --detach "$rev" + popd +} + +ensure_buildroot_checkout() { + checkout_repo_rev buildroot "$BUILDROOT_REPO" "$BUILDROOT_REV" +} + +# Reuse Meson build directories when possible. If the old directory was created +# with incompatible options or stale metadata, rebuild it from scratch. +meson_setup_or_reconfigure() { + local build_dir=$1 + shift + + if [ -f "$build_dir/build.ninja" ]; then + if ! meson setup --reconfigure "$@" "$build_dir"; then + echo "Recreating stale Meson build directory: $build_dir" + rm -rf "$build_dir" + ASSERT meson setup "$@" "$build_dir" + fi + else + ASSERT meson setup "$@" "$build_dir" + fi +} + +buildroot_make() { + local output_dir=$1 + shift + + ASSERT mkdir -p "$output_dir" + ASSERT make -C buildroot O="$PWD/$output_dir" "$@" +} + +# The default mode copies the base config; the x11 mode overlays the X11 +# fragment used by test-tools variants. Keep default rootfs and test-tools in +# separate Buildroot output directories so `all` can build default ext4.img +# before the larger test-tools profile without contaminating the next run. +configure_buildroot() { + local mode=${1:-default} + local output_dir=$2 + local buildroot_config=configs/buildroot.config + local x11_config=configs/x11.config + local merge_tool=buildroot/support/kconfig/merge_config.sh + + ASSERT mkdir -p "$output_dir" + if [[ "$mode" == "x11" ]]; then + echo "Preparing Buildroot config with X11 fragment..." + ASSERT "$merge_tool" -m -r -O "$output_dir" "$buildroot_config" "$x11_config" + else + echo "Preparing default Buildroot config..." + ASSERT cp -f "$buildroot_config" "$output_dir/.config" + fi +} + +# Linux and DirectFB builds use the Buildroot-hosted RISC-V cross toolchain. +# Building only the toolchain avoids publishing a rootfs as a side effect. +build_buildroot_toolchain() { + local buildroot_output=$BUILDROOT_DEFAULT_OUTPUT + + ensure_buildroot_checkout + configure_buildroot default "$buildroot_output" + + unset LD_LIBRARY_PATH + buildroot_make "$buildroot_output" olddefconfig + buildroot_make "$buildroot_output" toolchain "$PARALLEL" +} + +# Build the pinned Linux tree with the Buildroot cross toolchain, then publish +# only arch/riscv/boot/Image as the kernel artifact. +do_linux_unlocked() { + local buildroot_output=$BUILDROOT_DEFAULT_OUTPUT + + build_buildroot_toolchain + + checkout_repo_rev linux "$LINUX_REPO" "$LINUX_REV" + + ASSERT cp -f configs/linux.config linux/.config + + export PATH="$PWD/$buildroot_output/host/bin:$PATH" + export CROSS_COMPILE=riscv32-buildroot-linux-gnu- + export ARCH=riscv + pushd linux + ASSERT make olddefconfig + ASSERT make "$PARALLEL" + ASSERT cp -f arch/riscv/boot/Image ../Image + popd +} + +do_linux() { + with_buildroot_lock do_linux_unlocked +} + +build_buildroot_rootfs() { + local mode=${1:-default} + local buildroot_output=$2 + + ensure_buildroot_checkout + configure_buildroot "$mode" "$buildroot_output" + ASSERT cp -f configs/busybox.config buildroot/busybox.config + ASSERT cp -f target/init buildroot/fs/cpio/init + + # Buildroot rejects an LD_LIBRARY_PATH containing the current working + # directory, so keep the environment clean before invoking make. + unset LD_LIBRARY_PATH + buildroot_make "$buildroot_output" olddefconfig + if [[ "$mode" == "x11" && \ + ! -x "$buildroot_output/host/bin/riscv32-buildroot-linux-gnu-g++" ]]; then + echo "Rebuilding Buildroot final GCC with C++ support..." + buildroot_make "$buildroot_output" host-gcc-final-dirclean + fi + buildroot_make "$buildroot_output" "$PARALLEL" +} + +do_rootfs_unlocked() { + build_buildroot_rootfs default "$BUILDROOT_DEFAULT_OUTPUT" + + # rootfs.cpio is the canonical raw rootfs artifact and also serves as the + # legacy initramfs payload. Local ext4.img generation is a caller-level + # convenience layered on top of this raw artifact. + echo "Publishing rootfs.cpio" + ASSERT cp -f "$BUILDROOT_DEFAULT_OUTPUT/images/rootfs.cpio" ./rootfs.cpio +} + +do_rootfs() { + with_buildroot_lock do_rootfs_unlocked +} + +# DirectFB2 is installed into two places: the Buildroot sysroot so examples can +# link against it, and directfb/ so rootfs_ext4.sh can overlay runtime payloads. +do_directfb() { + local buildroot_output=$1 + + export BUILDROOT_OUT=$PWD/$buildroot_output + export DIRECTFB_STAGE=$PWD/directfb + ASSERT mkdir -p directfb + + checkout_repo_rev DirectFB2 "$DIRECTFB2_REPO" "$DIRECTFB2_REV" + pushd DirectFB2 + sed "s#@BUILDROOT_OUTPUT@#$buildroot_output#g" \ + ../configs/meson-riscv-cross-file > meson-riscv-cross-file + meson_setup_or_reconfigure build/riscv -Ddrmkms=true --cross-file \ + meson-riscv-cross-file + ASSERT meson compile -C build/riscv + ASSERT env DESTDIR="$BUILDROOT_OUT/host/riscv32-buildroot-linux-gnu/sysroot" meson install -C build/riscv + ASSERT env DESTDIR="$DIRECTFB_STAGE" meson install -C build/riscv + popd + + checkout_repo_rev DirectFB-examples "$DIRECTFB_EXAMPLES_REPO" \ + "$DIRECTFB_EXAMPLES_REV" + pushd DirectFB-examples/ + sed "s#@BUILDROOT_OUTPUT@#$buildroot_output#g" \ + ../configs/meson-riscv-cross-file > meson-riscv-cross-file + meson_setup_or_reconfigure build/riscv --cross-file meson-riscv-cross-file + ASSERT meson compile -C build/riscv + ASSERT env DESTDIR="$DIRECTFB_STAGE" meson install -C build/riscv + popd +} + +# Build optional runtime payloads and collect them under extra_packages/. That +# directory is later overlaid into test-tools.img, not the default rootfs. +do_extra_packages() { + local buildroot_output=$1 + + export PATH="$PWD/$buildroot_output/host/bin:$PATH" + export CROSS_COMPILE=riscv32-buildroot-linux-gnu- + + ASSERT rm -rf directfb extra_packages + ASSERT mkdir -p extra_packages/root + + do_directfb "$buildroot_output" + OK + + if ! find directfb -mindepth 1 -print -quit | grep -q .; then + echo "Error: DirectFB staging tree is empty." + exit 1 + fi + + ASSERT cp -r directfb/. extra_packages/ + ASSERT cp target/local-env.sh extra_packages/root/ +} + +# The x11 test-tools variant needs libstdc++ at runtime, but the default +# guest rootfs should stay small. Stage only the needed C++ libraries. +stage_cxx_runtime() { + local buildroot_output=$1 + local toolchain_lib=$buildroot_output/host/riscv32-buildroot-linux-gnu/lib + local libstdcpp=$toolchain_lib/libstdc++.so.6 + local libstdcpp_real + + if [ ! -e "$libstdcpp" ]; then + echo "Error: libstdc++.so.6 not found in $toolchain_lib" + exit 1 + fi + + libstdcpp_real=$(readlink "$libstdcpp" || basename "$libstdcpp") + if [[ "$libstdcpp_real" != /* ]]; then + libstdcpp_real=$toolchain_lib/$libstdcpp_real + fi + ASSERT mkdir -p extra_packages/lib + ASSERT cp -a "$toolchain_lib/libstdc++.so" "$libstdcpp" \ + "$libstdcpp_real" extra_packages/lib/ +} + +# Translate the normalized test-tools recipe into build decisions. x11 selects +# the Buildroot profile; directfb2 adds the DirectFB2 test payload. +configure_test_tools_recipe_flags() { + BUILD_X11=0 + BUILD_DIRECTFB_TEST=0 + + if prebuilt_test_tools_recipe_includes x11; then + BUILD_X11=1 + fi + if prebuilt_test_tools_recipe_includes directfb2; then + BUILD_DIRECTFB_TEST=1 + fi +} + +# Build test-tools.img from a Buildroot rootfs plus optional overlays. The +# recipe can be x11-only, directfb2-only, or the canonical x11,directfb2 set. +do_test_tools_unlocked() { + local buildroot_mode + local buildroot_output=$BUILDROOT_TEST_TOOLS_OUTPUT + local test_tools_rootfs + + configure_test_tools_recipe_flags + + if [[ $BUILD_X11 -eq 1 ]]; then + buildroot_mode=x11 + else + buildroot_mode=default + fi + build_buildroot_rootfs "$buildroot_mode" "$buildroot_output" + test_tools_rootfs=./$buildroot_output/images/rootfs.cpio + + if [[ $BUILD_DIRECTFB_TEST -eq 1 ]]; then + do_extra_packages "$buildroot_output" + if [[ $BUILD_X11 -eq 1 ]]; then + stage_cxx_runtime "$buildroot_output" + fi + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" ./extra_packages + elif [[ $BUILD_X11 -eq 1 ]]; then + ASSERT rm -rf extra_packages + stage_cxx_runtime "$buildroot_output" + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" ./extra_packages + else + ASSERT ./scripts/rootfs_ext4.sh "$test_tools_rootfs" ./test-tools.img \ + "$TEST_TOOLS_SIZE_MB" + fi +} + +do_test_tools() { + with_buildroot_lock do_test_tools_unlocked +} + +source_artifact_class_selected() { + local want=$1 + local class + shift + + for class in "$@"; do + if [[ $class == "$want" ]]; then + return 0 + fi + done + + return 1 +} + +build_source_artifact_class() { + case "$1" in + image) + do_linux + ;; + rootfs) + do_rootfs + ;; + test-tools) + do_test_tools + ;; + *) + echo "[!] Unknown source artifact class: $1" >&2 + return 1 + ;; + esac +} + +# CI asks for classes by name, but this dispatcher keeps execution order fixed. +# That keeps shared Buildroot state deterministic even if caller argument order +# changes. +build_prebuilt_artifact_classes() { + local class + + if [ "$#" -eq 0 ]; then + echo "[!] build_prebuilt_artifact_classes requires at least one class" >&2 + return 1 + fi + + for class in "$@"; do + source_artifact_class_outputs "$class" >/dev/null || return 1 + done + + while IFS= read -r class; do + if source_artifact_class_selected "$class" "$@"; then + build_source_artifact_class "$class" + OK + fi + done < <(source_artifact_classes) +} diff --git a/scripts/prebuilt/download-release-artifact.sh b/scripts/prebuilt/download-release-artifact.sh new file mode 100755 index 00000000..87d395bc --- /dev/null +++ b/scripts/prebuilt/download-release-artifact.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# +# Fetch one raw artifact from the rolling prebuilt release and decompress it in +# place. The rolling prebuilt manifest provides archive byte checksums for +# transport integrity. CI callers may pass the expected recipe key so release +# manifest drift between resolve and download fails before stamping. + +set -euo pipefail + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + echo "usage: $0 [expected-recipe-key]" >&2 + exit 2 +fi + +artifact=$1 +expected_recipe_key=${2:-} +archive=${artifact}.bz2 +recipe_entry=${artifact}.recipe-key + +: "${PREBUILT_URL:?Need PREBUILT_URL pointing at the prebuilt artifact directory}" + +tmpdir=$(mktemp -d) +cleanup_download_parts() { + rm -rf "$tmpdir" + rm -f "$archive.part" "$artifact.tmp" +} +trap cleanup_download_parts EXIT + +manifest=$tmpdir/prebuilt.sha1 + +curl_progress=(--silent --show-error) +if [ -t 2 ]; then + curl_progress=(--progress-bar) +fi + +sha1_file() { + local -a sha1_cmd + + if command -v sha1sum >/dev/null 2>&1; then + sha1_cmd=(sha1sum) + elif command -v shasum >/dev/null 2>&1; then + sha1_cmd=(shasum -a 1) + else + echo "[!] Need sha1sum or shasum on PATH" >&2 + return 1 + fi + + "${sha1_cmd[@]}" "$1" | awk '{print $1}' +} + +manifest_entry_value() { + local entry=$1 + + awk -v f="$entry" '$2 == f {print $1; found = 1; exit} END {exit !found}' "$manifest" +} + +if ! curl --fail --retry 3 --retry-delay 1 --silent --show-error \ + -L -o "$manifest.part" "${PREBUILT_URL}/prebuilt.sha1"; then + rm -f "$manifest.part" + exit 1 +fi +mv -f "$manifest.part" "$manifest" + +if ! expected_archive_sha=$(manifest_entry_value "$archive"); then + echo "[!] prebuilt manifest missing $archive" >&2 + exit 1 +fi + +if [ -n "$expected_recipe_key" ]; then + if ! manifest_recipe_key=$(manifest_entry_value "$recipe_entry"); then + echo "[!] prebuilt manifest missing $recipe_entry" >&2 + exit 1 + fi + if [ "$manifest_recipe_key" != "$expected_recipe_key" ]; then + echo "[!] prebuilt manifest recipe key mismatch for $artifact" >&2 + echo "[!] expected $expected_recipe_key" >&2 + echo "[!] actual $manifest_recipe_key" >&2 + exit 1 + fi +fi + +if ! curl --fail --retry 3 --retry-delay 1 "${curl_progress[@]}" \ + -L -o "$archive.part" "${PREBUILT_URL}/${archive}"; then + rm -f "$archive.part" + exit 1 +fi + +actual_archive_sha=$(sha1_file "$archive.part") +if [ "$actual_archive_sha" != "$expected_archive_sha" ]; then + echo "[!] prebuilt archive checksum mismatch for $archive" >&2 + echo "[!] expected $expected_archive_sha" >&2 + echo "[!] actual $actual_archive_sha" >&2 + exit 1 +fi + +mv -f "$archive.part" "$archive" +if ! bunzip2 -c "$archive" > "$artifact.tmp"; then + rm -f "$artifact.tmp" + exit 1 +fi +mv -f "$artifact.tmp" "$artifact" +rm -f "$archive" diff --git a/scripts/prebuilt/test-tools-recipe.sh b/scripts/prebuilt/test-tools-recipe.sh new file mode 100644 index 00000000..b6f6f26d --- /dev/null +++ b/scripts/prebuilt/test-tools-recipe.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash + +# Build-layer helper for selecting optional test-tools.img recipe entries. CI +# recipe keys source this same helper so recipe selection affects both local +# builds and CI cache/release decisions without making the build recipe depend +# on CI state. +prebuilt_test_tools_recipe_entries() { + local raw=${PREBUILT_TEST_TOOLS_RECIPE+x} + local recipe=${PREBUILT_TEST_TOOLS_RECIPE-} + local item + local want_x11=false + local want_directfb2=false + + if [ -z "$raw" ]; then + printf "%s\n" x11 directfb2 + return 0 + fi + + recipe=${recipe//,/ } + if [ -z "${recipe// /}" ]; then + echo "[!] Empty test-tools recipe" >&2 + echo "[!] Expected recipe entries: x11, directfb2" >&2 + return 1 + fi + + for item in $recipe; do + case "$item" in + x11) + want_x11=true + ;; + directfb2) + want_directfb2=true + ;; + *) + echo "[!] Unknown test-tools recipe entry: $item" >&2 + echo "[!] Expected recipe entries: x11, directfb2" >&2 + return 1 + ;; + esac + done + + if [ "$want_x11" = true ]; then + printf "%s\n" x11 + fi + if [ "$want_directfb2" = true ]; then + printf "%s\n" directfb2 + fi +} + +prebuilt_test_tools_recipe_includes() { + local want=$1 + local entry + local recipe_entries + + recipe_entries=$(prebuilt_test_tools_recipe_entries) || return 1 + while IFS= read -r entry; do + if [ "$entry" = "$want" ]; then + return 0 + fi + done <<< "$recipe_entries" + + return 1 +} + +prebuilt_test_tools_recipe_key() { + local entry + local recipe_entries + local result= + + recipe_entries=$(prebuilt_test_tools_recipe_entries) || return 1 + while IFS= read -r entry; do + if [ -z "$entry" ]; then + continue + fi + if [ -n "$result" ]; then + result=$result,$entry + else + result=$entry + fi + done <<< "$recipe_entries" + + if [ -z "$result" ]; then + echo "[!] Empty test-tools recipe" >&2 + return 1 + fi + + printf "%s\n" "$result" +} diff --git a/scripts/rootfs_ext4.sh b/scripts/rootfs_ext4.sh index c8d783a0..f6f18455 100755 --- a/scripts/rootfs_ext4.sh +++ b/scripts/rootfs_ext4.sh @@ -17,6 +17,13 @@ OUT_IMG="${2:-ext4.img}" SIZE_MB="${3:-32}" EXTRA_DIR="${4:-}" MKFS_EXT4="${MKFS_EXT4:-mkfs.ext4}" +# Caller-facing hooks for hosts where the default toolchain is unsuitable. +# CI can point these at a package-manager-provided toolchain without baking +# platform policy into this helper. +ROOTFS_CPIO="${ROOTFS_CPIO:-cpio}" +ROOTFS_CP="${ROOTFS_CP:-cp}" +ROOTFS_CHOWN="${ROOTFS_CHOWN:-chown}" +ROOTFS_FAKEROOT_SHELL="${ROOTFS_FAKEROOT_SHELL:-/bin/sh}" if [ ! -f "$SRC_CPIO" ]; then echo "[!] Source cpio not found: $SRC_CPIO" >&2 @@ -28,7 +35,27 @@ if ! command -v fakeroot >/dev/null 2>&1; then exit 1 fi -if ! command -v "$MKFS_EXT4" >/dev/null 2>&1; then +if ! [ -x "$ROOTFS_CPIO" ] && ! command -v "$ROOTFS_CPIO" >/dev/null 2>&1; then + echo "[!] cpio is required to build the ext4 image" >&2 + exit 1 +fi + +if ! [ -x "$ROOTFS_CP" ] && ! command -v "$ROOTFS_CP" >/dev/null 2>&1; then + echo "[!] cp is required to build the ext4 image" >&2 + exit 1 +fi + +if ! [ -x "$ROOTFS_CHOWN" ] && ! command -v "$ROOTFS_CHOWN" >/dev/null 2>&1; then + echo "[!] chown is required to build the ext4 image" >&2 + exit 1 +fi + +if ! [ -x "$ROOTFS_FAKEROOT_SHELL" ] && ! command -v "$ROOTFS_FAKEROOT_SHELL" >/dev/null 2>&1; then + echo "[!] fakeroot shell is required to build the ext4 image" >&2 + exit 1 +fi + +if ! [ -x "$MKFS_EXT4" ] && ! command -v "$MKFS_EXT4" >/dev/null 2>&1; then echo "[!] mkfs.ext4 is required to build the ext4 image" >&2 exit 1 fi @@ -49,7 +76,8 @@ fi # is a template) -- spell out the full template instead. STAGE="$(mktemp -d "${TMPDIR:-/tmp}/semu-rootfs.XXXXXX")" OUT_TMP="$(mktemp "$OUT_DIR/.$(basename "$OUT_IMG").XXXXXX")" -trap 'rm -rf "$STAGE" "$OUT_TMP"' EXIT +FAKEROOT_SCRIPT="$(mktemp "${TMPDIR:-/tmp}/semu-fakeroot.XXXXXX")" +trap 'rm -rf "$STAGE" "$OUT_TMP" "$FAKEROOT_SCRIPT"' EXIT echo "[*] Creating empty image: $OUT_IMG (${SIZE_MB} MiB)" # bs=1024k works on both GNU and BSD dd; bs=1M is GNU-only and bs=1m is @@ -61,28 +89,64 @@ echo "[*] Extracting $SRC_CPIO -> $STAGE" if [ -n "$EXTRA_ABS" ]; then echo "[*] Applying extra files: $EXTRA_DIR" fi + # -E lazy_*_init=0: do all init at mkfs time so the first guest mount does -# not pay the lazy-init cost. Stripping the journal (-O ^has_journal) -# would also speed mount, but the prebuilt Linux Image is built with -# CONFIG_EXT4_USE_FOR_EXT2=n and refuses to mount a no-journal image. -fakeroot bash -c ' - set -e - stage="$1" - src_cpio="$2" - extra_dir="$3" - mkfs_ext4="$4" - out_img="$5" - - cd "$stage" - cpio -idm < "$src_cpio" - if [ -n "$extra_dir" ]; then - cp -a "$extra_dir"/. . - fi - chown -R 0:0 . - "$mkfs_ext4" -q -F \ - -E lazy_itable_init=0,lazy_journal_init=0 \ - -d . "$out_img" - ' sh "$STAGE" "$SRC_ABS" "$EXTRA_ABS" "$MKFS_EXT4" "$OUT_TMP" +# not pay the lazy-init cost. Stripping the journal (-O ^has_journal) +# would also speed mount, but the prebuilt Linux Image is built with +# CONFIG_EXT4_USE_FOR_EXT2=n and refuses to mount a no-journal image. +# +# Keep cpio extraction and mkfs in the same fakeroot child so ownership and +# device-node metadata are preserved in the generated image. Run a temporary +# script through an explicit shell so callers can choose a shell from the same +# toolchain as the other fakeroot child commands. +cat >"$FAKEROOT_SCRIPT" <<'EOF' +#!/bin/sh +set -e + +if [ -z "${SEMU_ROOTFS_STAGE:-}" ] || \ + [ -z "${SEMU_SRC_CPIO:-}" ] || \ + [ -z "${SEMU_ROOTFS_CPIO:-}" ] || \ + [ -z "${SEMU_ROOTFS_CP:-}" ] || \ + [ -z "${SEMU_ROOTFS_CHOWN:-}" ] || \ + [ -z "${SEMU_MKFS_EXT4:-}" ] || \ + [ -z "${SEMU_OUT_IMG:-}" ]; then + echo "[!] Missing rootfs_ext4.sh fakeroot environment" >&2 + exit 1 +fi + +cd "$SEMU_ROOTFS_STAGE" +set +e +"$SEMU_ROOTFS_CPIO" -idm < "$SEMU_SRC_CPIO" +cpio_status=$? +set -e +if [ ! -x sbin/init ]; then + echo "[!] Extracted rootfs does not contain executable /sbin/init" >&2 + exit 1 +fi +if [ "$cpio_status" -ne 0 ]; then + echo "[*] cpio reported non-fatal extraction warnings; continuing after rootfs validation" >&2 +fi +if [ -n "$SEMU_EXTRA_DIR" ]; then + "$SEMU_ROOTFS_CP" -a "$SEMU_EXTRA_DIR"/. . +fi +# Preserve the historical ext4.img policy: generated images are fully +# root-owned after cpio extraction and any optional overlay copy. +"$SEMU_ROOTFS_CHOWN" -R 0:0 . +"$SEMU_MKFS_EXT4" -q -F \ + -E lazy_itable_init=0,lazy_journal_init=0 \ + -d . "$SEMU_OUT_IMG" +EOF +chmod +x "$FAKEROOT_SCRIPT" + +SEMU_ROOTFS_STAGE="$STAGE" \ +SEMU_SRC_CPIO="$SRC_ABS" \ +SEMU_ROOTFS_CPIO="$ROOTFS_CPIO" \ +SEMU_ROOTFS_CP="$ROOTFS_CP" \ +SEMU_ROOTFS_CHOWN="$ROOTFS_CHOWN" \ +SEMU_EXTRA_DIR="$EXTRA_ABS" \ +SEMU_MKFS_EXT4="$MKFS_EXT4" \ +SEMU_OUT_IMG="$OUT_TMP" \ +fakeroot "$ROOTFS_FAKEROOT_SHELL" "$FAKEROOT_SCRIPT" mv -f "$OUT_TMP" "$OUT_ABS" du -h "$OUT_ABS"