diff --git a/.env.example b/.env.example index 237f47c..4304863 100755 --- a/.env.example +++ b/.env.example @@ -16,3 +16,11 @@ VDM_LOCAL_LLM_MODEL= # Runtime tokens are deliberately absent. Supply them only in a session file under: # /run/user/$UID/vdm-opencode/.env + +# Local release controls. The Gitea package token must be exported only at runtime. +LOCAL_BUILD_VARIANTS=all +LOCAL_PUBLISH_GITEA=false +LOCAL_REQUIRE_CLEAN=true +LOCAL_REQUIRE_TAG=false +GITEA_PACKAGE_OWNER= +GITEA_PACKAGE_USER=package-publisher diff --git a/Makefile b/Makefile index 8ee14fd..d4e3b96 100644 --- a/Makefile +++ b/Makefile @@ -1,18 +1,20 @@ SHELL := /usr/bin/env bash -.PHONY: help validate bootstrap apply-incus build-base build-php build-python build-cpp build-typescript build-full build-all +.PHONY: help validate bootstrap apply-incus build-base build-php build-python build-cpp build-typescript build-full build-all local-release help: @printf '%s\n' \ 'make validate Validate repository syntax and secret hygiene' \ 'make bootstrap Install host prerequisites and initialise Incus' \ 'make apply-incus Apply project, networks, ACL and profiles' \ + 'make build-base Build the base image' \ 'make build-php Build the PHP/Joomla image' \ 'make build-python Build the Python image' \ 'make build-cpp Build the C/C++ image' \ 'make build-typescript Build the TypeScript/browser image' \ 'make build-full Build the full mixed-language image' \ - 'make build-all Build every image variant' + 'make build-all Build every image variant' \ + 'make local-release Build, package, verify and optionally publish locally' validate: ./tests/validate-repository.sh @@ -43,3 +45,6 @@ build-full: build-all: ./scripts/build-all-images.sh + +local-release: + ./scripts/local-image-release.sh diff --git a/README.md b/README.md index 09937e7..c8e98e5 100755 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ The repository is the authority. Incus images and Gitea packages are generated a - Runtime-only credential handling under the guest's `/run` tmpfs. - A reference external broker stack for OpenBao, an LLM gateway and a TLS reverse proxy. - Provider-correct GitHub Actions and Gitea Actions workflows. -- Downloadable GitHub workflow artifacts and durable Gitea Generic Package publication. +- Downloadable GitHub workflow artifacts, fully local image releases, and durable Gitea Generic Package publication. - Image sanitisation and secret-scanning tests. - Host-side voice recording and transcription through any OpenAI-compatible transcription endpoint. @@ -43,8 +43,8 @@ cp .env.example .env ./tests/validate-repository.sh ./scripts/bootstrap-host.sh ./scripts/apply-incus.sh -./scripts/build-image.sh php -./scripts/package-image.sh php +export LOCAL_BUILD_VARIANTS=php +./scripts/local-image-release.sh ./scripts/launch-vm.sh php opencode-llewellyn ./scripts/start-session.sh opencode-llewellyn ``` @@ -88,6 +88,7 @@ Start with: - `docs/browser-testing.md` - `docs/credentials-and-brokers.md` - `docs/scaling-and-operations.md` +- `docs/local-image-builds.md` - `docs/gitea-packages.md` - `docs/github-and-gitea.md` - `docs/backup-and-migration.md` diff --git a/docs/gitea-packages.md b/docs/gitea-packages.md index 7b81e81..cafe46d 100755 --- a/docs/gitea-packages.md +++ b/docs/gitea-packages.md @@ -9,6 +9,8 @@ ## Publish +For a complete local build, verification, version check, and optional direct publication, use `scripts/local-image-release.sh`; see `docs/local-image-builds.md`. The lower-level commands below remain available for publishing an already-built variant. + ```bash ./scripts/package-image.sh php diff --git a/docs/github-and-gitea.md b/docs/github-and-gitea.md index 4107f57..44c4af2 100755 --- a/docs/github-and-gitea.md +++ b/docs/github-and-gitea.md @@ -4,11 +4,15 @@ GitHub can host the source, validate every push and retain downloadable workflow artifacts. Incus image creation still requires a trusted self-hosted Ubuntu machine with hardware virtualisation, Incus and sufficient storage. -Gitea remains fully supported for source hosting, Actions, its first-party MCP and durable Generic Packages. +Gitea remains fully supported for source hosting, Actions, its first-party MCP and durable Generic Packages. A third supported mode builds and publishes directly from an in-house machine without registering any Actions runner; GitHub does not need inbound access to that host. Do not make GitHub and Gitea independently writable authorities for the same branches. Choose one source of truth and mirror in one direction. +## Local builder without GitHub Actions + +Run `scripts/local-image-release.sh` on the trusted Incus builder. It validates, builds, packages and verifies locally, and can upload directly to Gitea when `LOCAL_PUBLISH_GITEA=true`. See `docs/local-image-builds.md` for the exact environment variables and tagged production-release procedure. This is the simplest option for a builder behind a firewall because all network connections originate from the builder. + ## Recommended migration path back to Gitea 1. Keep GitHub as the temporary writable authority while the first `0.2.x` images are proven. diff --git a/docs/local-image-builds.md b/docs/local-image-builds.md new file mode 100644 index 0000000..240f132 --- /dev/null +++ b/docs/local-image-builds.md @@ -0,0 +1,107 @@ +# Local image builds and releases + +This path builds, packages, verifies and optionally publishes Incus images entirely on an in-house builder. GitHub does not need network access to the builder. The host only needs outbound access to clone the repository, obtain build dependencies and, when enabled, upload packages to Gitea. + +## One-time builder setup + +Use a trusted Ubuntu host with hardware virtualisation, read/write access to `/dev/kvm`, Incus, and at least the configured `MIN_BUILD_FREE_GIB`. + +```bash +git clone https://github.com/vast-development-method/opencode-platform.git +cd opencode-platform +cp .env.example .env +./scripts/bootstrap-host.sh +./scripts/ci/check-incus-runner.sh +``` + +The builder does not need a GitHub Actions runner or a Gitea Actions runner. + +## Build locally without publishing + +Build all six variants: + +```bash +export LOCAL_BUILD_VARIANTS=all +./scripts/local-image-release.sh +``` + +Build only the most useful initial variants: + +```bash +export LOCAL_BUILD_VARIANTS='php,typescript,full' +./scripts/local-image-release.sh +``` + +Packages remain under `build/packages///`. Every package contains the Incus export, `manifest.json`, and `SHA256SUMS`; the release script verifies the checksums before it succeeds. + +## Publish directly to Gitea + +Create a dedicated Gitea package publisher account and a package-write-only token. Keep the token out of `.env`, shell history, Git, and the built images. + +```bash +read -rsp 'Gitea package token: ' GITEA_PACKAGE_TOKEN +printf '\n' +export GITEA_PACKAGE_TOKEN + +export LOCAL_BUILD_VARIANTS=all +export LOCAL_PUBLISH_GITEA=true +export GITEA_BASE_URL='https://git.vdm.dev' +export GITEA_PACKAGE_OWNER='vast-development-method' +export GITEA_PACKAGE_USER='package-publisher' + +./scripts/local-image-release.sh +unset GITEA_PACKAGE_TOKEN +``` + +The package names are `vdm-opencode-`, and the package version is read from both `VERSION` and `manifest/platform.env`. The command refuses to run if those versions disagree. + +## Production tagged release + +For a reproducible production release, build the exact protected tag and require both a clean checkout and the matching tag: + +```bash +git fetch --tags --prune origin +git switch --detach v0.2.0 + +export LOCAL_BUILD_VARIANTS=all +export LOCAL_REQUIRE_CLEAN=true +export LOCAL_REQUIRE_TAG=true +export LOCAL_PUBLISH_GITEA=true +export GITEA_BASE_URL='https://git.vdm.dev' +export GITEA_PACKAGE_OWNER='vast-development-method' +export GITEA_PACKAGE_USER='package-publisher' + +read -rsp 'Gitea package token: ' GITEA_PACKAGE_TOKEN +printf '\n' +export GITEA_PACKAGE_TOKEN + +./scripts/local-image-release.sh +unset GITEA_PACKAGE_TOKEN +``` + +Replace `v0.2.0` with the version being released. Tags must not be recreated or force-moved after packages are published. + +## Supported environment variables + +| Variable | Default | Purpose | +|---|---:|---| +| `LOCAL_BUILD_VARIANTS` | `all` | Comma- or space-separated variants, or `all`. | +| `LOCAL_PUBLISH_GITEA` | `false` | Upload verified packages directly to Gitea. | +| `LOCAL_REQUIRE_CLEAN` | `true` | Refuse a dirty Git checkout. | +| `LOCAL_REQUIRE_TAG` | `false` | Require HEAD to have tag `v`. | +| `GITEA_BASE_URL` | `https://git.vdm.dev` | Gitea base URL. | +| `GITEA_PACKAGE_OWNER` | unset | Gitea package owner. | +| `GITEA_PACKAGE_USER` | unset | Dedicated publishing user. | +| `GITEA_PACKAGE_TOKEN` | unset | Runtime-only package-write token. | + +## Import and use + +Copy or download all files for one variant, then import them on an Incus host: + +```bash +./scripts/import-image-package.sh /absolute/path/to/package +incus --project vdm-agents image list +./scripts/launch-vm.sh php opencode-llewellyn +``` + +Runtime GitHub, Gitea MCP, model, and other authentication is added only when starting a VM session. It is not embedded in released images. diff --git a/scripts/lib/variant-selection.sh b/scripts/lib/variant-selection.sh new file mode 100644 index 0000000..d440171 --- /dev/null +++ b/scripts/lib/variant-selection.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +LOCAL_RELEASE_ALL_VARIANTS=(base php python cpp typescript full) + +parse_variant_selection() { + local raw_selection="${1-}" + local output_name="${2:?Output array name is required}" + local normalized + local variant + local -a requested=() + local -n output="$output_name" + local -A seen=() + + output=() + normalized="${raw_selection//,/ }" + + IFS=$' \t\r\n' read -r -d '' -a requested < <(printf '%s\0' "$normalized") || true + if (("${#requested[@]}" == 0)); then + printf 'LOCAL_BUILD_VARIANTS selected no variants.\n' >&2 + return 2 + fi + + if (("${#requested[@]}" > 1)); then + for variant in "${requested[@]}"; do + if [[ "$variant" == "all" ]]; then + printf 'LOCAL_BUILD_VARIANTS cannot combine all with named variants.\n' >&2 + return 2 + fi + done + fi + + for variant in "${requested[@]}"; do + if [[ "$variant" == "all" ]]; then + output=("${LOCAL_RELEASE_ALL_VARIANTS[@]}") + return 0 + fi + + case "$variant" in + base|php|python|cpp|typescript|full) ;; + *) + printf 'Unknown variant: %s\n' "$variant" >&2 + return 2 + ;; + esac + + if [[ -z "${seen[$variant]:-}" ]]; then + output+=("$variant") + seen["$variant"]=1 + fi + done +} diff --git a/scripts/local-image-release.sh b/scripts/local-image-release.sh new file mode 100755 index 0000000..f75d0eb --- /dev/null +++ b/scripts/local-image-release.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib/common.sh" +# shellcheck disable=SC1091 +source "$SCRIPT_DIR/lib/variant-selection.sh" + +RAW_VARIANTS="${LOCAL_BUILD_VARIANTS:-all}" +PUBLISH_GITEA="${LOCAL_PUBLISH_GITEA:-false}" +REQUIRE_CLEAN="${LOCAL_REQUIRE_CLEAN:-true}" +REQUIRE_TAG="${LOCAL_REQUIRE_TAG:-false}" + +normalize_bool() { + case "${1,,}" in + true|1|yes) printf 'true' ;; + false|0|no) printf 'false' ;; + *) die "Boolean value must be true or false: $1" ;; + esac +} + +PUBLISH_GITEA="$(normalize_bool "$PUBLISH_GITEA")" +REQUIRE_CLEAN="$(normalize_bool "$REQUIRE_CLEAN")" +REQUIRE_TAG="$(normalize_bool "$REQUIRE_TAG")" + +variants=() +parse_variant_selection "$RAW_VARIANTS" variants || die "Invalid LOCAL_BUILD_VARIANTS value." + +version_file="$(tr -d '[:space:]' < "$ROOT_DIR/VERSION")" +[ -n "$version_file" ] || die "VERSION is empty." +[ "$version_file" = "$PLATFORM_VERSION" ] || + die "VERSION ($version_file) does not match PLATFORM_VERSION ($PLATFORM_VERSION)." + +require_command git +require_command flock +require_command sha256sum + +if [ "$REQUIRE_CLEAN" = true ]; then + git -C "$ROOT_DIR" diff --quiet --ignore-submodules -- + git -C "$ROOT_DIR" diff --cached --quiet --ignore-submodules -- + [ -z "$(git -C "$ROOT_DIR" status --porcelain --untracked-files=normal)" ] || + die "The source tree is not clean. Commit or remove local changes, or set LOCAL_REQUIRE_CLEAN=false for a non-release build." +fi + +source_revision="$(git -C "$ROOT_DIR" rev-parse --verify HEAD)" +source_tag="$(git -C "$ROOT_DIR" tag --points-at "$source_revision" | grep -Fx "v$PLATFORM_VERSION" || true)" +if [ "$REQUIRE_TAG" = true ] && [ -z "$source_tag" ]; then + die "HEAD must be tagged v$PLATFORM_VERSION when LOCAL_REQUIRE_TAG=true." +fi + +if [ "$PUBLISH_GITEA" = true ]; then + : "${GITEA_BASE_URL:?Set GITEA_BASE_URL}" + : "${GITEA_PACKAGE_OWNER:?Set GITEA_PACKAGE_OWNER}" + : "${GITEA_PACKAGE_USER:?Set GITEA_PACKAGE_USER}" + : "${GITEA_PACKAGE_TOKEN:?Set GITEA_PACKAGE_TOKEN in this process only}" +fi + +"$ROOT_DIR/tests/validate-repository.sh" +"$SCRIPT_DIR/ci/check-incus-runner.sh" + +install -d -m 0750 "$ROOT_DIR/build" +exec 9>"$ROOT_DIR/build/.local-release.lock" +flock -n 9 || die "Another local image release is already running." + +log "Local image release $PLATFORM_VERSION from $source_revision" +log "Variants: ${variants[*]}" + +for variant in "${variants[@]}"; do + log "Building $variant" + "$SCRIPT_DIR/build-image.sh" "$variant" + + package_dir="$ROOT_DIR/build/packages/$PLATFORM_VERSION/$variant" + "$SCRIPT_DIR/package-image.sh" "$variant" "$package_dir" + ( + cd "$package_dir" + sha256sum --check --strict SHA256SUMS + ) + + if [ "$PUBLISH_GITEA" = true ]; then + "$SCRIPT_DIR/publish-gitea-package.sh" "$variant" "$package_dir" + fi +done + +log "Local release completed" +printf 'Version: %s\nRevision: %s\nPackages: %s\n' \ + "$PLATFORM_VERSION" "$source_revision" "$ROOT_DIR/build/packages/$PLATFORM_VERSION" +if [ "$PUBLISH_GITEA" = true ]; then + printf 'Published to: %s/api/packages/%s\n' "$GITEA_BASE_URL" "$GITEA_PACKAGE_OWNER" +fi diff --git a/tests/validate-repository.sh b/tests/validate-repository.sh index 4b5ed7f..6ccff56 100755 --- a/tests/validate-repository.sh +++ b/tests/validate-repository.sh @@ -12,11 +12,17 @@ done < <(find "${find_args[@]}" -name '*.sh' -print0) if command -v shellcheck >/dev/null 2>&1; then mapfile -d '' scripts < <(find "${find_args[@]}" -name '*.sh' -print0) - if ((${#scripts[@]} > 0)); then + if (("${#scripts[@]}" > 0)); then shellcheck -x "${scripts[@]}" || fail=1 fi fi +bash "$ROOT_DIR/tests/variant-selection.sh" || fail=1 + +if command -v make >/dev/null 2>&1; then + make -C "$ROOT_DIR" --dry-run help local-release >/dev/null || fail=1 +fi + while IFS= read -r -d '' json; do jq empty "$json" || fail=1 done < <(find "${find_args[@]}" -name '*.json' -print0) diff --git a/tests/variant-selection.sh b/tests/variant-selection.sh new file mode 100644 index 0000000..795824e --- /dev/null +++ b/tests/variant-selection.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck disable=SC1091 +source "$ROOT_DIR/scripts/lib/variant-selection.sh" + +assert_selection() { + local input="$1" + local expected="$2" + local -a actual=() + + parse_variant_selection "$input" actual + [[ "${actual[*]}" == "$expected" ]] || { + printf 'Selection %q resolved to %q; expected %q.\n' "$input" "${actual[*]}" "$expected" >&2 + exit 1 + } +} + +assert_rejected() { + local input="$1" + local -a actual=() + + if parse_variant_selection "$input" actual 2>/dev/null; then + printf 'Selection %q should have been rejected.\n' "$input" >&2 + exit 1 + fi +} + +assert_selection "all" "base php python cpp typescript full" +assert_selection "php,typescript,full" "php typescript full" +assert_selection $'php typescript\nfull' "php typescript full" +assert_selection "php,php,full" "php full" +assert_rejected "" +assert_rejected "unknown" +assert_rejected "all,php" +assert_rejected "php,,unknown" + +printf 'Variant selection tests passed.\n'