diff --git a/.github/workflows/build-kernel-deb.yml b/.github/workflows/build-kernel-deb.yml index 967ca75..26a2265 100644 --- a/.github/workflows/build-kernel-deb.yml +++ b/.github/workflows/build-kernel-deb.yml @@ -3,77 +3,158 @@ name: build-kernel-deb on: workflow_dispatch: inputs: + kernel-variant: + description: 'Kernel variant identifier' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target suite' + required: true + default: 'trixie' + type: string + ref-strategy: + description: 'Kernel ref strategy' + required: true + default: 'latest_tag' + type: choice + options: + - latest_tag + - branch_tip + - pinned_ref kernel-branch: - description: 'Kernel branch or tag to build' + description: 'Branch for branch_tip or immutable ref for pinned_ref (ignored for latest_tag)' required: false default: 'qcom-next' + tag-pattern: + description: 'Tag glob used only by latest_tag (ignored for branch_tip and pinned_ref)' + required: false + default: 'qcom-next-*' kernel-url: - description: 'Custom kernel repo URL (leave empty for qualcomm-linux/kernel)' + description: 'Advanced: custom kernel repository URL (empty uses qualcomm-linux/kernel)' required: false default: '' - latest-tag: - description: 'Use latest qcom-next-* tag (ignores kernel-branch)' - type: boolean - default: true - debug-build: - description: 'Enable debug build' - type: boolean + srcpkg: + description: 'Advanced: Debian source package name' required: false - default: false + default: 'linux-qcom-next' + binpkg: + description: 'Advanced: kernel image metapackage name' + required: false + default: 'linux-image-qcom-next' + kernel-config: + description: 'Advanced: comma-separated config fragments from debian/config-available/' + required: false + default: 'squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can' + debian-version-stub: + description: 'Advanced: Debian version stub; the selected suite''s mapped suffix and a Daily-style trailing ~ are applied automatically' + required: false + default: '0qli+staging1' localversion: - description: 'LOCALVERSION suffix (e.g. qcom-next-20260312). Omit if the tree provides it via localversion* files.' + description: 'Advanced: LOCALVERSION override (auto-derived from the resolved ref if empty)' required: false default: '' kver-extra: - description: 'Extra suffix appended to the package version (e.g. -ci42)' + description: 'Advanced: extra package-version suffix (e.g. -ci42)' required: false default: '' + debug-build: + description: 'Advanced: enable the debug build configuration' + type: boolean + required: false + default: false pkg-linux-qcom-ref: - description: pkg-linux-qcom ref branch or commit + description: 'Advanced: pkg-linux-qcom packaging ref (branch or commit)' required: false default: 'qcom/debian/latest' self-pr: - description: 'pkg-linux-qcom PR number to apply before building (e.g. 42)' + description: 'Advanced: pkg-linux-qcom PR number to apply before building (e.g. 42)' required: false default: '' - distro: - description: 'Target distro' - required: false - default: 'trixie' - type: choice - options: - - trixie - - noble - - questing - - resolute - - sid qcom-next-pr: - description: 'Space-separated PR numbers to merge from qcom-next (e.g. "42 43")' + description: 'Advanced Qualcomm-only override: qcom-next PR numbers to merge (e.g. "42 43")' type: string required: false default: '' kernel-topics-pr: - description: 'Space-separated PR numbers from kernel-topics to apply as patches (e.g. "7 8")' + description: 'Advanced Qualcomm-only override: kernel-topics PR numbers to apply as patches' type: string required: false default: '' - # Daily builds are driven by daily.yml which reads ci/build-matrix.json - # and calls this workflow as a reusable workflow for each matrix entry. + # Called by daily.yml and release.yml as a reusable workflow. workflow_call: inputs: - distro: - description: Target distro (trixie, noble, questing, resolute, sid) + kernel-variant: + description: 'Kernel variant identifier' + type: string + default: 'qcom-next' + suite: + description: 'Target suite (trixie, forky, noble, questing, resolute, sid)' type: string default: 'trixie' - latest-tag: - description: Use latest qcom-next-* tag - type: boolean - default: true + kernel-branch: + description: 'Source branch or pinned tag used by branch_tip or pinned_ref' + type: string + default: 'qcom-next' + git-clone: + description: 'Kernel repository URL' + type: string + default: 'https://github.com/qualcomm-linux/kernel' + ref-strategy: + description: 'Kernel ref strategy: latest_tag, branch_tip, or pinned_ref' + type: string + default: 'latest_tag' + tag-pattern: + description: 'Tag glob used when ref-strategy=latest_tag' + type: string + default: 'qcom-next-*' + srcpkg: + description: 'Source package name' + type: string + default: 'linux-qcom-next' + binpkg: + description: 'Binary metapackage name' + type: string + default: 'linux-image-qcom-next' + kernel-config: + description: 'Comma-separated config fragments to activate from debian/config-available/' + type: string + default: 'squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can' + debian-revision: + description: 'Debian revision component of the package version' + type: string + default: '0qli+staging1~' + localversion: + description: 'Override LOCALVERSION suffix (auto-derived from the resolved ref if empty)' + type: string + default: '' + kver-extra: + description: 'Extra suffix appended to the package version' + type: string + default: '' + debusine-parent-workspace: + description: 'Parent Debusine workspace for CI child workspace creation' + type: string + default: '' + target-workspace: + description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' + type: string + default: '' pkg-linux-qcom-ref: - description: pkg-linux-qcom ref branch or commit + description: 'pkg-linux-qcom packaging ref' type: string default: 'qcom/debian/latest' + secrets: + DEBUSINE_USER: + description: 'Debusine account used to submit CI and release builds' + required: false + DEBUSINE_TOKEN: + description: 'Debusine token for the CI child workspace' + required: false + DEBUSINE_RELEASE_TOKEN: + description: 'Debusine token for package-publish promotion to the release target workspace' + required: false permissions: contents: read @@ -82,9 +163,6 @@ permissions: jobs: # --------------------------------------------------------------------------- # resolve: classify the target suite as 'debian' or 'ubuntu'. - # - # Debian suites (trixie, sid, unstable, bookworm, forky) → Debusine. - # Ubuntu suites (noble, questing, resolute) → docker build on self-hosted runner. # --------------------------------------------------------------------------- resolve: name: Resolve suite family @@ -95,10 +173,18 @@ jobs: - name: Classify suite id: classify shell: bash + env: + SUITE_INPUT: ${{ inputs.suite || 'trixie' }} + KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} run: | - suite="${{ inputs.distro || 'trixie' }}" + suite="$SUITE_INPUT" + kernel_variant="$KERNEL_VARIANT_INPUT" + [[ "$kernel_variant" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: kernel variant must use lowercase letters, digits, and internal hyphens" >&2 + exit 1 + } case "$suite" in - trixie|sid|unstable|bookworm|forky) + trixie|forky|sid|unstable|bookworm) family=debian ;; *) family=ubuntu ;; @@ -106,17 +192,22 @@ jobs: echo "family=$family" >> "$GITHUB_OUTPUT" echo "Suite '$suite' classified as family=$family" + # --------------------------------------------------------------------------- + # prepare: clone kernel source, inject packaging, run prepare-source.sh. + # --------------------------------------------------------------------------- prepare: name: Prepare kernel source runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] env: - KERNEL_BRANCH: ${{ inputs.kernel-branch || 'qcom-next' }} - KERNEL_URL: ${{ inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} - DISTRO: ${{ inputs.distro || 'trixie' }} + KERNEL_URL: ${{ inputs.git-clone || inputs.kernel-url || 'https://github.com/qualcomm-linux/kernel' }} + DISTRO: ${{ inputs.suite || 'trixie' }} + # For Debian-family suites (trixie, forky, sid) use trixie as the docker + # image distro: docker-pkg-build and pkg-builder only support trixie for + # Debian. Debusine handles the actual suite-specific build internally. + # For Ubuntu-family suites (noble, questing, resolute) use the real suite. + DOCKER_DISTRO: ${{ (inputs.suite == 'forky' || inputs.suite == 'sid' || inputs.suite == 'unstable') && 'trixie' || inputs.suite || 'trixie' }} - # bash with -e -o pipefail for every run step (parity with the family - # modules; guarantees pipefail for the tag-resolution pipeline below). defaults: run: shell: bash @@ -127,27 +218,41 @@ jobs: with: ref: ${{ inputs.pkg-linux-qcom-ref || 'qcom/debian/latest' }} + - name: Checkout CI scripts + # ci/scripts/ and ci/build-matrix.json live on the CI branch (main), + # not on the packaging branch (qcom/debian/latest). Check out the CI + # branch into a separate path so resolve-kernel-ref.sh, + # derive-localversion.sh, derive-debian-revision.sh, and the + # suite_suffix_mapping are available. + uses: actions/checkout@v4 + with: + persist-credentials: false + ref: ${{ github.sha }} + path: .ci-branch + sparse-checkout: | + ci/scripts + ci/build-matrix.json + fetch-depth: 1 + + - name: Install CI scripts into workspace + run: | + mkdir -p ci/scripts + cp .ci-branch/ci/scripts/*.sh ci/scripts/ + chmod +x ci/scripts/*.sh + cp .ci-branch/ci/build-matrix.json ci/build-matrix.json + rm -rf .ci-branch + - name: Apply packaging PR (if specified) if: ${{ inputs.self-pr != '' }} env: GH_TOKEN: ${{ github.token }} SELF_PR: ${{ inputs.self-pr }} run: | - # Dereference the input via env and validate it is numeric before it - # reaches a git refspec, rather than splicing the raw input into shell. [[ "$SELF_PR" =~ ^[0-9]+$ ]] || { echo "ERROR: self-pr must be numeric (got '$SELF_PR')"; exit 1; } echo "Applying pkg-linux-qcom PR #$SELF_PR..." git fetch origin "pull/$SELF_PR/head:pr-$SELF_PR" git checkout "pr-$SELF_PR" - - name: Extract org and repo name - run: | - # split GITHUB_REPOSITORY (org/repo) - echo "ORG_NAME=${GITHUB_REPOSITORY%%/*}" >> "$GITHUB_ENV" - echo "REPO_NAME=${GITHUB_REPOSITORY#*/}" >> "$GITHUB_ENV" - - # Pinned to @main while native kernel-build support is still landing - # upstream in qualcomm-linux/docker-pkg-build. - name: Checkout docker-pkg-build uses: actions/checkout@v4 with: @@ -157,59 +262,97 @@ jobs: - name: Build docker image run: | - ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DISTRO" + ./docker-pkg-build/docker_deb_build.py --rebuild -d "$DOCKER_DISTRO" docker image ls - - name: Sync kernel source + - name: Resolve kernel ref + id: kernel-ref + env: + BRANCH_INPUT: ${{ inputs.kernel-branch || 'qcom-next' }} + REF_STRATEGY_INPUT: ${{ inputs.ref-strategy || 'latest_tag' }} + TAG_PATTERN_INPUT: ${{ inputs.tag-pattern || 'qcom-next-*' }} + KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} run: | - KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" - echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" + set -euo pipefail + BRANCH="$BRANCH_INPUT" + REF_STRATEGY="$REF_STRATEGY_INPUT" + TAG_PATTERN="$TAG_PATTERN_INPUT" - # Resolve the latest qcom-next-* tag when explicitly requested or on - # scheduled runs (inputs are not available for schedule events). - # Pick by the trailing YYYYMMDD date snapshot, not by `sort -V`: version - # sort keys on the base kernel version first, so an older-dated rc would - # outrank a newer final (and rc sorts after its final in -V). Keying on - # the date makes the newest snapshot win across the whole tag namespace. - if [[ "${{ inputs.latest-tag }}" == "true" || "${{ github.event_name }}" == "schedule" ]]; then - KERNEL_REF=$(git ls-remote --tags "$KERNEL_URL" 'refs/tags/qcom-next-*' \ - | awk '{print $2}' | sed 's|refs/tags/||' | grep -v '\^{}' \ - | grep -E -- '-[0-9]{8}$' \ - | awk -F- '{print $NF"\t"$0}' | sort -k1,1n | tail -1 | cut -f2-) - [[ -n "$KERNEL_REF" ]] || { echo "ERROR: No qcom-next-*-YYYYMMDD tags found in $KERNEL_URL"; exit 1; } - echo "Resolved latest tag: $KERNEL_REF" - else - KERNEL_REF="$KERNEL_BRANCH" - fi + case "$REF_STRATEGY" in + latest_tag) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh \ + --url "$KERNEL_URL" \ + --latest-tag "$TAG_PATTERN") + ;; + branch_tip|pinned_ref) + KERNEL_REF=$(ci/scripts/resolve-kernel-ref.sh --url "$KERNEL_URL" --ref "$BRANCH") + ;; + *) + echo "ERROR: unsupported ref strategy: $REF_STRATEGY" >&2 + exit 1 + ;; + esac echo "KERNEL_REF=$KERNEL_REF" >> "$GITHUB_ENV" - echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." - git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags "$KERNEL_URL" "$KERNEL_DIR" + echo "Kernel variant: $KERNEL_VARIANT_INPUT" + echo "Ref strategy: $REF_STRATEGY" + echo "Resolved kernel ref: $KERNEL_REF" + - name: Clone kernel source + run: | + KERNEL_DIR="$GITHUB_WORKSPACE/kernel-source" + echo "KERNEL_DIR=$KERNEL_DIR" >> "$GITHUB_ENV" + echo "Cloning $KERNEL_URL @ $KERNEL_REF (shallow)..." + git clone --depth 1 --single-branch --branch "$KERNEL_REF" --no-tags \ + "$KERNEL_URL" "$KERNEL_DIR" KERNEL_SHA=$(git -C "$KERNEL_DIR" rev-parse HEAD) echo "KERNEL_SHA=$KERNEL_SHA" >> "$GITHUB_ENV" echo "Kernel HEAD SHA: $KERNEL_SHA" - - name: Resolve LOCALVERSION from kernel ref + - name: Derive LOCALVERSION + env: + LOCALVERSION_INPUT: ${{ inputs.localversion }} + KERNEL_VARIANT_INPUT: ${{ inputs.kernel-variant || 'qcom-next' }} run: | - # Derive LOCALVERSION from KERNEL_REF (already resolved above), so no - # git describe is needed inside the container: - # tag build: qcom-next-7.0-rc7-20260429 -> qcom-next-20260429 - # (prefix + date; the version is already in KVER) - # branch-tip build: qcom-next -> qcom-next-g - # An explicit localversion input wins (applied in Prepare source below). - LOCALVERSION=$(echo "$KERNEL_REF" | sed -E 's/^([a-z-]+)-[0-9]+\.[0-9]+.*-([0-9]+)$/\1-\2/') - if [[ "$LOCALVERSION" != "$KERNEL_REF" ]]; then - # Tag build: LOCALVERSION derived from tag name - echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Resolved LOCALVERSION='$LOCALVERSION' from tag '$KERNEL_REF'" + set -euo pipefail + OVERRIDE_LV="$LOCALVERSION_INPUT" + if [[ -n "$OVERRIDE_LV" ]]; then + echo "Using explicit LOCALVERSION override: $OVERRIDE_LV" + echo "LOCALVERSION=$OVERRIDE_LV" >> "$GITHUB_ENV" else - # Branch-tip build: use branch name + short SHA for uniqueness - SHORT_SHA=$(echo "$KERNEL_SHA" | cut -c1-12) - LOCALVERSION="${KERNEL_REF}-g${SHORT_SHA}" + LOCALVERSION=$(ci/scripts/derive-localversion.sh \ + --variant "$KERNEL_VARIANT_INPUT" \ + --ref "$KERNEL_REF" \ + --sha "$(echo "$KERNEL_SHA" | cut -c1-12)") echo "LOCALVERSION=$LOCALVERSION" >> "$GITHUB_ENV" - echo "Resolved LOCALVERSION='$LOCALVERSION' from branch tip '$KERNEL_REF' @ $SHORT_SHA" + echo "Derived LOCALVERSION: $LOCALVERSION" + fi + + - name: Resolve Debian revision + # workflow_call callers (daily.yml, release.yml) pass an already + # fully-derived debian-revision from resolve-matrix.sh. Direct + # workflow_dispatch runs pass only debian-version-stub and get the + # selected suite's mapped suffix applied automatically, always using + # Daily semantics since direct dispatch is build-only and + # non-promoting. + env: + DEBIAN_REVISION_INPUT: ${{ inputs.debian-revision }} + DEBIAN_VERSION_STUB_INPUT: ${{ inputs.debian-version-stub || '0qli+staging1' }} + run: | + set -euo pipefail + INPUT_REVISION="$DEBIAN_REVISION_INPUT" + if [[ -n "$INPUT_REVISION" ]]; then + DEBIAN_REVISION="$INPUT_REVISION" + echo "Using matrix-derived Debian revision: $DEBIAN_REVISION" + else + DEBIAN_REVISION=$(ci/scripts/derive-debian-revision.sh \ + --stub "$DEBIAN_VERSION_STUB_INPUT" \ + --suite "$DISTRO" \ + --delivery-type Daily \ + --matrix-file ci/build-matrix.json) + echo "Derived Debian revision for direct dispatch: $DEBIAN_REVISION" fi + echo "DEBIAN_REVISION=$DEBIAN_REVISION" >> "$GITHUB_ENV" - name: Merge qcom-next PR patches if: ${{ inputs.qcom-next-pr != '' }} @@ -222,16 +365,11 @@ jobs: [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid qcom-next PR number: $pr"; exit 1; } echo "::group::Merging qcom-next PR #$pr" git fetch --no-tags origin "pull/$pr/head:pr-$pr" - - # --no-ff so the --no-commit semantics hold even for a fast-forwardable - # PR; otherwise git would fast-forward HEAD and the branch below would - # mislabel a real merge as "nothing to merge". if ! git merge --no-ff --no-commit "pr-$pr"; then echo "ERROR: Merge conflict while merging PR #$pr. Aborting." git merge --abort || true exit 1 fi - if ! git diff --cached --quiet; then git commit -m "Merged qcom-next PR #$pr" echo "PR #$pr merged successfully." @@ -252,7 +390,6 @@ jobs: [[ "$pr" =~ ^[0-9]+$ ]] || { echo "::error::invalid kernel-topics PR number: $pr"; exit 1; } echo "::group::Applying kernel-topics PR #$pr" wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - if ! git am "$pr.patch"; then echo "ERROR: Patch application failed for PR #$pr. Aborting." git am --abort || true @@ -263,32 +400,36 @@ jobs: done - name: Prepare source + env: + SRCPKG_INPUT: ${{ inputs.srcpkg || 'linux-qcom-next' }} + BINPKG_INPUT: ${{ inputs.binpkg || 'linux-image-qcom-next' }} + KVER_EXTRA_INPUT: ${{ inputs.kver-extra }} + KERNEL_CONFIG_INPUT: ${{ inputs.kernel-config || 'squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can' }} + DEBUG_BUILD_INPUT: ${{ inputs.debug-build }} run: | - # Inject debian/, activate config fragments, and run 'debian/rules - # prepare' (generates debian/control + debian/changelog from *.in), - # inside the distro-matched pkg-builder container for reproducibility. - ARGS=(--source-dir "$KERNEL_DIR" --distro "$DISTRO") + # No -x here: keep xtrace off for this step as it assembles the + # prepare-source.sh argument list which may include sensitive paths. + set -euo pipefail - # Prefer an explicit localversion input; else the LOCALVERSION derived - # above. Passing it explicitly avoids git describe (and safe.directory - # issues) inside the container. - EFFECTIVE_LOCALVERSION="${{ inputs.localversion }}" - [[ -z "$EFFECTIVE_LOCALVERSION" && -n "$LOCALVERSION" ]] && EFFECTIVE_LOCALVERSION="$LOCALVERSION" - [[ -n "$EFFECTIVE_LOCALVERSION" ]] && ARGS+=(--localversion "$EFFECTIVE_LOCALVERSION") - [[ -n "${{ inputs.kver-extra }}" ]] && ARGS+=(--kver-extra "${{ inputs.kver-extra }}") + ARGS=( + --source-dir "$KERNEL_DIR" + --distro "$DISTRO" + --srcpkg "$SRCPKG_INPUT" + --binpkg "$BINPKG_INPUT" + --debian-revision "$DEBIAN_REVISION" + ) - # --debug makes prepare-source.sh add arch/arm64/configs/debug.config - # as a standard config fragment. - [[ "${{ inputs.debug-build }}" == "true" ]] && ARGS+=(--debug) + [[ -n "$LOCALVERSION" ]] && ARGS+=(--localversion "$LOCALVERSION") + [[ -n "$KVER_EXTRA_INPUT" ]] && ARGS+=(--kver-extra "$KVER_EXTRA_INPUT") + [[ -n "$KERNEL_CONFIG_INPUT" ]] && ARGS+=(--kernel-config "$KERNEL_CONFIG_INPUT") + [[ "$DEBUG_BUILD_INPUT" == "true" ]] && ARGS+=(--debug) docker run -i --rm \ -v "$PWD:$PWD" --workdir="$PWD" \ - "ghcr.io/qualcomm-linux/pkg-builder:$DISTRO" \ + "ghcr.io/qualcomm-linux/pkg-builder:$DOCKER_DISTRO" \ ./prepare-source.sh "${ARGS[@]}" - name: Upload prepared source tree - # Archive as tar.gz so upload-artifact's zip does not strip the execute - # bits on kernel build scripts (see README). run: | tar czf /tmp/kernel-srcpkg.tar.gz \ -C "$(dirname "$KERNEL_DIR")" \ @@ -297,17 +438,13 @@ jobs: - name: Upload prepared source tree artifact uses: actions/upload-artifact@v4 with: - name: kernel-srcpkg + name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} path: /tmp/kernel-srcpkg.tar.gz if-no-files-found: error retention-days: 7 # --------------------------------------------------------------------------- - # debusine-build: Debian suites only (trixie, sid). - # - # Delegates to build-kernel-debusine.yml which owns the complete Debian build - # path end-to-end: source package generation, Debusine submission, and - # publishing the built .deb files to S3. + # debusine-build: Debian suites only (trixie, forky, sid). # --------------------------------------------------------------------------- debusine-build: name: Build in Debusine @@ -315,20 +452,18 @@ jobs: if: ${{ needs.resolve.outputs.family == 'debian' }} uses: ./.github/workflows/build-kernel-debusine.yml with: - suite: ${{ inputs.distro || 'trixie' }} - debusine-parent-workspace: ${{ vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + suite: ${{ inputs.suite || 'trixie' }} + kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} + srcpkg: ${{ inputs.srcpkg || 'linux-qcom-next' }} + debusine-parent-workspace: ${{ inputs.debusine-parent-workspace || vars.DEBUSINE_PARENT_WORKSPACE || 'qli-ci' }} + target-workspace: ${{ inputs.target-workspace || '' }} secrets: DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} # --------------------------------------------------------------------------- # ubuntu-build: Ubuntu suites only (noble, questing, resolute). - # - # Delegates to build-kernel-ubuntu.yml, which encapsulates the full - # build-kernel.sh pipeline. Keeping this as a separate reusable workflow - # means the Ubuntu build path can be deprecated or replaced (e.g. when - # Debusine gains Ubuntu support) by deleting one file and one job, no - # surgery on the orchestrator logic below. # --------------------------------------------------------------------------- ubuntu-build: name: Build (Ubuntu, docker) @@ -336,5 +471,6 @@ jobs: if: ${{ needs.resolve.outputs.family == 'ubuntu' }} uses: ./.github/workflows/build-kernel-ubuntu.yml with: - distro: ${{ inputs.distro || 'resolute' }} + distro: ${{ inputs.suite || 'resolute' }} + kernel-variant: ${{ inputs.kernel-variant || 'qcom-next' }} pkg-linux-qcom-ref: ${{ inputs.pkg-linux-qcom-ref || 'qcom/debian/latest' }} diff --git a/.github/workflows/build-kernel-debusine.yml b/.github/workflows/build-kernel-debusine.yml index 9483d9b..42eebe6 100644 --- a/.github/workflows/build-kernel-debusine.yml +++ b/.github/workflows/build-kernel-debusine.yml @@ -1,38 +1,52 @@ name: build-debusine -# Reusable workflow: Debian kernel build via Debusine, then publish to S3. -# The Debian counterpart of build-kernel-ubuntu.yml; called by -# build-kernel-deb.yml for Debian-family suites (trixie, sid). The kernel -# source is already prepared by the caller's prepare job and shared as the -# kernel-srcpkg artifact (via github.run_id). +# Reusable workflow: Debian kernel build via Debusine, then publish. # -# build job (debusine-pkg-builder container): assembles the Debian source -# package and submits it to Debusine for a distributed build. -# Uses DEBUSINE_ASSEMBLE_ORIG=true to bypass gbp and build the -# .orig.tar.gz directly, since this tree has no upstream tag, -# pristine-tar branch, or gbp.conf (it is assembled at CI time). -# publish job (self-hosted runner, for direct IAM): downloads the built .deb -# files from the private Debusine workspace with chdist (hermetic, -# authenticated apt, no host pollution) and uploads them to S3. +# Called by build-kernel-deb.yml for Debian-family suites (trixie, forky, sid). +# The kernel source is already prepared by the caller's prepare job and shared +# as the kernel-srcpkg artifact. Artifact and workspace identity include both +# kernel_variant and suite so parallel variants cannot share inputs or outputs. # -# To retire this path, delete this file and its job in build-kernel-deb.yml. +# Publish path is determined by target-workspace: +# target-workspace == '' -> daily path: download .deb files, upload to S3. +# target-workspace != '' -> release path: promote from CI workspace to the +# target Debusine workspace via debusine-action lib/release. +# +# Jobs: +# build (debusine-pkg-builder container): generates .dsc, submits to Debusine. +# publish (self-hosted runner): S3 upload (daily path only). +# release (debusine-pkg-builder container): Debusine promotion (release path only). on: workflow_call: inputs: suite: - description: Target Debian suite (trixie, sid) + description: 'Target Debian suite (trixie, forky, sid)' + type: string + required: true + kernel-variant: + description: 'Kernel variant identifier used to isolate artifacts and Debusine workspaces' type: string required: true + srcpkg: + description: 'Source package name (e.g. linux-qcom-next)' + type: string + default: 'linux-qcom-next' debusine-parent-workspace: - description: Parent Debusine workspace for child CI workspace creation + description: 'Parent Debusine workspace for CI child workspace creation' type: string default: 'qli-ci' + target-workspace: + description: 'Debusine target workspace for release promotion (empty = daily/S3 path)' + type: string + default: '' secrets: DEBUSINE_USER: required: true DEBUSINE_TOKEN: required: true + DEBUSINE_RELEASE_TOKEN: + required: false permissions: contents: read @@ -44,16 +58,12 @@ env: jobs: # --------------------------------------------------------------------------- # build: generate Debian source package and submit to Debusine. - # - # Runs inside the debusine-pkg-builder container which has all required - # tooling pre-installed: debusine-client, python3-debusine, yq, devscripts, - # dctrl-tools, git-buildpackage, dpkg-dev, git, pristine-tar. # --------------------------------------------------------------------------- build: name: Build (Debusine) runs-on: ubuntu-latest container: - image: ghcr.io/qualcomm-linux/debusine-pkg-builder:${{ inputs.suite }} + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie options: --user 0:0 credentials: username: ${{ github.actor }} @@ -61,9 +71,11 @@ jobs: outputs: workspace: ${{ steps.build-debusine.outputs.workspace }} workspace_url: ${{ steps.build-debusine.outputs.workspace_url }} + srcpkg_version: ${{ steps.generate-srcpkg.outputs.srcpkg_version }} defaults: run: shell: bash + environment: Staging steps: - name: Checkout debusine-action helpers uses: actions/checkout@v5 @@ -78,29 +90,21 @@ jobs: - name: Download prepared source tree uses: actions/download-artifact@v4 with: - name: kernel-srcpkg + name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.suite }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree run: | - # tar.gz preserves the execute bits that upload-artifact's zip would - # strip (see README); --strip-components=1 drops the archive's top dir. mkdir srcpkg tar xzf /tmp/srcpkg-artifact/*.tar.gz \ -C srcpkg --strip-components=1 - name: Generate source package + id: generate-srcpkg env: SUITE: ${{ inputs.suite }} DEBUSINE_ASSEMBLE_ORIG: "true" run: | - # DEBUSINE_ASSEMBLE_ORIG=true: bypasses gbp entirely. Creates the - # .orig.tar.gz directly from the assembled source tree (excluding - # debian/ and .git/) and invokes dpkg-buildpackage -S to produce - # the .dsc, .debian.tar.xz, and .changes files. - # Required because pkg-linux-qcom has no upstream tag, no - # pristine-tar branch, and no debian/gbp.conf; the source tree - # is assembled at CI time from separate kernel and packaging repos. SUITE="$SUITE" debusine-action/lib/generate-source-package - name: Build in Debusine @@ -109,7 +113,7 @@ jobs: GITHUB_REPOSITORY_ID: ${{ github.repository_id }} GITHUB_RUN_ID: ${{ github.run_id }} GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - JOB_INDEX: "0" + JOB_INDEX: ${{ inputs.kernel-variant }}-${{ inputs.suite }} DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} @@ -117,32 +121,26 @@ jobs: DEBUSINE_PARENT_WORKSPACE: ${{ inputs.debusine-parent-workspace }} SUITE: ${{ inputs.suite }} run: | - # No -x here: this step has DEBUSINE_TOKEN in its env, so keep xtrace - # off so a future edit cannot echo a secret onto the trace. + # No -x here: DEBUSINE_TOKEN is in env, keep xtrace off. set -euo pipefail debusine-action/lib/build - # debusine-action/lib/build writes workspace/workflow_id directly to - # $GITHUB_OUTPUT (it no longer produces a separate 'output' file), so - # read the workspace back out of there to derive workspace_url. workspace=$(sed -n 's/^workspace=//p' "$GITHUB_OUTPUT") echo "workspace_url=https://${DEBUSINE_HOST}/${DEBUSINE_SCOPE}/${workspace}/" >> "$GITHUB_OUTPUT" - name: Note Debusine workspace URL + env: + WORKSPACE_URL: ${{ steps.build-debusine.outputs.workspace_url }} run: | - echo "Debusine Workspace URL: ${{ steps.build-debusine.outputs.workspace_url }}" >> "$GITHUB_STEP_SUMMARY" + echo "Debusine Workspace URL: $WORKSPACE_URL" >> "$GITHUB_STEP_SUMMARY" # --------------------------------------------------------------------------- - # publish: download built .deb files from Debusine and upload to S3. - # - # Runs on the self-hosted runner (not in a container) because the S3 upload - # action requires direct IAM credentials available on the runner. chdist - # creates a hermetic, authenticated apt environment pointing exclusively at - # the Debusine workspace (no host pollution, no package installation). + # publish: download .deb files from Debusine and upload to S3. + # Daily path only (target-workspace is empty). # --------------------------------------------------------------------------- publish: - name: Publish Debusine artifacts + name: Publish to S3 needs: build - if: ${{ needs.build.result == 'success' }} + if: ${{ needs.build.result == 'success' && inputs.target-workspace == '' }} runs-on: [self-hosted, lecore-prd-u2404-arm64-xlrg-od-ephem] defaults: run: @@ -167,22 +165,13 @@ jobs: DEBUSINE_WORKSPACE: ${{ needs.build.outputs.workspace }} SUITE: ${{ inputs.suite }} run: | - # Generates debusine-ci.sources (DEB822 apt source) and - # debusine-ci-auth.conf (netrc credentials) for the workspace. debusine-action/lib/generate-apt-config - name: Download .deb packages from Debusine workspace run: | set -euxo pipefail - - # chdist (from devscripts) creates a hermetic apt environment - # isolated from the host, authenticated against the private Debusine - # workspace. apt-get download fetches .deb files without installing - # them or resolving dependencies against the host. sudo apt-get install -y --no-install-recommends devscripts - mkdir -p "$GITHUB_WORKSPACE/deb-artifacts" - chdist create debusine-workspace rm -f ~/.chdist/debusine-workspace/etc/apt/sources.list install -d ~/.chdist/debusine-workspace/etc/apt/sources.list.d @@ -191,15 +180,11 @@ jobs: ~/.chdist/debusine-workspace/etc/apt/sources.list.d/ install -m 0600 debusine-ci-auth.conf \ ~/.chdist/debusine-workspace/etc/apt/auth.conf.d/ - chdist apt-get debusine-workspace update - packages=$(chdist apt-cache debusine-workspace search . | awk '{print $1}' | tr '\n' ' ') - [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace (auth or update issue?)"; exit 1; } + [[ -n "$packages" ]] || { echo "ERROR: no packages found in Debusine workspace"; exit 1; } echo "Packages to download: $packages" - cd "$GITHUB_WORKSPACE/deb-artifacts" - # word-split is intentional: each package name is a separate argument. # shellcheck disable=SC2086 chdist apt-get debusine-workspace download $packages @@ -208,4 +193,49 @@ jobs: with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: deb-artifacts - destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + destination: ${{ github.repository_owner }}/pkg/debusine/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.suite }}/${{ github.run_id }}-${{ github.run_attempt }}/ + + # --------------------------------------------------------------------------- + # release: promote packages from CI workspace to target Debusine workspace. + # Release path only (target-workspace is non-empty). + # --------------------------------------------------------------------------- + release: + name: Release to Debusine + needs: build + if: ${{ needs.build.result == 'success' && inputs.target-workspace != '' }} + runs-on: ubuntu-latest + environment: Staging + container: + image: ghcr.io/qualcomm-linux/debusine-pkg-builder:trixie + options: --user 0:0 + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + defaults: + run: + shell: bash + steps: + - name: Checkout debusine-action helpers + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + persist-credentials: false + repository: qualcomm-linux/debusine-action + ref: ${{ env.DEBUSINE_ACTION_REF }} + path: debusine-action + fetch-depth: 1 + sparse-checkout: | + lib + + - name: Promote packages to target workspace + env: + DEBUSINE_HOST: ${{ vars.DEBUSINE_HOST }} + DEBUSINE_SCOPE: ${{ vars.DEBUSINE_SCOPE }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} + DEBUSINE_CI_WORKSPACE: ${{ needs.build.outputs.workspace }} + DEBUSINE_TARGET_WORKSPACE: ${{ inputs.target-workspace }} + SRCPKG_NAME: ${{ inputs.srcpkg }} + SRCPKG_VERSION: ${{ needs.build.outputs.srcpkg_version }} + SUITE: ${{ inputs.suite }} + run: | + set -ex + debusine-action/lib/release diff --git a/.github/workflows/build-kernel-ubuntu.yml b/.github/workflows/build-kernel-ubuntu.yml index 105da94..c4bbcda 100644 --- a/.github/workflows/build-kernel-ubuntu.yml +++ b/.github/workflows/build-kernel-ubuntu.yml @@ -19,6 +19,10 @@ on: description: Target Ubuntu suite (noble, questing, resolute) type: string required: true + kernel-variant: + description: 'Kernel variant identifier used to isolate prepared-source artifacts and published packages' + type: string + required: true pkg-linux-qcom-ref: description: | pkg-linux-qcom ref to check out. Provides build-kernel.sh and the @@ -67,7 +71,7 @@ jobs: - name: Download prepared source tree uses: actions/download-artifact@v4 with: - name: kernel-srcpkg + name: kernel-srcpkg-${{ inputs.kernel-variant }}-${{ inputs.distro }} path: /tmp/srcpkg-artifact - name: Extract prepared source tree @@ -96,11 +100,11 @@ jobs: - name: Upload .deb packages to S3 # build-kernel.sh outputs to kernel-build// by default. - # Upload from that subdirectory directly so the suite name does not - # appear in the S3 path; downstream consumers (e.g. qcom-distro-images) - # expect the flat pkg/temp//-/ layout. + # Keep kernel variant and suite in the destination so concurrent matrix + # legs cannot overwrite or mix package outputs. Consumers must select + # the variant and suite they intend to install. uses: qualcomm-linux/upload-private-artifact-action@aws-v4 with: s3_bucket: ${{ vars.ARTIFACT_S3_BUCKET }} path: kernel-build/${{ inputs.distro }} - destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ github.run_id }}-${{ github.run_attempt }}/ + destination: ${{ github.repository_owner }}/pkg/temp/${{ github.event.repository.name }}/${{ inputs.kernel-variant }}/${{ inputs.distro }}/${{ github.run_id }}-${{ github.run_attempt }}/ diff --git a/.github/workflows/build-kernel.yml b/.github/workflows/build-kernel.yml index 60c0885..aa37b23 100644 --- a/.github/workflows/build-kernel.yml +++ b/.github/workflows/build-kernel.yml @@ -82,6 +82,8 @@ jobs: # Runs if kernel-url is EMPTY - name: Sync kernel codebase (Default) if: ${{ inputs.kernel-url == '' }} + env: + PAT: ${{ secrets.PAT }} run: | cd qcom-build-utils cd kernel && export BUILD_TOP=`pwd` @@ -91,11 +93,11 @@ jobs: echo "kernel_repo=$REPO" >> "$GITHUB_ENV" echo "Syncing ${REPO} ref ${KERNEL_BRANCH}" - git ls-remote --exit-code --heads https://${{ secrets.PAT }}@github.com/${REPO}.git ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false + git ls-remote --exit-code --heads "https://${PAT}@github.com/${REPO}.git" ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false git init $BUILD_TOP/qcom-kernel cd $BUILD_TOP/qcom-kernel - git remote add origin https://${{ secrets.PAT }}@github.com/${REPO}.git + git remote add origin "https://${PAT}@github.com/${REPO}.git" if [ "$IS_BRANCH" = true ]; then echo "Detected branch: ${KERNEL_BRANCH}" @@ -114,13 +116,15 @@ jobs: # Runs if kernel-url is PROVIDED - name: Sync custom kernel codebase if: ${{ inputs.kernel-url != '' }} + env: + KERNEL_URL_INPUT: ${{ inputs.kernel-url }} run: | cd qcom-build-utils mkdir -p kernel cd kernel && export BUILD_TOP=`pwd` echo "build_top=$BUILD_TOP" >> "$GITHUB_ENV" - KERNEL_URL="${{ inputs.kernel-url }}" + KERNEL_URL="$KERNEL_URL_INPUT" echo "Syncing custom repo: ${KERNEL_URL} ref ${KERNEL_BRANCH}" git ls-remote --exit-code --heads ${KERNEL_URL} ${KERNEL_BRANCH} && IS_BRANCH=true || IS_BRANCH=false @@ -145,18 +149,21 @@ jobs: - name: Merge qcom-next PR patches if: ${{ inputs.kernel-url == '' }} + env: + QCOM_NEXT_PR_INPUT: ${{ github.event.inputs.qcom-next-pr }} + PAT: ${{ secrets.PAT }} run: | - cd "${{ env.build_top }}/" - if [ -z "${{ github.event.inputs.qcom-next-pr }}" ]; then + cd "$build_top/" + if [ -z "$QCOM_NEXT_PR_INPUT" ]; then echo "No PR number specified, skipping merge." else - echo "Merging PR(s) #${{ github.event.inputs.qcom-next-pr }}" + echo "Merging PR(s) #$QCOM_NEXT_PR_INPUT" - cd "${{ env.build_top }}/qcom-kernel" + cd "$build_top/qcom-kernel" - for pr in ${{ github.event.inputs.qcom-next-pr }}; do + for pr in $QCOM_NEXT_PR_INPUT; do echo "::group::Merging qcom-next PR #$pr" - git fetch "https://${{ secrets.PAT }}@github.com/${{ env.kernel_repo }}.git" "pull/$pr/head:pr-$pr" + git fetch "https://${PAT}@github.com/${kernel_repo}.git" "pull/$pr/head:pr-$pr" if ! git merge "pr-$pr" --no-commit; then echo "Merge conflict while merging PR #$pr. Aborting merge and failing the job." @@ -176,26 +183,28 @@ jobs: - name: Apply kernel-topics PR patches if: ${{ inputs.kernel-url == '' }} + env: + KERNEL_TOPICS_PR_INPUT: ${{ github.event.inputs.kernel-topics-pr }} run: | - cd "${{ env.build_top }}/" - if [ -z "${{ github.event.inputs.kernel-topics-pr }}" ]; then + cd "$build_top/" + if [ -z "$KERNEL_TOPICS_PR_INPUT" ]; then echo "Nothing to apply in kernel topics." else - echo "Applying PR(s) from topics branch #${{ github.event.inputs.kernel-topics-pr }}" - for pr in ${{ github.event.inputs.kernel-topics-pr }}; do + echo "Applying PR(s) from topics branch #$KERNEL_TOPICS_PR_INPUT" + for pr in $KERNEL_TOPICS_PR_INPUT; do echo "::group::Applying kernel-topics PR #$pr" wget -q "https://github.com/qualcomm-linux/kernel-topics/pull/$pr.patch" -O "$pr.patch" - - cd "${{ env.build_top }}/qcom-kernel" - + + cd "$build_top/qcom-kernel" + if ! git am "../$pr.patch"; then echo "Patch application failed for PR #$pr. Aborting 'git am' and failing the job." git am --abort || true exit 1 fi echo "::endgroup::" - cd "${{ env.build_top }}/" + cd "$build_top/" done fi @@ -216,35 +225,53 @@ jobs: ' - name: Build Kernel Debian Package + env: + JOB_ID: ${{ github.run_id }} + JOB_ATTEMPT: ${{ github.run_attempt }} run: | docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" kmake-image:ubuntu-noble-arm64 \ + --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ + -e JOB_ID -e JOB_ATTEMPT \ + kmake-image:ubuntu-noble-arm64 \ -c ' cd qcom-build-utils/kernel export BUILD_TOP=`pwd` - ./scripts/build-kernel-deb.sh out/ ${{ github.run_id }}-${{ github.run_attempt }} + ./scripts/build-kernel-deb.sh out/ ${JOB_ID}-${JOB_ATTEMPT} mkdir -p deb_artifact cp ./*.deb deb_artifact/ ' - name: Generate build_info metadata + env: + JOB_ID: ${{ github.run_id }} + JOB_ATTEMPT: ${{ github.run_attempt }} + QCOM_BUILD_UTILS_REF_INPUT: ${{ inputs.qcom-build-utils-ref }} + KERNEL_URL_INPUT: ${{ inputs.kernel-url }} + KERNEL_BRANCH_INPUT: ${{ inputs.kernel-branch }} + QCOM_NEXT_PR_INPUT: ${{ inputs.qcom-next-pr }} + KERNEL_TOPICS_PR_INPUT: ${{ inputs.kernel-topics-pr }} run: | docker run -i \ - --privileged --rm -v $PWD:$PWD --workdir="$PWD" kmake-image:ubuntu-noble-arm64 \ + --privileged --rm -v $PWD:$PWD --workdir="$PWD" \ + -e JOB_ID -e JOB_ATTEMPT -e ORG_NAME -e REPO_NAME \ + -e QCOM_BUILD_UTILS_REF_INPUT -e QCOM_BUILD_UTILS_SHA \ + -e KERNEL_URL_INPUT -e KERNEL_BRANCH_INPUT -e QCOM_KERNEL_SHA \ + -e QCOM_NEXT_PR_INPUT -e KERNEL_TOPICS_PR_INPUT \ + kmake-image:ubuntu-noble-arm64 \ -c ' cd qcom-build-utils/kernel { - echo "JOB_ID: ${{ github.run_id }}" - echo "JOB_ATTEMPT: ${{ github.run_attempt }}" - echo "ORG_NAME: ${{ env.ORG_NAME }}" - echo "REPO_NAME: ${{ env.REPO_NAME }}" - echo "QCOM-BUILD-UTILS BRANCH/TAG: ${{ inputs.qcom-build-utils-ref }}" - echo "QCOM-BUILD-UTILS HEAD SHA: ${{ env.QCOM_BUILD_UTILS_SHA }}" - echo "CUSTOM KERNEL URL: ${{ inputs.kernel-url }}" - echo "KERNEL BRANCH/TAG: ${{ inputs.kernel-branch }}" - echo "KERNEL HEAD SHA: ${{ env.QCOM_KERNEL_SHA }}" - echo "PRs FROM QCOM-NEXT: ${{ inputs.qcom-next-pr }}" - echo "PRs FROM KERNEL TOPICS: ${{ inputs.kernel-topics-pr }}" + echo "JOB_ID: $JOB_ID" + echo "JOB_ATTEMPT: $JOB_ATTEMPT" + echo "ORG_NAME: $ORG_NAME" + echo "REPO_NAME: $REPO_NAME" + echo "QCOM-BUILD-UTILS BRANCH/TAG: $QCOM_BUILD_UTILS_REF_INPUT" + echo "QCOM-BUILD-UTILS HEAD SHA: $QCOM_BUILD_UTILS_SHA" + echo "CUSTOM KERNEL URL: $KERNEL_URL_INPUT" + echo "KERNEL BRANCH/TAG: $KERNEL_BRANCH_INPUT" + echo "KERNEL HEAD SHA: $QCOM_KERNEL_SHA" + echo "PRs FROM QCOM-NEXT: $QCOM_NEXT_PR_INPUT" + echo "PRs FROM KERNEL TOPICS: $KERNEL_TOPICS_PR_INPUT" } > deb_artifact/build_info ' diff --git a/.github/workflows/daily.yml b/.github/workflows/daily.yml index 9c004be..8d82e1b 100644 --- a/.github/workflows/daily.yml +++ b/.github/workflows/daily.yml @@ -1,11 +1,20 @@ name: daily # Daily kernel package builds. Runs at 3:00 PM PST (23:00 UTC), one parallel -# build per entry in ci/build-matrix.json. Manual dispatch can run the full -# matrix ("Test Daily Build") or a single distro. +# build per Daily entry in ci/build-matrix.json after suite flattening. +# Manual dispatch can run the full Daily matrix, every suite for one kernel +# variant, or one kernel variant and suite. Variant and suite are matrix values, +# so future variants need no workflow edit. # -# To add a daily target, append { "distro": "" } to -# ci/build-matrix.json; no workflow change is needed. +# Matrix schema: ci/build-matrix.json +# Each row with "type": "Daily" is expanded by ci/scripts/resolve-matrix.sh: +# the "suites" array is flattened into one job per suite. All other fields +# (kernel_variant, srcpkg, binpkg, kernel_config, debian_revision, +# git_clone, branch_or_tag, ref_strategy) are passed through to +# build-kernel-deb.yml as inputs. +# +# To add a daily build target: add a row to ci/build-matrix.json with +# "type": "Daily". No workflow changes needed. on: schedule: @@ -13,22 +22,25 @@ on: - cron: '0 23 * * *' workflow_dispatch: inputs: - run-full-matrix: - description: 'Test Daily Build (runs all matrix combinations, same as the scheduled daily build)' - type: boolean - required: false - default: false - distro: - description: 'Target distro (ignored when Test Daily Build is checked)' - required: false - default: 'trixie' + build-scope: + description: 'Choose all variants, all suites for one variant, or one variant and suite' type: choice + required: true + default: 'Selected variant and suite' options: - - trixie - - noble - - questing - - resolute - - sid + - 'Full matrix' + - 'Selected variant (all suites)' + - 'Selected variant and suite' + kernel-variant: + description: 'Kernel variant to build (ignored for Full matrix)' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target suite to build (used only for Selected variant and suite)' + required: true + default: 'trixie' + type: string permissions: contents: read @@ -36,12 +48,12 @@ permissions: jobs: # --------------------------------------------------------------------------- - # configure-matrix: determine which distros to build. - # - # Schedule / Test Daily Build checked → full matrix from ci/build-matrix.json. - # Specific distro input → single-entry matrix from the input. + # configure-matrix: flatten the Daily rows from ci/build-matrix.json. # - # Mirrors the configure-matrix pattern used in qcom-distro-images. + # Delegates to ci/scripts/resolve-matrix.sh which filters type=Daily rows + # and expands each row's "suites" array into one flat entry per suite. + # Manual dispatch selects the complete matrix, every suite for a variant, or + # one variant and suite. # --------------------------------------------------------------------------- configure-matrix: name: Configure build matrix @@ -53,37 +65,52 @@ jobs: - name: Generate build matrix id: set-matrix + env: + BUILD_SCOPE_INPUT: ${{ github.event.inputs.build-scope }} + INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} + INPUT_SUITE: ${{ github.event.inputs.suite }} run: | - INPUT_DISTRO="${{ github.event.inputs.distro }}" - RUN_FULL="${{ github.event.inputs.run-full-matrix }}" + set -euo pipefail - if [ "${{ github.event_name }}" == "schedule" ] || [ "$RUN_FULL" == "true" ]; then - echo "Source: ci/build-matrix.json (full matrix)" + BUILD_SCOPE="$BUILD_SCOPE_INPUT" - if [ ! -f "./ci/build-matrix.json" ]; then - echo "::error::ci/build-matrix.json not found" - exit 1 - fi - - MATRIX=$(jq -c . < ./ci/build-matrix.json) - else - echo "Source: manual input (single target: $INPUT_DISTRO)" - MATRIX="[{\"distro\": \"$INPUT_DISTRO\"}]" + if [[ "${{ github.event_name }}" == "schedule" ]]; then + BUILD_SCOPE="Full matrix" fi + case "$BUILD_SCOPE" in + "Full matrix") + echo "Source: ci/build-matrix.json (full Daily matrix)" + args=(--type Daily) + ;; + "Selected variant (all suites)") + echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" + args=(--type Daily --kernel-variant "$INPUT_VARIANT") + ;; + "Selected variant and suite") + echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" + args=(--type Daily --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + ;; + *) + echo "ERROR: unsupported build scope: $BUILD_SCOPE" >&2 + exit 1 + ;; + esac + MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" - echo "Matrix: $MATRIX" + echo "Matrix entries: $(echo "$MATRIX" | jq length)" # --------------------------------------------------------------------------- - # build: one parallel job per matrix entry. + # build: one parallel job per flat matrix entry. # - # Each entry calls build-kernel-deb.yml as a reusable workflow, which owns - # the full build pipeline (prepare → debusine-build or ubuntu-build → - # upload-artifacts). Jobs are independent; fail-fast is disabled so a - # failed trixie build does not cancel the resolute build. + # Each entry calls build-kernel-deb.yml as a reusable workflow. The full + # set of matrix fields (suite, srcpkg, binpkg, kernel_config, etc.) is + # threaded through as inputs. fail-fast is disabled so a failed trixie + # build does not cancel the forky build. # --------------------------------------------------------------------------- build: - name: ${{ matrix.distro }} + name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} needs: configure-matrix strategy: fail-fast: false @@ -91,5 +118,21 @@ jobs: include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} uses: ./.github/workflows/build-kernel-deb.yml with: - distro: ${{ matrix.distro }} - secrets: inherit + suite: ${{ matrix.suite }} + kernel-variant: ${{ matrix.kernel_variant }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} + debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + pkg-linux-qcom-ref: ${{ matrix.pkg_linux_qcom_ref || 'qcom/debian/latest' }} + secrets: + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..9b478ec --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,121 @@ +name: release + +# Release kernel package builds. Triggered manually only. +# +# Reads Release rows from ci/build-matrix.json, flattens suites, and builds +# each entry using the matrix-selected ref strategy and branch_or_tag. After a +# successful build, packages are promoted from the ephemeral Debusine CI +# workspace to the stable target workspace (e.g. qli) via debusine-action +# lib/release. +# +# To update a release ref: open a PR that changes the Release row for that +# kernel variant in ci/build-matrix.json. +# +# This workflow intentionally has no schedule trigger. Releases are always +# initiated manually after validation. + +on: + workflow_dispatch: + inputs: + release-scope: + description: 'Choose all suites for one variant or one variant and suite' + type: choice + required: true + default: 'Selected variant (all suites)' + options: + - 'Selected variant (all suites)' + - 'Selected variant and suite' + kernel-variant: + description: 'Kernel variant to release' + required: true + default: 'qcom-next' + type: string + suite: + description: 'Target suite to release (used only for Selected variant and suite)' + required: true + default: 'trixie' + type: string + +permissions: + contents: read + packages: read + +jobs: + # --------------------------------------------------------------------------- + # configure-matrix: flatten the Release rows from ci/build-matrix.json. + # --------------------------------------------------------------------------- + configure-matrix: + name: Configure release matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + + - name: Generate release matrix + id: set-matrix + env: + RELEASE_SCOPE_INPUT: ${{ github.event.inputs.release-scope }} + INPUT_VARIANT: ${{ github.event.inputs.kernel-variant }} + INPUT_SUITE: ${{ github.event.inputs.suite }} + run: | + set -euo pipefail + + RELEASE_SCOPE="$RELEASE_SCOPE_INPUT" + + case "$RELEASE_SCOPE" in + "Selected variant (all suites)") + echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, all suites)" + args=(--type Release --kernel-variant "$INPUT_VARIANT") + ;; + "Selected variant and suite") + echo "Source: ci/build-matrix.json (variant: $INPUT_VARIANT, suite: $INPUT_SUITE)" + args=(--type Release --kernel-variant "$INPUT_VARIANT" --single-suite "$INPUT_SUITE") + ;; + *) + echo "ERROR: unsupported release scope: $RELEASE_SCOPE" >&2 + exit 1 + ;; + esac + MATRIX=$(ci/scripts/resolve-matrix.sh "${args[@]}") + + echo "matrix=$MATRIX" >> "$GITHUB_OUTPUT" + echo "Matrix entries: $(echo "$MATRIX" | jq length)" + + # --------------------------------------------------------------------------- + # build: one parallel job per flat Release matrix entry. + # + # Uses the ref strategy and branch_or_tag selected by the matrix. + # Passes target_workspace so build-kernel-debusine.yml triggers the Debusine + # release path (lib/release) instead of the S3 publish path. + # --------------------------------------------------------------------------- + build: + name: ${{ matrix.kernel_variant }} / ${{ matrix.suite }} + needs: configure-matrix + strategy: + fail-fast: false + matrix: + include: ${{ fromJson(needs.configure-matrix.outputs.matrix) }} + uses: ./.github/workflows/build-kernel-deb.yml + with: + suite: ${{ matrix.suite }} + kernel-variant: ${{ matrix.kernel_variant }} + git-clone: ${{ matrix.git_clone }} + kernel-branch: ${{ matrix.branch_or_tag }} + ref-strategy: ${{ matrix.ref_strategy }} + tag-pattern: ${{ matrix.tag_pattern || '' }} + srcpkg: ${{ matrix.srcpkg }} + binpkg: ${{ matrix.binpkg }} + kernel-config: ${{ matrix.kernel_config }} + debian-revision: ${{ matrix.debian_revision }} + localversion: ${{ matrix.localversion || '' }} + kver-extra: ${{ matrix.kver_extra || '' }} + debusine-parent-workspace: ${{ matrix.debusine_parent_workspace || '' }} + target-workspace: ${{ matrix.target_workspace }} + pkg-linux-qcom-ref: ${{ matrix.pkg_linux_qcom_ref || 'qcom/debian/latest' }} + secrets: + DEBUSINE_USER: ${{ secrets.DEBUSINE_USER }} + DEBUSINE_TOKEN: ${{ secrets.DEBUSINE_TOKEN }} + DEBUSINE_RELEASE_TOKEN: ${{ secrets.DEBUSINE_RELEASE_TOKEN }} diff --git a/README.md b/README.md index 2ac7004..bcd39c3 100644 --- a/README.md +++ b/README.md @@ -1,157 +1,282 @@ # pkg-linux-qcom -Debian/Ubuntu kernel packaging for `qualcomm-linux/kernel` on ARM64 Qualcomm platforms. -Produces installable `.deb` packages via a dual-path CI pipeline: **Debusine** for Debian suites, **docker** for Ubuntu suites. +CI orchestration for ARM64 Linux kernel package variants. The current variant +builds from [`qualcomm-linux/kernel`](https://github.com/qualcomm-linux/kernel). -This branch (`main`) is the CI orchestrator: the workflows, the build matrix, and this guide. -The `debian/` packaging tree and the local build tooling live on `qcom/debian/latest`; see that branch to build the kernel package locally. +This branch owns the build matrix and GitHub Actions workflows. The Debian +packaging tree and local packaging tools live on +[`qcom/debian/latest`](https://github.com/qualcomm-linux/pkg-linux-qcom/tree/qcom/debian/latest). ---- +## Overview -## Dispatching a build +The CI model is matrix-driven. This single repository can deliver multiple +kernel variants, each with independent source/package identity, kernel source +and ref strategy, configuration fragments, Debian revision, target suites, and +release destination. -To start a build manually, open the repo's **Actions** tab and click **Run workflow**: +Every kernel variant owns exactly two complete matrix rows: one `Daily` row and +one `Release` row. The resolver expands every suite in those rows into an +isolated `kernel_variant + suite` build leg. -- **Single build:** `build-kernel-deb` then **Run workflow**. Pick a `distro` and set the inputs below. -- **Nightly-style run:** `daily` then **Run workflow**. Check **Test Daily Build** to run the full `ci/build-matrix.json` matrix, or leave it unchecked and pick one distro. +Two entry points use the same reusable build pipeline: -Builds also run automatically every night (see [Daily Build Matrix](#daily-build-matrix)). +- **Daily** uses the matrix-selected latest-tag or branch-tip strategy and + builds every configured Daily suite. +- **Release** uses a pinned matrix ref and promotes successful Debian packages + to the selected production Debusine workspace. -### `build-kernel-deb.yml` Inputs - -#### `workflow_dispatch` (manual single build) - -| Input | Default | Description | -|---|---|---| -| `distro` | `trixie` | Target suite | -| `latest-tag` | `true` | Use latest `qcom-next-*` tag | -| `kernel-branch` | `qcom-next` | Branch/tag when `latest-tag=false` | -| `kernel-url` | qualcomm-linux/kernel | Custom kernel repo URL | -| `pkg-linux-qcom-ref` | `qcom/debian/latest` | Packaging metadata ref | -| `localversion` | | Override LOCALVERSION suffix | -| `kver-extra` | | Extra suffix appended to package version | -| `debug-build` | `false` | Copies `debug.config` into `debian/config/` | -| `self-pr` | | Apply a pkg-linux-qcom PR before building | -| `qcom-next-pr` | | Space-separated qcom-next PR numbers to merge | -| `kernel-topics-pr` | | Space-separated kernel-topics PR numbers to apply | - -#### `workflow_call` (called by `daily.yml`) - -| Input | Default | Description | -|---|---|---| -| `distro` | `trixie` | Target suite | -| `latest-tag` | `true` | Always true for daily builds | -| `pkg-linux-qcom-ref` | `qcom/debian/latest` | Packaging metadata ref | - -### Daily Build Matrix - -**`ci/build-matrix.json`**: one entry per nightly build target: +The final Production matrix is conceptually: ```json -[ - { "distro": "trixie" }, - { "distro": "resolute" } -] +{ + "suite_suffix_mapping": { + "trixie": "~bpo13+1", + "forky": "", + "resolute": "~26.04.1" + }, + "deliveries": [ + { + "kernel_variant": "qcom-next", + "type": "Daily", + "suites": ["trixie", "forky", "resolute"], + "git_clone": "https://github.com/qualcomm-linux/kernel", + "branch_or_tag": "qcom-next", + "ref_strategy": "latest_tag", + "tag_pattern": "qcom-next-*", + "srcpkg": "linux-qcom-next", + "binpkg": "linux-image-qcom-next", + "kernel_config": "squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can", + "debian_version_stub": "0qli", + "debian_version_suffix": "~", + "pkg_linux_qcom_ref": "qcom/debian/latest" + }, + { + "kernel_variant": "qcom-next", + "type": "Release", + "suites": ["trixie", "forky"], + "git_clone": "https://github.com/qualcomm-linux/kernel", + "branch_or_tag": "", + "ref_strategy": "pinned_ref", + "srcpkg": "linux-qcom-next", + "binpkg": "linux-image-qcom-next", + "kernel_config": "squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can", + "debian_version_stub": "0qli", + "debian_version_suffix": "", + "pkg_linux_qcom_ref": "qcom/debian/latest", + "target_workspace": "qli" + } + ] +} ``` -> To add a nightly target: append one entry. No workflow changes needed. - -#### Manual Dispatch Options (`daily.yml`) - -| Input | Type | Behaviour | -|---|---|---| -| `run-full-matrix` checked | boolean | Runs all matrix entries, identical to scheduled daily build | -| `run-full-matrix` unchecked + `distro` | choice | Runs a single distro build | - -### Build Outputs - -| Package | Contents | Install | -|---|---|---| -| `linux-image--qcom__arm64.deb` | Kernel image, `.config`, DTBs, modules | **Required** | -| `linux-headers--qcom__arm64.deb` | Headers for out-of-tree modules (DKMS) | Optional | -| `linux-image--qcom-dbg__arm64.deb` | Full debug symbols (`vmlinux`, per-module) | Optional | -| `*.buildinfo` | Reproducible build metadata | Do not install | -| `*.changes` | Upload manifest | Do not install | - -```bash -sudo dpkg -i linux-image--qcom__arm64.deb - -# Optional: headers for DKMS / out-of-tree modules -sudo dpkg -i linux-headers--qcom__arm64.deb +`suite_suffix_mapping` is matrix-wide policy, not duplicated per row: every +suite referenced by any row's `suites` must have an entry here, and every +delivery for a variant derives its final `debian_revision` as +`debian_version_stub + suite_suffix_mapping[suite] + delivery_suffix`, where +`delivery_suffix` is `~` for Daily and empty for Release. For the values +above: + +| Suite | Daily | Release | +| --- | --- | --- | +| Trixie | `0qli~bpo13+1~` | `0qli~bpo13+1` | +| Forky | `0qli~` | `0qli` | +| Resolute | `0qli~26.04.1~` | (not a configured Release suite) | + +`~` always sorts below the same prefix without it in Debian version +ordering, so Daily always sorts below Release for the same suite and stub. +Ordering across *different* suites depends entirely on the configured +suffixes: with the mapping above, Resolute < Trixie < Forky for the same +delivery type, matching a Debian-backports-then-unstable promotion chain. +This is a deliberate ordering policy, not an automatic guarantee — adding a +suite means choosing a suffix that sorts where that suite belongs relative to +the others. One nuance to be aware of: because Forky's suffix is empty, its +Daily revision ends immediately after the trailing `~`, so Trixie Daily does +not sort below Forky Daily even though Trixie Release sorts below Forky +Release. This does not affect the supported Release-to-Release upgrade path. + +`ci/build-matrix.json` is the authoritative configuration. Adding a kernel +variant is a two-row matrix change, not a workflow redesign. + +## Workflows + +| Workflow | Purpose | Trigger | +| --- | --- | --- | +| `daily.yml` | Resolves and runs the Daily matrix. | Scheduled daily at `23:00 UTC`, or manual dispatch. | +| `release.yml` | Resolves and runs the Release matrix. | Manual dispatch only. | +| `build-kernel-deb.yml` | Reusable orchestrator for one kernel variant and suite. | Manual dispatch or called by Daily and Release. | +| `build-kernel-debusine.yml` | Builds Debian suites in Debusine and either publishes Daily artifacts or promotes Releases. | Called by `build-kernel-deb.yml`. | +| `build-kernel-ubuntu.yml` | Builds Ubuntu-family suites with the Docker path. | Called by `build-kernel-deb.yml`. | + +### Daily + +Daily is the recurring build and artifact-publication path. + +- The scheduled run resolves the full `Daily` matrix. +- A manual run selects one **Build scope**: + - **Full matrix** builds every configured variant and suite. + - **Selected variant (all suites)** builds every configured suite for one variant. + - **Selected variant and suite** builds one isolated matrix leg. +- `latest_tag` resolves the newest matching dated tag; `branch_tip` resolves + the configured branch directly. +- Debian suites build in Debusine, then their `.deb` outputs are downloaded and + uploaded to the configured S3 bucket. +- `resolute` stays on the Docker-based Ubuntu path and uploads its package + outputs to the existing temporary-package S3 location. + +### Release + +Release is the controlled promotion path. + +- It is manual only and uses one **Release scope** for a kernel variant: + - **Selected variant (all suites)** is the normal release action and promotes + every configured Release suite for that variant. + - **Selected variant and suite** promotes one configured Release suite for + that variant when a targeted action is required. +- It uses the pinned `branch_or_tag` from the selected `Release` matrix row; it + never resolves a newest tag. +- Debian source and binary artifacts are built in per-variant, per-suite + Debusine CI workspaces. +- Successful builds are promoted with Debusine's `package-publish` workflow to + the `qli` workspace, where they are available through the production Debusine + APT repository. +- The Release job runs in the **Production** GitHub environment. This provides + the release credential and enforces the required approval gate before + promotion to `qli`. + +Direct `build-kernel-deb.yml` dispatches are build-only. Release promotion is +initiated exclusively by `release.yml`, which owns the target workspace and +production release controls. + +## Matrix Model + +`ci/build-matrix.json` is an object with two top-level keys: `deliveries` +(the matrix rows) and `suite_suffix_mapping` (matrix-wide Debian suffix +policy, shared by every variant and delivery type). `ci/scripts/resolve-matrix.sh` +validates the document, requires each `kernel_variant` to have exactly one +`Daily` and one `Release` row in `deliveries`, filters by delivery type, and +flattens each `suites` array into independent suite legs. Each leg carries +its own values for: + +| Field | Purpose | +| --- | --- | +| `kernel_variant` | Stable identifier for a separately packaged kernel variant. Lowercase letters, digits, and internal hyphens only. | +| `type` | `Daily` or `Release`. | +| `suites` | Suites to flatten into individual build legs. Each must have a `suite_suffix_mapping` entry. | +| `git_clone` | Kernel source repository. | +| `branch_or_tag` | Source branch or pinned tag, according to `ref_strategy`. | +| `ref_strategy` | `latest_tag`, `branch_tip`, or `pinned_ref`. | +| `tag_pattern` | Required only for `latest_tag`; matching tags must end in `-YYYYMMDD`, which determines newest-first ordering. | +| `srcpkg` | Debian source package name. | +| `binpkg` | Kernel image metapackage name. | +| `kernel_config` | Comma-separated fragments activated from `debian/config-available/`. | +| `debian_version_stub` | Base Debian revision, shared by a variant's Daily and Release rows. Must not end in `~`; the suite suffix is derived, not stored here. | +| `debian_version_suffix` | `~` for Daily rows, empty for Release rows. Documents the delivery-type half of the revision formula on the row itself; `resolve-matrix.sh` rejects a row where this disagrees with `type`, but derivation always computes this suffix from `type`, never reads this field. | +| `localversion`, `kver_extra` | Optional version overrides forwarded to packaging. | +| `pkg_linux_qcom_ref` | Packaging branch or commit used during source preparation. | +| `debusine_parent_workspace` | Optional parent workspace override for the variant's CI child workspaces. | +| `target_workspace` | Debusine destination for Release entries only. | + +`target_workspace` is required for `Release` and rejected for `Daily`. +`tag_pattern` is required for `latest_tag` and rejected for other strategies. +The resolver selects the most recent trailing `YYYYMMDD` date, and rejects +duplicate suites and malformed variant identifiers before any build jobs +start. It also rejects a matrix where any configured suite has no +`suite_suffix_mapping` entry, where two suites share the same suffix, where a +suffix is non-empty and doesn't start with `~`, where a variant's Daily +and Release rows disagree on `debian_version_stub`, or where a row's +`debian_version_suffix` doesn't match what its `type` implies — all before +any build job starts. + +Each flattened leg's final `debian_revision` is derived by +`ci/scripts/derive-debian-revision.sh` from `debian_version_stub`, +`suite_suffix_mapping[suite]`, and the delivery type +(`stub + suffix + "~"` for Daily, `stub + suffix` for Release). This script is +the single implementation of the formula: `resolve-matrix.sh` calls it once +per flattened leg, and `build-kernel-deb.yml`'s direct-dispatch path (which +has no full-matrix context) calls the same script for the one suite it was +given. + +Each leg has a distinct prepared-source artifact, Debusine child workspace, and +S3 path keyed by `kernel_variant + suite`. This prevents two variants that both +build, for example, `trixie` from consuming or publishing each other's inputs +or outputs. + +Daily S3 outputs use these layouts, where `` is +`-`: + +```text +/pkg/debusine///// +/pkg/temp///// ``` -S3 destinations: +The first layout is for Debian/Debusine builds; the second is for Ubuntu Docker +builds. Consumers must select the intended kernel variant and suite. -| Path | Build type | -|---|---| -| `s3:////pkg/debusine///-/` | Debian (Debusine) | -| `s3:////pkg/temp//-/` | Ubuntu (docker) | +Supporting scripts keep workflow YAML small and testable: ---- +| Script | Responsibility | +| --- | --- | +| `ci/scripts/resolve-matrix.sh` | Validates and flattens matrix rows. | +| `ci/scripts/resolve-kernel-ref.sh` | Resolves a matrix-selected dated tag or validates a direct ref. | +| `ci/scripts/derive-localversion.sh` | Derives `LOCALVERSION` from the variant and resolved kernel ref. | +| `ci/scripts/derive-debian-revision.sh` | Derives the final suite-specific `debian_revision` from `debian_version_stub`, `suite_suffix_mapping`, and delivery type. | ## Architecture ```mermaid flowchart LR - IN["distro input"] --> R{resolve job} + IN["Matrix variant + suite input"] --> R{"Resolve suite family"} - R -->|"trixie · sid\nunstable · bookworm · forky"| DEB["family = debian"] - R -->|"noble · questing\nresolute"| UBU["family = ubuntu"] + R -->|"trixie · forky"| DEB["Debian path\nbuild-kernel-debusine.yml\nGenerate source package\nSubmit with lib/build\nDebusine builds binaries"] + R -->|"resolute"| UBU["Ubuntu path\nbuild-kernel-ubuntu.yml\nbuild-kernel.sh in Docker\nBuild binary packages"] - DEB --> DB["debusine-build\nbuild-kernel-debusine.yml\nGenerates .dsc\nSubmits to Debusine\nchdist download\nPublish to S3"] - UBU --> UB["ubuntu-build\nbuild-kernel-ubuntu.yml\nbuild-kernel.sh\ndocker pkg-builder\nPublish to S3"] + DEB --> DOUT{"Build type"} + DOUT -->|Daily| S3["Download .deb files\nPublish to S3"] + DOUT -->|Release| QLI["Promote source and binaries\nto qli"] + UBU --> US3["Publish .deb files to S3"] ``` -| File | Role | Trigger | -|---|---|---| -| `daily.yml` | Daily orchestrator: reads matrix, spawns parallel builds | `schedule` · `workflow_dispatch` | -| `build-kernel-deb.yml` | Main pipeline: resolve + prepare, delegates to family modules | `workflow_dispatch` · `workflow_call` | -| `build-kernel-debusine.yml` | Debian build module: source package generation, Debusine submission, publish to S3 | `workflow_call` only | -| `build-kernel-ubuntu.yml` | Ubuntu build module: `build-kernel.sh` via docker, upload to S3 | `workflow_call` only | - ---- - -## For CI maintainers - -Internal pipeline detail. Most users do not need this section. +## For CI Maintainers ### Pipeline overview ```mermaid flowchart TD - subgraph Triggers - A1["⏰ daily.yml\n3 PM PST · cron"] - A2["🖱 daily.yml\nManual dispatch"] - A3["🖱 build-kernel-deb.yml\nManual dispatch"] + subgraph triggers[Triggers] + A1["daily.yml\nScheduled full matrix"] + A2["daily.yml\nManual full or filtered variant + suite"] + A3["release.yml\nManual full or filtered variant + suite"] + A4["build-kernel-deb.yml\nManual one-off build"] end - subgraph daily["daily.yml"] - B1["configure-matrix\nReads ci/build-matrix.json"] - B2["build · trixie"] - B3["build · resolute"] + subgraph matrix[Matrix entry points] + B1["Daily configure-matrix\nFlatten Daily rows"] + B2["Daily variant + suite legs\nqcom-next / trixie · forky · resolute"] + B3["Release configure-matrix\nFlatten Release rows"] + B4["Release variant + suite legs\nqcom-next / trixie · forky"] end - subgraph orchestrator["build-kernel-deb.yml"] + subgraph orchestrator[build-kernel-deb.yml] C1["resolve\nClassify suite family"] - C2["prepare\nClone kernel\nRun prepare-source.sh\nUpload kernel-srcpkg"] - C3["debusine-build\nDebian suites only\nbuild-kernel-debusine.yml"] - C4["ubuntu-build\nUbuntu suites only\nbuild-kernel-ubuntu.yml"] + C2["prepare\nClone selected kernel ref\nRun prepare-source.sh\nUpload kernel-srcpkg-variant-suite"] + C3["debusine-build\nDebian suites only"] + C4["ubuntu-build\nUbuntu suites only"] end - subgraph out["Outputs"] - D1["S3\nlinux-image\nlinux-headers\ndbg"] + subgraph outputs[Outputs] + D1["Daily S3 artifacts"] + D2["Release qli APT repository"] end A1 --> B1 A2 --> B1 - B1 --> B2 & B3 - B2 -->|workflow_call distro=trixie| C1 - B3 -->|workflow_call distro=resolute| C1 - A3 -->|workflow_dispatch| C1 - + A3 --> B3 + B1 --> B2 --> C1 + B3 --> B4 --> C1 + A4 --> C1 C1 --> C2 C2 --> C3 & C4 - C3 --> D1 + C3 --> D1 & D2 C4 --> D1 ``` @@ -159,77 +284,206 @@ flowchart TD ```mermaid flowchart LR - K["qualcomm-linux/kernel\nlatest qcom-next-* tag"] --> PS - M["pkg-linux-qcom\ndebian/ metadata"] --> PS - - PS["prepare-source.sh\npkg-builder:DISTRO container\n\nInject debian/\nActivate config fragments\nGenerate debian/control\nGenerate debian/changelog"] --> TAR + K["Matrix-selected kernel repository\nDaily: latest tag or branch tip\nRelease: pinned ref"] --> PS + M["pkg-linux-qcom\nMatrix-selected packaging ref\nFinal: qcom/debian/latest"] --> PS - TAR["tar czf kernel-srcpkg.tar.gz\nPreserves execute permissions"] --> ART - - ART["GitHub Actions Artifact\nkernel-srcpkg\nShared via run_id"] + PS["prepare-source.sh\n\nInject debian/\nActivate selected config fragments\nGenerate control, changelog, localversion, pkgversion"] --> TAR + TAR["tar czf kernel-srcpkg-variant-suite.tar.gz\nPreserves execute permissions"] --> ART + ART["GitHub Actions artifact\nOne prepared source tree per variant + suite"] ``` -> **Why `tar.gz`?** `actions/upload-artifact` uses zip internally, which strips Unix execute bits. -> Kernel build scripts (e.g. `scripts/cc-version.sh`) require execute permission. -> `tar` preserves them end-to-end; `--strip-components=1` restores them on extraction. +> **Why `tar.gz`?** `actions/upload-artifact` uses zip internally, which strips +> Unix execute bits. Kernel build scripts require those permissions. The tar +> archive preserves them between the prepare and build jobs. -### Debian path +### Debian Daily path ```mermaid flowchart LR - ART["kernel-srcpkg\nartifact"] --> GSP + ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP - subgraph build["build job (debusine-pkg-builder container)"] - GSP["generate-source-package\nDEBUSINE_ASSEMBLE_ORIG=true\n\nCreate .orig.tar.gz\nRun dpkg-buildpackage -S\nProduce .dsc"] --> DEB - DEB["Debusine\nDistributed build"] --> WS - WS["workspace ID"] + subgraph source[GitHub build job: debusine-pkg-builder container] + GSP["generate-source-package\nDEBUSINE_ASSEMBLE_ORIG=true\n\nCreate .orig.tar.gz\nRun dpkg-buildpackage -S\nProduce .dsc"] --> SUBMIT + SUBMIT["lib/build\nCreate CI child workspace\nSubmit source package to Debusine"] end - subgraph pub["publish job (self-hosted runner)"] - WS --> CHDIST - CHDIST["generate-apt-config\nchdist hermetic apt env\napt-get download\nNo installation"] --> S3 + SUBMIT --> DEB["Debusine\nBuild binary packages"] + DEB --> WS["Unique variant + suite workspace"] + + subgraph publish[Daily publish job] + WS --> APT["generate-apt-config\nchdist isolated APT environment\nDownload .deb files"] + APT --> S3["S3\nDaily package artifacts"] end +``` + +### Debian Release path - S3["S3\n"] +```mermaid +flowchart LR + ART["kernel-srcpkg-variant-suite\nartifact"] --> GSP["generate-source-package\nProduce .dsc"] + GSP --> SUBMIT["lib/build\nSubmit source package to a unique\nDebusine CI child workspace"] + SUBMIT --> DEB["Debusine\nBuild binary packages"] + DEB --> WS["CI workspace\nsource and binary artifacts"] + + subgraph release[Release job: Production GitHub environment] + WS --> PROMOTE["lib/release\nStart package-publish"] + PROMOTE --> QLI["qli\nProduction Debusine APT repository"] + end ``` ### Ubuntu path ```mermaid flowchart LR - ART["kernel-srcpkg\nartifact"] --> EXT + ART["kernel-srcpkg-variant-suite\nartifact"] --> EXT - subgraph build["build job (self-hosted runner)"] - EXT["Extract source tree\n--strip-components=1"] --> BK + subgraph build[Ubuntu build job] + EXT["Extract prepared source tree\n--strip-components=1"] --> BK BK["build-kernel.sh\n--skip-prepare\n--local-source\n--build-mode docker\ndpkg-buildpackage -b"] --> S3 end - S3["S3\n"] + S3["S3\nDaily package artifacts"] ``` -> `--skip-prepare` is safe because `prepare-source.sh` already ran in the `prepare` job. -> `debian/control`, `debian/changelog`, and all config fragments are baked into the artifact. +`--skip-prepare` is safe because `prepare-source.sh` has already generated the +packaging metadata and activated the selected fragments before the artifact is +created. + +## Packages + +The matrix provides the source package and image metapackage identity. The +resolved kernel release remains the source of truth for versioned package names +and installed kernel paths. + +For the current matrix, package generation produces: -### Required configuration +| Package | Purpose | +| --- | --- | +| `linux-qcom-next_.dsc` and related source files | Debian source package. | +| `linux-image-__arm64.deb` | Versioned kernel image, modules, DTBs, and boot assets. | +| `linux-image-qcom-next__arm64.deb` | Image metapackage that tracks the newest kernel image. | +| `linux-headers-__arm64.deb` | Versioned headers for DKMS and out-of-tree modules. | +| `linux-headers-qcom-next__arm64.deb` | Headers metapackage. | +| `linux-image--dbg__arm64.deb` | Kernel and module debug symbols. | -Set these in the repository (or organization) settings. The Debusine path needs -all of them; the docker path needs only `ARTIFACT_S3_BUCKET`. +`-rcN` remains in `uname -r`, module paths, boot assets, and versioned package +names. Only the Debian version field converts it to `~rcN`, so a release +candidate correctly sorts before the corresponding final kernel release. -| Type | Name | Purpose | -|---|---|---| -| Variable | `ARTIFACT_S3_BUCKET` | S3 bucket the built packages are uploaded to | -| Variable | `DEBUSINE_HOST` | Debusine instance host | -| Variable | `DEBUSINE_SCOPE` | Debusine scope | -| Variable | `DEBUSINE_PARENT_WORKSPACE` | Parent workspace for the CI child workspace | -| Secret | `DEBUSINE_USER` | Debusine API user | -| Secret | `DEBUSINE_TOKEN` | Debusine API token | +`KVER_EXTRA` is supported for explicit suffixes such as `-ci42` or `-local`. +The packaging rules verify that the declared versioned image package matches the +resolved kernel release and fail instead of creating inconsistent metadata. -`vars.*` are available to all jobs (including `workflow_call` callees) without -forwarding. `secrets.*` do not cross a `workflow_call` boundary unless forwarded, -so `build-kernel-deb.yml` forwards only the two Debusine secrets. +For an APT repository installation, install the image metapackage: + +```bash +sudo apt update +sudo apt install linux-image-qcom-next +``` ---- +When installing downloaded artifacts directly, install the versioned image and +its metapackage together. Add the headers packages when DKMS or other +out-of-tree module builds are required. + +## Manual Builds + +Use **Actions** → **build-kernel-deb** for a one-off build. It is an explicit +override workflow, not a matrix-derived delivery flow: use `daily.yml` and +`release.yml` for normal Daily and Release operations. + +`kernel-variant`, `suite`, and `ref-strategy` are the required build selection. +All remaining package, configuration, and PR inputs are advanced overrides for +validation or debugging. Variant and suite are free-text matrix values rather +than static dropdowns, so adding a matrix entry never requires editing the +workflow UI. + +The available inputs are: + +| Input | Default | Purpose | +| --- | --- | --- | +| `kernel-variant` | `qcom-next` | Stable variant identifier used in artifact and workspace identity. | +| `suite` | `trixie` | Target suite. | +| `ref-strategy` | `latest_tag` | `latest_tag`, `branch_tip`, or `pinned_ref`. | +| `kernel-branch` | `qcom-next` | Branch for `branch_tip`, or immutable ref for `pinned_ref`; ignored by `latest_tag`. | +| `tag-pattern` | `qcom-next-*` | Tag glob for `latest_tag`; ignored by `branch_tip` and `pinned_ref`. | +| `kernel-url` | `qualcomm-linux/kernel` | Advanced alternate kernel repository. | +| `srcpkg` | `linux-qcom-next` | Advanced source package identity override. | +| `binpkg` | `linux-image-qcom-next` | Advanced image metapackage identity override. | +| `kernel-config` | `squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can` | Advanced packaging fragments to activate. | +| `debian-version-stub` | `0qli+staging1` | Advanced Debian version stub. The selected suite's mapped suffix and a Daily-style trailing `~` are applied automatically; direct builds always use Daily semantics since they are build-only and non-promoting. | +| `localversion` | Auto-derived | Advanced explicit `LOCALVERSION` override. | +| `kver-extra` | Empty | Advanced kernel-release suffix. | +| `debug-build` | `false` | Advanced debug configuration toggle. | +| `pkg-linux-qcom-ref` | `qcom/debian/latest` | Advanced packaging revision used to prepare the source tree. | + +The workflow also supports advanced Qualcomm-only PR overrides for validation +builds. Direct builds are artifact builds; Release promotion is performed only +through `release.yml`. + +## Configuration + +### Repository and organization variables + +| Variable | Purpose | +| --- | --- | +| `ARTIFACT_S3_BUCKET` | S3 bucket for Daily Debian artifacts and Ubuntu build artifacts. | +| `DEBUSINE_HOST` | Production Debusine host. | +| `DEBUSINE_SCOPE` | Debusine scope. | +| `DEBUSINE_PARENT_WORKSPACE` | Parent workspace used to create per-run CI child workspaces. | + +### Secrets + +| Secret | Scope | Purpose | +| --- | --- | --- | +| `DEBUSINE_USER` | Repository | User for Debusine archive and signing-key access. | +| `DEBUSINE_TOKEN` | Repository | Token for Debusine build and artifact operations. | +| `DEBUSINE_RELEASE_TOKEN` | Production environment | Token used only to promote Release artifacts to `qli`. | + +The Debian build and Release jobs select the **Production** GitHub environment. +This makes environment-scoped release credentials available to the promotion job +and keeps production approval controls in the workflow path. + +## Maintaining the Matrix + +To add a kernel variant: + +1. Add exactly two rows to `deliveries` with the same `kernel_variant`: one + `Daily` and one `Release`. +2. Define all package identity, source/ref strategy, configuration, + `debian_version_stub`, and suite values in both rows. Do not rely on + another variant's values. `srcpkg`, `binpkg`, and `debian_version_stub` + must remain identical across the pair. Set `debian_version_suffix` to `~` + on the Daily row and `""` on the Release row; `resolve-matrix.sh` rejects + the pair if either disagrees with its row's `type`. +3. Use `latest_tag` with a dated tag glob or `branch_tip` for Daily. Use + `pinned_ref` for Release, and update that ref through a reviewed PR. +4. Give the variant distinct `srcpkg` and `binpkg` values. Set the Release + `target_workspace` explicitly. +5. Confirm suite-family routing: Debian suites use Debusine; Ubuntu suites use + the Docker path. +6. Run a filtered Daily validation for the new variant, then its full Daily and + Release flows. + +To add a new suite (for an existing or new variant): + +1. Add an entry for it to the shared top-level `suite_suffix_mapping`, empty + or starting with `~`, and distinct from every other suite's suffix. +2. Add the suite to the `suites` array of the relevant Daily and/or Release + rows. `resolve-matrix.sh` rejects any configured suite with no mapping + entry before any build job starts. +3. Choose the suffix so the suite sorts where it belongs relative to the + others for the same delivery type (see the ordering discussion in + [Overview](#overview)). + +No workflow dispatch choices need to be updated: manual Daily and Release +inputs accept matrix-defined variant and suite strings. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md) for branch, review, and DCO +requirements. ## License -pkg-linux-qcom is licensed under the BSD-3-clause License. See LICENSE.txt for the full license text. +pkg-linux-qcom is licensed under the BSD 3-Clause License. See +[LICENSE.txt](LICENSE.txt). diff --git a/ci/build-matrix.json b/ci/build-matrix.json index 8941032..ce82588 100644 --- a/ci/build-matrix.json +++ b/ci/build-matrix.json @@ -1,4 +1,46 @@ -[ - { "distro": "trixie" }, - { "distro": "resolute" } -] \ No newline at end of file +{ + "suite_suffix_mapping": { + "trixie": "~bpo13+1", + "forky": "", + "resolute": "~26.04.1" + }, + "deliveries": [ + { + "kernel_variant": "qcom-next", + "type": "Daily", + "suites": [ + "trixie", + "forky", + "resolute" + ], + "git_clone": "https://github.com/qualcomm-linux/kernel", + "branch_or_tag": "qcom-next", + "ref_strategy": "latest_tag", + "tag_pattern": "qcom-next-*", + "srcpkg": "linux-qcom-next", + "binpkg": "linux-image-qcom-next", + "kernel_config": "squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can", + "debian_version_stub": "0qli+staging1", + "debian_version_suffix": "~", + "pkg_linux_qcom_ref": "feat/matrix-expansion-packaging" + }, + { + "kernel_variant": "qcom-next", + "type": "Release", + "target_workspace": "qli-staging", + "suites": [ + "trixie", + "forky" + ], + "git_clone": "https://github.com/qualcomm-linux/kernel", + "branch_or_tag": "qcom-next-7.2-rc7-20260821", + "ref_strategy": "pinned_ref", + "srcpkg": "linux-qcom-next", + "binpkg": "linux-image-qcom-next", + "kernel_config": "squashfs,systemd-boot,qcom-imsdk,docker,qemu-boot,usb-can", + "debian_version_stub": "0qli+staging1", + "debian_version_suffix": "", + "pkg_linux_qcom_ref": "feat/matrix-expansion-packaging" + } + ] +} diff --git a/ci/scripts/derive-debian-revision.sh b/ci/scripts/derive-debian-revision.sh new file mode 100755 index 0000000..e27a385 --- /dev/null +++ b/ci/scripts/derive-debian-revision.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +set -euo pipefail + +# Derive the suite-specific Debian revision for one delivery leg. +# +# Formula: +# debian_revision = stub + suite_suffix_mapping[suite] + delivery_suffix +# delivery_suffix: Daily -> "~", Release -> "" +# +# This is the single implementation of the formula. It is called both by +# resolve-matrix.sh (once per flattened Daily/Release leg) and by +# build-kernel-deb.yml's direct-dispatch path (one suite, no full matrix +# context), so the derivation and its validation live in exactly one place. +# +# Usage: +# ci/scripts/derive-debian-revision.sh --stub 0qli+staging1 --suite trixie --delivery-type Daily +# ci/scripts/derive-debian-revision.sh --stub 0qli+staging1 --suite forky --delivery-type Release --matrix-file ci/build-matrix.json +# +# Options: +# --stub STUB Debian version stub. Must be non-empty and must not +# end in ~ (the delivery suffix supplies any +# trailing ~). Required. +# --suite SUITE Target suite; must have an entry in +# suite_suffix_mapping. Required. +# --delivery-type TYPE Daily or Release. Required. +# --matrix-file FILE Path to the matrix JSON containing +# suite_suffix_mapping +# (default: ci/build-matrix.json relative to CWD). +# +# Output: +# Final Debian revision printed to stdout. +# +# Exit codes: +# 0 Success. +# 1 Error (invalid args, malformed or missing suite_suffix_mapping, +# unmapped suite, unsupported delivery type). + +STUB="" +SUITE="" +DELIVERY_TYPE="" +MATRIX_FILE="ci/build-matrix.json" + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --stub) STUB="$2"; shift 2 ;; + --suite) SUITE="$2"; shift 2 ;; + --delivery-type) DELIVERY_TYPE="$2"; shift 2 ;; + --matrix-file) MATRIX_FILE="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "ERROR: Unknown option: $1" >&2; usage ;; + esac +done + +[[ -n "$STUB" ]] || { echo "ERROR: --stub is required" >&2; exit 1; } +[[ -n "$SUITE" ]] || { echo "ERROR: --suite is required" >&2; exit 1; } +[[ -n "$DELIVERY_TYPE" ]] || { echo "ERROR: --delivery-type is required" >&2; exit 1; } +[[ "$STUB" != *"~" ]] || { echo "ERROR: --stub must not end in ~ (got '$STUB')" >&2; exit 1; } +[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } + +jq empty "$MATRIX_FILE" 2>/dev/null \ + || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } + +mapping_errors=$(jq -r ' + .suite_suffix_mapping as $mapping | + if ($mapping | type) != "object" + then "suite_suffix_mapping is missing or not an object" + else + ( + [$mapping | to_entries[] | select(.value | type != "string") | "suite_suffix_mapping[" + .key + "] must be a string"] + + + [ + $mapping + | to_entries[] + | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) + | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" + ] + + + [ + $mapping + | to_entries + | group_by(.value) + | map(select(length > 1)) + | .[]? + | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" + ] + ) | .[] + end +' "$MATRIX_FILE") + +if [[ -n "$mapping_errors" ]]; then + echo "ERROR: Invalid suite_suffix_mapping in $MATRIX_FILE:" >&2 + while IFS= read -r error; do + [[ -n "$error" ]] && echo " - $error" >&2 + done <<< "$mapping_errors" + exit 1 +fi + +SUFFIX=$(jq -r --arg suite "$SUITE" '.suite_suffix_mapping[$suite] // "__MISSING__"' "$MATRIX_FILE") +[[ "$SUFFIX" != "__MISSING__" ]] || { + echo "ERROR: no suite_suffix_mapping entry for suite '$SUITE'" >&2 + exit 1 +} + +case "$DELIVERY_TYPE" in + Daily) DELIVERY_SUFFIX="~" ;; + Release) DELIVERY_SUFFIX="" ;; + *) + echo "ERROR: --delivery-type must be Daily or Release (got '$DELIVERY_TYPE')" >&2 + exit 1 + ;; +esac + +echo "${STUB}${SUFFIX}${DELIVERY_SUFFIX}" diff --git a/ci/scripts/derive-localversion.sh b/ci/scripts/derive-localversion.sh new file mode 100755 index 0000000..d36e9b0 --- /dev/null +++ b/ci/scripts/derive-localversion.sh @@ -0,0 +1,73 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +set -euo pipefail + +# Derive the LOCALVERSION suffix from a kernel variant and resolved ref. +# +# For dated tag builds (ref ends in -YYYYMMDD): +# Produces --. +# Example: qcom-next-7.2-rc3-20260722 -> -qcom-next-20260722 +# +# For branch-tip builds (ref does not end in a date): +# Uses the kernel variant and a short SHA for uniqueness. +# Example: qcom-next @ 07f50dc44edd -> -qcom-next-g07f50dc44edd +# --sha is required for branch-tip builds. +# +# Usage: +# ci/scripts/derive-localversion.sh --variant qcom-next --ref qcom-next-7.2-rc3-20260722 +# ci/scripts/derive-localversion.sh --variant arduino --ref main --sha 07f50dc44edd +# +# Options: +# --variant VARIANT Kernel variant identifier. Defaults to qcom-next. +# --ref REF Kernel ref (tag name or branch name). Required. +# --sha SHA Short commit SHA (required for branch-tip builds). +# +# Output: +# LOCALVERSION suffix printed to stdout (e.g. -qcom-next-20260722). +# Always starts with a dash. +# +# Exit codes: +# 0 Success. +# 1 Error (invalid args, branch-tip without --sha). + +VARIANT="qcom-next" +REF="" +SHA="" + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --variant) VARIANT="$2"; shift 2 ;; + --ref) REF="$2"; shift 2 ;; + --sha) SHA="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "ERROR: Unknown option: $1" >&2; usage ;; + esac +done + +[[ -n "$REF" ]] || { echo "ERROR: --ref is required" >&2; exit 1; } +[[ "$VARIANT" =~ ^[a-z0-9]+([a-z0-9-]*[a-z0-9])?$ ]] || { + echo "ERROR: --variant must use lowercase letters, digits, and internal hyphens" >&2 + exit 1 +} + +# Dated tags use a trailing YYYYMMDD snapshot. The matrix selects the tag set; +# the variant supplies the stable package identity used in LOCALVERSION. +if [[ "$REF" =~ -([0-9]{8})$ ]]; then + DATE="${BASH_REMATCH[1]}" + echo "-${VARIANT}-${DATE}" +else + # Branch-tip build: need SHA for uniqueness. + [[ -n "$SHA" ]] || { + echo "ERROR: --sha is required for branch-tip builds (ref '$REF' is not a dated tag)" >&2 + exit 1 + } + # Use first 12 chars of SHA for a compact but unambiguous suffix. + SHORT_SHA="${SHA:0:12}" + echo "-${VARIANT}-g${SHORT_SHA}" +fi diff --git a/ci/scripts/resolve-kernel-ref.sh b/ci/scripts/resolve-kernel-ref.sh new file mode 100755 index 0000000..351a4b2 --- /dev/null +++ b/ci/scripts/resolve-kernel-ref.sh @@ -0,0 +1,89 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +set -euo pipefail + +# Resolve the kernel git ref to build. +# +# In --latest-tag mode: queries the remote repository for tags matching a +# matrix-provided pattern and returns the one with the most recent trailing +# YYYYMMDD snapshot. Date-based sorting is used (not version sort) so that a +# newer-dated rc always wins over an older-dated final release. +# +# In --ref mode: validates the given ref is non-empty and returns it as-is. +# This is the passthrough path for pinned release builds. +# +# Usage: +# ci/scripts/resolve-kernel-ref.sh --url --latest-tag '' +# ci/scripts/resolve-kernel-ref.sh --url --ref +# +# Options: +# --url URL Kernel repository URL. Required. +# --latest-tag PATTERN Resolve the newest dated tag matching PATTERN. +# --ref REF Use this ref directly (passthrough for branch and +# pinned builds). +# +# Output: +# Resolved ref printed to stdout. +# +# Exit codes: +# 0 Success. +# 1 Error (missing args, no tags found, empty ref). + +URL="" +LATEST_TAG_PATTERN="" +REF="" + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --url) URL="$2"; shift 2 ;; + --latest-tag) LATEST_TAG_PATTERN="$2"; shift 2 ;; + --ref) REF="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "ERROR: Unknown option: $1" >&2; usage ;; + esac +done + +[[ -n "$URL" ]] || { echo "ERROR: --url is required" >&2; exit 1; } + +if [[ -n "$LATEST_TAG_PATTERN" && -n "$REF" ]]; then + echo "ERROR: --latest-tag and --ref are mutually exclusive" >&2 + exit 1 +fi + +if [[ -z "$LATEST_TAG_PATTERN" && -z "$REF" ]]; then + echo "ERROR: one of --latest-tag or --ref is required" >&2 + exit 1 +fi + +if [[ -n "$LATEST_TAG_PATTERN" ]]; then + # Query remote tags matching the matrix-provided pattern. + # Sort by the trailing 8-digit date (field after last -), pick the newest. + # Using date-based sort rather than version sort (-V) because version sort + # keys on the base kernel version first, causing an older-dated rc to + # outrank a newer-dated final release. + RESOLVED=$( + git ls-remote --tags "$URL" "refs/tags/${LATEST_TAG_PATTERN}" \ + | awk '{print $2}' \ + | sed 's|refs/tags/||' \ + | grep -v '\^{}' \ + | grep -E -- '-[0-9]{8}$' \ + | awk -F- '{print $NF"\t"$0}' \ + | sort -k1,1n \ + | tail -1 \ + | cut -f2- + ) + [[ -n "$RESOLVED" ]] || { + echo "ERROR: No dated tags matching '${LATEST_TAG_PATTERN}' found in $URL" >&2 + exit 1 + } + echo "$RESOLVED" +else + [[ -n "$REF" ]] || { echo "ERROR: --ref value is empty" >&2; exit 1; } + echo "$REF" +fi diff --git a/ci/scripts/resolve-matrix.sh b/ci/scripts/resolve-matrix.sh new file mode 100755 index 0000000..776f1df --- /dev/null +++ b/ci/scripts/resolve-matrix.sh @@ -0,0 +1,333 @@ +#!/bin/bash +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause-Clear +set -euo pipefail + +# Resolve and flatten the kernel delivery matrix for a given delivery type. +# +# The matrix root is an object with two top-level keys: +# - "suite_suffix_mapping": a suite -> Debian suffix map shared by every +# kernel variant and delivery type (e.g. "trixie": "~bpo13+1"). +# - "deliveries": the matrix rows. Each kernel_variant owns exactly one +# Daily row and one Release row. A row declares every input needed by +# that delivery, including a debian_version_stub; suites are the only +# list-valued field and are expanded into isolated legs. +# +# Each flattened leg's final debian_revision is derived from +# debian_version_stub, suite_suffix_mapping[suite], and the delivery type via +# ci/scripts/derive-debian-revision.sh, so the formula has exactly one +# implementation shared with build-kernel-deb.yml's direct-dispatch path. Each +# row also carries debian_version_suffix ("~" for Daily, "" for Release) as a +# visible, validated record of that same delivery-type mapping; it is checked +# against the row's type but never fed into derivation, so a copy/paste error +# here fails fast instead of silently drifting from the formula's single +# implementation. +# +# Usage: +# ci/scripts/resolve-matrix.sh --type Daily +# ci/scripts/resolve-matrix.sh --type Release +# ci/scripts/resolve-matrix.sh --type Daily --single-suite trixie +# ci/scripts/resolve-matrix.sh --type Daily --kernel-variant qcom-next +# ci/scripts/resolve-matrix.sh --type Daily --matrix-file path/to/matrix.json +# +# Options: +# --type TYPE Delivery type to filter (Daily or Release). +# Required. +# --single-suite SUITE Emit only entries for this suite. +# --kernel-variant VARIANT Emit only entries for this kernel variant. +# --matrix-file FILE Path to the matrix JSON file +# (default: ci/build-matrix.json relative to CWD). +# +# Output: +# Compact JSON array to stdout. Every entry has a single suite, the +# kernel_variant that scopes its artifacts, Debusine workspace, and logs, +# and a suite-specific debian_revision (debian_version_stub and +# debian_version_suffix are consumed and removed). +# +# Exit codes: +# 0 Success, at least one entry emitted. +# 1 Error (invalid arguments, matrix validation failure, no matching +# entry, revision derivation failure). + +TYPE="" +SINGLE_SUITE="" +KERNEL_VARIANT="" +MATRIX_FILE="ci/build-matrix.json" + +usage() { + sed -n '/^# Usage:/,/^$/p' "$0" | sed 's/^# \?//' + exit 1 +} + +while [[ $# -gt 0 ]]; do + case $1 in + --type) TYPE="$2"; shift 2 ;; + --single-suite) SINGLE_SUITE="$2"; shift 2 ;; + --kernel-variant) KERNEL_VARIANT="$2"; shift 2 ;; + --matrix-file) MATRIX_FILE="$2"; shift 2 ;; + -h|--help) usage ;; + *) echo "ERROR: Unknown option: $1" >&2; usage ;; + esac +done + +[[ "$TYPE" == "Daily" || "$TYPE" == "Release" ]] || { + echo "ERROR: --type must be Daily or Release" >&2 + exit 1 +} +[[ -f "$MATRIX_FILE" ]] || { echo "ERROR: Matrix file not found: $MATRIX_FILE" >&2; exit 1; } + +jq empty "$MATRIX_FILE" 2>/dev/null \ + || { echo "ERROR: Invalid JSON in $MATRIX_FILE" >&2; exit 1; } + +validation_errors=$(jq -r ' + def required_string($field): + if (has($field) and (.[$field] | type == "string") and (.[$field] | length > 0)) + then empty + else "missing or invalid " + $field + end; + + def optional_string($field): + if (has($field) | not) or (.[$field] | type == "string") + then empty + else "invalid " + $field + end; + + def variant_name_valid: + if (.kernel_variant | type) != "string" + then empty + elif (.kernel_variant | test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$")) + then empty + else "kernel_variant must use lowercase letters, digits, and internal hyphens" + end; + + def suites_valid: + if (.suites | type) != "array" or (.suites | length) == 0 + then "suites must be a non-empty array" + elif any(.suites[]; type != "string" or length == 0) + then "suites must contain only non-empty strings" + elif any(.suites[]; test("^[a-z0-9]+(?:[a-z0-9-]*[a-z0-9])?$") | not) + then "suites must use lowercase letters, digits, and internal hyphens" + elif ([.suites[]] | unique | length) != (.suites | length) + then "suites must not contain duplicates" + else empty + end; + + def row_errors($index): + if type != "object" + then "row " + ($index | tostring) + ": matrix entries must be objects" + else + . as $row | + [ + required_string("kernel_variant"), + required_string("type"), + required_string("git_clone"), + required_string("branch_or_tag"), + required_string("ref_strategy"), + required_string("srcpkg"), + required_string("binpkg"), + required_string("kernel_config"), + required_string("debian_version_stub"), + optional_string("pkg_linux_qcom_ref"), + optional_string("debusine_parent_workspace"), + optional_string("localversion"), + optional_string("kver_extra"), + variant_name_valid, + suites_valid, + if (.debian_version_stub | type) == "string" and (.debian_version_stub | test("~$")) + then "debian_version_stub must not end in ~" + else empty end, + if (has("debian_version_suffix") | not) or (.debian_version_suffix | type) != "string" + then "missing or invalid debian_version_suffix" + elif .type == "Daily" and .debian_version_suffix != "~" + then "debian_version_suffix must be \"~\" for Daily rows (got \"" + (.debian_version_suffix | tostring) + "\")" + elif .type == "Release" and .debian_version_suffix != "" + then "debian_version_suffix must be \"\" for Release rows (got \"" + (.debian_version_suffix | tostring) + "\")" + else empty end, + if (.type == "Daily" or .type == "Release") + then empty else "type must be Daily or Release" end, + if (.ref_strategy == "latest_tag" or .ref_strategy == "branch_tip" or .ref_strategy == "pinned_ref") + then empty else "ref_strategy must be latest_tag, branch_tip, or pinned_ref" end, + if .type == "Daily" and (.ref_strategy != "latest_tag" and .ref_strategy != "branch_tip") + then "Daily rows must use ref_strategy=latest_tag or ref_strategy=branch_tip" + elif .type == "Release" and .ref_strategy != "pinned_ref" + then "Release rows must use ref_strategy=pinned_ref" + else empty + end, + if .ref_strategy == "latest_tag" + then required_string("tag_pattern") + elif has("tag_pattern") + then "tag_pattern is only valid with ref_strategy=latest_tag" + else empty + end, + if .type == "Release" + then required_string("target_workspace") + elif has("target_workspace") + then "target_workspace is only valid for Release" + else empty + end + ] | .[] | "row " + ($index | tostring) + " (" + (($row.kernel_variant // "unknown") | tostring) + "): " + . + end; + + if type != "object" + then "matrix root must be an object with suite_suffix_mapping and deliveries" + elif (.deliveries | type) != "array" + then "deliveries must be an array" + elif (.deliveries | length) == 0 + then "deliveries must contain at least one row" + elif (.suite_suffix_mapping | type) != "object" + then "suite_suffix_mapping is missing or not an object" + else + .deliveries as $matrix | + .suite_suffix_mapping as $mapping | + ( + [range(0; ($matrix | length)) as $index | $matrix[$index] | row_errors($index)] + + + [ + [$matrix[] | select(type == "object")] + | group_by(.kernel_variant) + | .[] + | . as $rows + | (($rows[0].kernel_variant // "unknown") | tostring) as $variant + | ([ $rows[].type ] | sort) as $types + | ([ $rows[].srcpkg ] | unique) as $srcpkgs + | ([ $rows[].binpkg ] | unique) as $binpkgs + | ([ $rows[].debian_version_stub ] | unique) as $stubs + | if ($rows | length) != 2 + then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" + elif $types != ["Daily", "Release"] + then "kernel_variant " + $variant + " must define exactly one Daily row and one Release row" + elif ($srcpkgs | length) != 1 + then "kernel_variant " + $variant + " must use one srcpkg across its Daily and Release rows" + elif ($binpkgs | length) != 1 + then "kernel_variant " + $variant + " must use one binpkg across its Daily and Release rows" + elif ($stubs | length) != 1 + then "kernel_variant " + $variant + " must use one debian_version_stub across its Daily and Release rows" + else empty + end + ] + + + [ + [ + $matrix[] + | select(type == "object") + | select((.kernel_variant | type) == "string") + | select((.srcpkg | type) == "string" and (.srcpkg | length) > 0) + | {package: .srcpkg, kernel_variant: .kernel_variant} + ] + | group_by(.package)[] + | ([.[].kernel_variant] | unique) as $variants + | select($variants | length > 1) + | "srcpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) + ] + + + [ + [ + $matrix[] + | select(type == "object") + | select((.kernel_variant | type) == "string") + | select((.binpkg | type) == "string" and (.binpkg | length) > 0) + | {package: .binpkg, kernel_variant: .kernel_variant} + ] + | group_by(.package)[] + | ([.[].kernel_variant] | unique) as $variants + | select($variants | length > 1) + | "binpkg " + .[0].package + " is shared by kernel variants " + ($variants | join(", ")) + ] + + + [ + $mapping | to_entries[] | select(.value | type != "string") + | "suite_suffix_mapping[" + .key + "] must be a string" + ] + + + [ + $mapping + | to_entries[] + | select((.value | type == "string") and .value != "" and (.value | test("^~") | not)) + | "suite_suffix_mapping[" + .key + "] must be empty or start with ~ (got \"" + .value + "\")" + ] + + + [ + $mapping + | to_entries + | group_by(.value) + | map(select(length > 1)) + | .[]? + | "suites " + ([.[].key] | join(", ")) + " share the same suffix \"" + .[0].value + "\"" + ] + + + [ + [$matrix[] | select(type == "object") | select((.suites | type) == "array") | .suites[]] + | unique + | .[] as $suite + | select(($mapping | has($suite)) | not) + | "suite " + $suite + " has no suite_suffix_mapping entry" + ] + ) | .[] + end +' "$MATRIX_FILE") + +if [[ -n "$validation_errors" ]]; then + echo "ERROR: Invalid kernel delivery matrix:" >&2 + while IFS= read -r error; do + [[ -n "$error" ]] && echo " - $error" >&2 + done <<< "$validation_errors" + exit 1 +fi + +result=$(jq -c \ + --arg type "$TYPE" \ + --arg single_suite "$SINGLE_SUITE" \ + --arg kernel_variant "$KERNEL_VARIANT" ' + [ + .deliveries[] + | select(.type == $type) + | select($kernel_variant == "" or .kernel_variant == $kernel_variant) + | . as $row + | ( + if $single_suite == "" + then .suites + elif (.suites | index($single_suite)) != null + then [$single_suite] + else [] + end + )[] as $suite + | $row | del(.suites) | . + {"suite": $suite} + ] + | if length == 0 + then error( + "no matrix entries found for type=" + $type + + (if $kernel_variant != "" then " kernel_variant=" + $kernel_variant else "" end) + + (if $single_suite != "" then " suite=" + $single_suite else "" end) + ) + else . + end + ' "$MATRIX_FILE") || { + echo "ERROR: Matrix resolution failed for type=$TYPE${KERNEL_VARIANT:+ kernel_variant=$KERNEL_VARIANT}${SINGLE_SUITE:+ suite=$SINGLE_SUITE}" >&2 + exit 1 +} + +# Derive each leg's final debian_revision from debian_version_stub, +# suite_suffix_mapping, and its delivery type. derive-debian-revision.sh is +# the single implementation of the formula; build-kernel-deb.yml's direct +# dispatch path calls the same script for the one-suite, no-matrix case. +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +final="[]" +while IFS= read -r leg; do + suite=$(jq -r '.suite' <<< "$leg") + stub=$(jq -r '.debian_version_stub' <<< "$leg") + delivery_type=$(jq -r '.type' <<< "$leg") + variant=$(jq -r '.kernel_variant' <<< "$leg") + + revision=$("$script_dir/derive-debian-revision.sh" \ + --stub "$stub" --suite "$suite" --delivery-type "$delivery_type" \ + --matrix-file "$MATRIX_FILE") || { + echo "ERROR: Failed to derive Debian revision for kernel_variant=$variant suite=$suite type=$delivery_type" >&2 + exit 1 + } + + leg=$(jq -c --arg rev "$revision" '(. + {debian_revision: $rev}) | del(.debian_version_stub, .debian_version_suffix)' <<< "$leg") + final=$(jq -c --argjson leg "$leg" '. + [$leg]' <<< "$final") +done < <(jq -c '.[]' <<< "$result") + +echo "$final"