From 88f9f0a8edcafd582642314707dd752aadc93209 Mon Sep 17 00:00:00 2001 From: Simon Beaudoin Date: Tue, 28 Jul 2026 09:52:10 -0700 Subject: [PATCH 1/4] Re-enable sid container rebuild in CI Remove temporary rebuild skip logic so rebuild mode checks all discovered Debian and Ubuntu Dockerfiles again. Restore sid in the container workflow's dummy-package validation loop and Debian image push list. Signed-off-by: Simon Beaudoin --- .github/workflows/container-build-and-upload.yml | 3 ++- docker_deb_build.py | 6 ------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container-build-and-upload.yml b/.github/workflows/container-build-and-upload.yml index 15d82ca..29f6e0d 100644 --- a/.github/workflows/container-build-and-upload.yml +++ b/.github/workflows/container-build-and-upload.yml @@ -51,7 +51,7 @@ jobs: source_dir="${GITHUB_WORKSPACE}/.github/dummy-package" output_root="${GITHUB_WORKSPACE}/build/dummy-package" - for distro in noble resolute trixie; do + for distro in noble resolute trixie sid; do output_dir="${output_root}/${distro}" mkdir -p "${output_dir}" echo "Testing image '${distro}' with docker_deb_build.py" @@ -80,6 +80,7 @@ jobs: # DEBIAN IMAGES docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:trixie + docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid build-rpm-arm64: permissions: diff --git a/docker_deb_build.py b/docker_deb_build.py index 96e7aec..f310fac 100755 --- a/docker_deb_build.py +++ b/docker_deb_build.py @@ -30,9 +30,6 @@ # Example: ghcr.io/qualcomm-linux/pkg-builder:noble DOCKER_IMAGE_NAME_FMT = "ghcr.io/qualcomm-linux/pkg-builder:{suite_name}" -# Distros excluded from automatic rebuild (e.g. temporarily broken upstream) -SKIP_REBUILD_DISTROS = {"questing", "sid"} - def _discover_available_distros() -> list: """ Identify supported debian-based distros. @@ -299,9 +296,6 @@ def rebuild_docker_images(distro: str = None) -> None: # Rebuild all available debian-based distros dockerfiles = [] for distro in _discover_available_distros(): - if distro in SKIP_REBUILD_DISTROS: - logger.warning(f"Skipping rebuild for '{distro}' (listed in SKIP_REBUILD_DISTROS)") - continue dockerfile_glob = os.path.join(docker_dir, f'Dockerfile.*.*{distro}') dockerfiles.extend(glob.glob(dockerfile_glob)) dockerfiles = sorted(dockerfiles) From bad76226e437ba42f7d2451a0f42cc72127a9e2d Mon Sep 17 00:00:00 2001 From: Simon Beaudoin Date: Tue, 28 Jul 2026 09:59:50 -0700 Subject: [PATCH 2/4] Limit rebuild sweep to supported distros Align rebuild-all behavior with the supported distro set already exposed through --distro choices. This keeps temporary skip-list logic removed while preventing unsupported suites like questing from breaking the sweep. Signed-off-by: Simon Beaudoin --- docker_deb_build.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docker_deb_build.py b/docker_deb_build.py index f310fac..da5c467 100755 --- a/docker_deb_build.py +++ b/docker_deb_build.py @@ -29,6 +29,8 @@ # suite_name: 'noble', 'resolute', 'trixie', 'sid' # Example: ghcr.io/qualcomm-linux/pkg-builder:noble DOCKER_IMAGE_NAME_FMT = "ghcr.io/qualcomm-linux/pkg-builder:{suite_name}" +# Debian/Ubuntu suites currently supported by this wrapper for package builds. +SUPPORTED_DEB_DISTROS = ("noble", "resolute", "trixie", "sid") def _discover_available_distros() -> list: """ @@ -93,7 +95,7 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument("-d", "--distro", type=str, - choices=['noble', 'resolute', 'trixie', 'sid'], + choices=SUPPORTED_DEB_DISTROS, default=None, help="The target distribution for the package build (or rebuild if --rebuild is used). If not specified with --rebuild, all distros will be rebuilt.") @@ -293,9 +295,14 @@ def rebuild_docker_images(distro: str = None) -> None: raise Exception(f"No Dockerfile found for distro={distro}") logger.info(f"Rebuilding docker image for {distro}: {dockerfiles}") else: - # Rebuild all available debian-based distros + # Rebuild all currently supported debian-based distros. dockerfiles = [] for distro in _discover_available_distros(): + if distro not in SUPPORTED_DEB_DISTROS: + logger.warning( + f"Skipping rebuild for unsupported distro '{distro}'" + ) + continue dockerfile_glob = os.path.join(docker_dir, f'Dockerfile.*.*{distro}') dockerfiles.extend(glob.glob(dockerfile_glob)) dockerfiles = sorted(dockerfiles) From 34e1d067987f1c6b11990822319fbec82065250f Mon Sep 17 00:00:00 2001 From: Simon Beaudoin Date: Tue, 28 Jul 2026 12:04:03 -0700 Subject: [PATCH 3/4] Add forky and soften sid CI failures Add Debian forky as a supported suite in docker_deb_build, container workflow validation, and docker image upload paths. Introduce sid best-effort handling in container-build-and-upload: if sid image build or validation fails, continue the run and skip sid push while keeping other suites blocking. Signed-off-by: Simon Beaudoin --- .../workflows/container-build-and-upload.yml | 59 ++++++++++++++++--- AGENTS.md | 4 +- Dockerfiles/Dockerfile.debian.forky | 56 ++++++++++++++++++ Dockerfiles/sources/forky/qli.sources | 7 +++ README.md | 22 +++---- docker_deb_build.py | 6 +- 6 files changed, 131 insertions(+), 23 deletions(-) create mode 100644 Dockerfiles/Dockerfile.debian.forky create mode 100644 Dockerfiles/sources/forky/qli.sources diff --git a/.github/workflows/container-build-and-upload.yml b/.github/workflows/container-build-and-upload.yml index 29f6e0d..ccd969a 100644 --- a/.github/workflows/container-build-and-upload.yml +++ b/.github/workflows/container-build-and-upload.yml @@ -41,25 +41,60 @@ jobs: uses: actions/checkout@v6 - name: Build Debian Images - run: ./docker_deb_build.py --rebuild --no-update-check + shell: bash + run: | + set -euo pipefail + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" + + # Fresh marker for this job run + rm -f "${sid_failed_marker}" + + for distro in noble resolute trixie forky sid; do + echo "Building image '${distro}'" + if [[ "${distro}" == "sid" ]]; then + if ! ./docker_deb_build.py --rebuild --distro "${distro}" --no-update-check; then + echo "sid build failed; continuing without sid." + touch "${sid_failed_marker}" + fi + else + ./docker_deb_build.py --rebuild --distro "${distro}" --no-update-check + fi + done - name: Validate Debian Images Using local dummy package shell: bash run: | set -euo pipefail + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" source_dir="${GITHUB_WORKSPACE}/.github/dummy-package" output_root="${GITHUB_WORKSPACE}/build/dummy-package" - for distro in noble resolute trixie sid; do + for distro in noble resolute trixie forky sid; do + if [[ "${distro}" == "sid" && -f "${sid_failed_marker}" ]]; then + echo "Skipping sid validation because sid image build failed." + continue + fi + output_dir="${output_root}/${distro}" mkdir -p "${output_dir}" echo "Testing image '${distro}' with docker_deb_build.py" - ./docker_deb_build.py \ - --source-dir "${source_dir}" \ - --output-dir "${output_dir}" \ - --distro "${distro}" \ - --no-update-check + if [[ "${distro}" == "sid" ]]; then + if ! ./docker_deb_build.py \ + --source-dir "${source_dir}" \ + --output-dir "${output_dir}" \ + --distro "${distro}" \ + --no-update-check; then + echo "sid validation failed; continuing without sid." + touch "${sid_failed_marker}" + fi + else + ./docker_deb_build.py \ + --source-dir "${source_dir}" \ + --output-dir "${output_dir}" \ + --distro "${distro}" \ + --no-update-check + fi done - name: Log in to GHCR @@ -73,14 +108,22 @@ jobs: - name: Upload Debian Images # Only upload on trusted events that land code on main if: ${{ github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + shell: bash run: | + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" + # UBUNTU IMAGES docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:noble docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:resolute # DEBIAN IMAGES docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:trixie - docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid + docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:forky + if [[ -f "${sid_failed_marker}" ]]; then + echo "Skipping sid push because sid build/validation failed." + else + docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid + fi build-rpm-arm64: permissions: diff --git a/AGENTS.md b/AGENTS.md index e747e2b..d700eb4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,9 +29,9 @@ Dockerfiles/ | Suite | OS | sbuild backend | Chroot format | |-----------|--------|----------------|---------------| | noble | Ubuntu | unshare | `/root/.cache/sbuild/noble-arm64.tar` (mmdebstrap) | -| questing | Ubuntu | unshare | `/root/.cache/sbuild/questing-arm64.tar` (mmdebstrap) | | resolute | Ubuntu | unshare | `/root/.cache/sbuild/resolute-arm64.tar` (mmdebstrap) | | trixie | Debian | unshare | `/root/.cache/sbuild/trixie-arm64.tar` (mmdebstrap) | +| forky | Debian | unshare | `/root/.cache/sbuild/forky-arm64.tar` (mmdebstrap) | | sid | Debian | unshare | `/root/.cache/sbuild/sid-arm64.tar` (mmdebstrap) | ## Key Design Decisions @@ -46,7 +46,7 @@ Dockerfiles/ so that HTTPS APT repositories work inside the chroot at build time. - **Qualcomm APT sources**: - Ubuntu chroots include `qsc-deb-releases.sources` from `qartifactory-edge.qualcomm.com`. - - Debian `trixie` chroots include `qli.sources` from `deb.debusine.qualcomm.com`. + - Debian `trixie` and `forky` chroots include `qli.sources` from `deb.debusine.qualcomm.com`. - `sid` includes no default Qualcomm source and relies on caller-provided `--extra-repo` when needed. ## Common Commands diff --git a/Dockerfiles/Dockerfile.debian.forky b/Dockerfiles/Dockerfile.debian.forky new file mode 100644 index 0000000..c55b73e --- /dev/null +++ b/Dockerfiles/Dockerfile.debian.forky @@ -0,0 +1,56 @@ +# Use an official Debian base image +FROM debian:forky +LABEL org.opencontainers.image.source=https://github.com/qualcomm-linux/docker-pkg-build + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +COPY base-packages.txt /tmp/base-packages.txt + +RUN apt-get update && \ + apt-get install -y $(cat /tmp/base-packages.txt | tr '\n' ' ') && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# sbuild on forky is standardized on the "unshare" backend which requires: +# uidmap - newuidmap/newgidmap for user-namespace id mapping +# mmdebstrap - used below to create the chroot tarball +# Also provision subuid/subgid ranges for root so unshare can map ids. +RUN apt-get update && \ + apt-get install -y uidmap mmdebstrap && \ + if ! grep -q '^root:100000:65536$' /etc/subuid; then \ + echo 'root:100000:65536' | tee -a /etc/subuid; \ + fi && \ + if ! grep -q '^root:100000:65536$' /etc/subgid; then \ + echo 'root:100000:65536' | tee -a /etc/subgid; \ + fi + +COPY extra-packages.txt /tmp/extra-packages.txt +COPY keyrings/ /tmp/keyrings/ +COPY sources/forky/qli.sources /tmp/qli.sources + +RUN EXTRA_PACKAGES=$(tr -s '[:space:]' ',' < /tmp/extra-packages.txt) && \ + mkdir -p /root/.cache/sbuild && \ + mmdebstrap --variant=buildd \ + --arch=arm64 \ + --include="$EXTRA_PACKAGES" \ + --skip=output/mknod \ + --format=tar \ + --components=main,contrib,non-free,non-free-firmware \ + --customize-hook='mkdir -p "$1/etc/apt/keyrings" && cp /tmp/keyrings/*.asc "$1/etc/apt/keyrings/" && mkdir -p "$1/etc/apt/sources.list.d" && cp /tmp/qli.sources "$1/etc/apt/sources.list.d/"' \ + forky \ + /root/.cache/sbuild/forky-arm64.tar + +RUN mkdir -p /root/.config/sbuild && \ + printf '%s\n' \ + '$chroot_mode = "unshare";' \ + '$unshare_mmdebstrap_keep_tarball = 1;' \ + '$unshare_mmdebstrap_max_age = -1;' \ + '$unshare_mmdebstrap_auto_create = 0;' \ + > /root/.config/sbuild/config.pl + +# Set working directory +WORKDIR /workspace + +# Default command +CMD [ "bash" ] diff --git a/Dockerfiles/sources/forky/qli.sources b/Dockerfiles/sources/forky/qli.sources new file mode 100644 index 0000000..6e277be --- /dev/null +++ b/Dockerfiles/sources/forky/qli.sources @@ -0,0 +1,7 @@ +# Qualcomm Linux repository +Types: deb +URIs: https://deb.debusine.qualcomm.com/qualcomm/qli +Suites: forky +Components: main contrib non-free-firmware non-free +Signed-By: /etc/apt/keyrings/debusine.asc +Enabled: yes diff --git a/README.md b/README.md index 437a97c..e861a26 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # docker-pkg-build A Python-based tool for building Debian packages inside Docker containers, designed for Qualcomm Linux projects. -It supports ARM64 package builds across Ubuntu and Debian suites (`noble`, `questing`, `resolute`, `trixie`, and `sid`) to ensure consistent and reproducible package builds. +It supports ARM64 package builds across Ubuntu and Debian suites (`noble`, `resolute`, `trixie`, `forky`, and `sid`) to ensure consistent and reproducible package builds. This repo encompases two use cases: - Builder-agnostic local builds for a users wanting to build debian packages on their local machines in a repeatable way. @@ -57,9 +57,9 @@ You should then see the following: $ docker image ls REPOSITORY TAG IMAGE ID CREATED SIZE ghcr.io/qualcomm-linux/pkg-builder noble bdbf1ec3b9bf 2 hours ago 1.25GB -ghcr.io/qualcomm-linux/pkg-builder questing ac7b0936a006 About an hour ago 1.32GB ghcr.io/qualcomm-linux/pkg-builder resolute c41a1b076a1b About an hour ago 1.35GB ghcr.io/qualcomm-linux/pkg-builder trixie d00e2414b324 2 hours ago 1.47GB +ghcr.io/qualcomm-linux/pkg-builder forky 7dbd93bc2e4d About an hour ago 1.49GB ghcr.io/qualcomm-linux/pkg-builder sid 8090ef2d71cc About an hour ago 1.52GB ``` @@ -95,7 +95,7 @@ git clone git@github.com:qualcomm-linux/pkg-example.git mkdir build -debb --source-dir pkg-example --output-dir build --distro questing +debb --source-dir pkg-example --output-dir build --distro resolute ``` ## Usage @@ -110,7 +110,7 @@ docker_deb_build.py --help - **Docker-based Builds**: Packages are built inside isolated Docker containers to ensure reproducibility. - **Per-suite Builder Images**: Includes one Dockerfile and one prebuilt sbuild environment per supported suite. -- **Supported Suites**: Supports Ubuntu `noble`, `questing`, `resolute` and Debian `trixie`, `sid`. +- **Supported Suites**: Supports Ubuntu `noble`, `resolute` and Debian `trixie`, `forky`, `sid`. - **Unified sbuild Backend**: All Debian/Ubuntu suites use sbuild unshare tarballs created with mmdebstrap. - **Host-backed `/tmp` option for large builds**: `--host-tmp-dir` bind-mounts a host directory to container `/tmp`. - **Automated Workflows**: Integrates with GitHub Actions via the `qcom-container-build-and-upload.yml` workflow for CI/CD. @@ -131,7 +131,7 @@ For normal/smaller packages, this option is usually not necessary. docker_deb_build.py \ --source-dir pkg-camx \ --output-dir build \ - --distro questing \ + --distro resolute \ --host-tmp-dir /var/tmp/sbuild ``` @@ -140,23 +140,25 @@ docker_deb_build.py \ To add a new suite, copy an existing suite Dockerfile in `Dockerfiles/` and adapt it for the new release. Also add any suite-specific Qualcomm source files under `Dockerfiles/sources//`: - Ubuntu suites use `qsc-deb-releases.sources` -- Debian `trixie` uses `qli.sources` +- Debian `trixie` and `forky` use `qli.sources` The last step is to ensure the new image is also pushed to GHCR as part of the -`.github/workflows/qcom-container-build-and-upload.yml` workflow by adding a new line in the +`.github/workflows/container-build-and-upload.yml` workflow by adding a new line in the `Upload Debian Images` step: ``` docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:noble -docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:questing docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:resolute docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:trixie +docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:forky docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid # Add one more line for the new suite ``` +`sid` is best-effort in this workflow. If sid build or validation fails, CI +continues with the remaining suites and sid push is skipped. ### GitHub Workflow -The repository includes a `qcom-container-build-and-upload.yml` workflow (located in `.github/workflows/`) that automates building and uploading Docker containers for package builds. +The repository includes a `container-build-and-upload.yml` workflow (located in `.github/workflows/`) that automates building and uploading Docker containers for package builds. This workflow is automatically executed every week so that the GHCR registry where the images are stored contains a one-week-or-less old image. This keeps build time as small as possible for workflows relying on those images. This is because when building using sbuild, the first step is doing an apt update; the older the image, the longer it takes doing this apt upgrade. This also applies for non-github-workflow local builds; doing a **docker_deb_build.py --rebuild** periodically ensures a recent image and reduces the apt upgrade time at the start of every build. @@ -171,7 +173,7 @@ For whatever reason, you may have to enter the container in interactive mode. It Note: adapt the suite name and mounted paths for your scenario. ``` -docker run --rm -it --privileged -v /local/mnt/workspace/sbeaudoi/extra-repo/libdmabufheap-1.0.r1.03200:/workspace/src:Z -v /local/mnt/workspace/sbeaudoi/extra-repo/build:/workspace/output:Z -w /workspace/src --name pkg-builder-questing ghcr.io/qualcomm-linux/pkg-builder:questing bash +docker run --rm -it --privileged -v /local/mnt/workspace/sbeaudoi/extra-repo/libdmabufheap-1.0.r1.03200:/workspace/src:Z -v /local/mnt/workspace/sbeaudoi/extra-repo/build:/workspace/output:Z -w /workspace/src --name pkg-builder-resolute ghcr.io/qualcomm-linux/pkg-builder:resolute bash ``` ## Development diff --git a/docker_deb_build.py b/docker_deb_build.py index da5c467..6b64138 100755 --- a/docker_deb_build.py +++ b/docker_deb_build.py @@ -26,11 +26,11 @@ from color_logger import logger # Docker image name template -# suite_name: 'noble', 'resolute', 'trixie', 'sid' +# suite_name: 'noble', 'resolute', 'trixie', 'forky', 'sid' # Example: ghcr.io/qualcomm-linux/pkg-builder:noble DOCKER_IMAGE_NAME_FMT = "ghcr.io/qualcomm-linux/pkg-builder:{suite_name}" # Debian/Ubuntu suites currently supported by this wrapper for package builds. -SUPPORTED_DEB_DISTROS = ("noble", "resolute", "trixie", "sid") +SUPPORTED_DEB_DISTROS = ("noble", "resolute", "trixie", "forky", "sid") def _discover_available_distros() -> list: """ @@ -222,7 +222,7 @@ def build_docker_image(distro: str) -> bool: Build a Docker image from the local Dockerfile. Args: - distro (str): The distribution (e.g., 'noble', 'questing'). + distro (str): The distribution (e.g., 'noble', 'forky'). Returns: bool: True if the build succeeded, False otherwise. From 3f22dd55989c198a6b95930eaa0d54fb0a09796b Mon Sep 17 00:00:00 2001 From: Simon Beaudoin Date: Tue, 28 Jul 2026 12:39:25 -0700 Subject: [PATCH 4/4] Handle missing gh package on forky Forky currently does not expose the gh package in default apt metadata. Guard the package install step to drop gh when unavailable so the forky image can still build with the remaining base tooling. Signed-off-by: Simon Beaudoin --- Dockerfiles/Dockerfile.debian.forky | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Dockerfiles/Dockerfile.debian.forky b/Dockerfiles/Dockerfile.debian.forky index c55b73e..4bde7aa 100644 --- a/Dockerfiles/Dockerfile.debian.forky +++ b/Dockerfiles/Dockerfile.debian.forky @@ -8,7 +8,12 @@ ENV DEBIAN_FRONTEND=noninteractive COPY base-packages.txt /tmp/base-packages.txt RUN apt-get update && \ - apt-get install -y $(cat /tmp/base-packages.txt | tr '\n' ' ') && \ + BASE_PACKAGES=$(tr '\n' ' ' < /tmp/base-packages.txt) && \ + if ! apt-cache show gh >/dev/null 2>&1; then \ + echo "Package 'gh' is unavailable in forky; continuing without it."; \ + BASE_PACKAGES=$(grep -vx 'gh' /tmp/base-packages.txt | tr '\n' ' '); \ + fi && \ + apt-get install -y ${BASE_PACKAGES} && \ apt-get clean && \ rm -rf /var/lib/apt/lists/*