diff --git a/.github/workflows/container-build-and-upload.yml b/.github/workflows/container-build-and-upload.yml index 15d82ca..ccd969a 100644 --- a/.github/workflows/container-build-and-upload.yml +++ b/.github/workflows/container-build-and-upload.yml @@ -41,25 +41,60 @@ jobs: uses: actions/checkout@v6 - name: Build Debian Images - run: ./docker_deb_build.py --rebuild --no-update-check + shell: bash + run: | + set -euo pipefail + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" + + # Fresh marker for this job run + rm -f "${sid_failed_marker}" + + for distro in noble resolute trixie forky sid; do + echo "Building image '${distro}'" + if [[ "${distro}" == "sid" ]]; then + if ! ./docker_deb_build.py --rebuild --distro "${distro}" --no-update-check; then + echo "sid build failed; continuing without sid." + touch "${sid_failed_marker}" + fi + else + ./docker_deb_build.py --rebuild --distro "${distro}" --no-update-check + fi + done - name: Validate Debian Images Using local dummy package shell: bash run: | set -euo pipefail + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" source_dir="${GITHUB_WORKSPACE}/.github/dummy-package" output_root="${GITHUB_WORKSPACE}/build/dummy-package" - for distro in noble resolute trixie; do + for distro in noble resolute trixie forky sid; do + if [[ "${distro}" == "sid" && -f "${sid_failed_marker}" ]]; then + echo "Skipping sid validation because sid image build failed." + continue + fi + output_dir="${output_root}/${distro}" mkdir -p "${output_dir}" echo "Testing image '${distro}' with docker_deb_build.py" - ./docker_deb_build.py \ - --source-dir "${source_dir}" \ - --output-dir "${output_dir}" \ - --distro "${distro}" \ - --no-update-check + if [[ "${distro}" == "sid" ]]; then + if ! ./docker_deb_build.py \ + --source-dir "${source_dir}" \ + --output-dir "${output_dir}" \ + --distro "${distro}" \ + --no-update-check; then + echo "sid validation failed; continuing without sid." + touch "${sid_failed_marker}" + fi + else + ./docker_deb_build.py \ + --source-dir "${source_dir}" \ + --output-dir "${output_dir}" \ + --distro "${distro}" \ + --no-update-check + fi done - name: Log in to GHCR @@ -73,13 +108,22 @@ jobs: - name: Upload Debian Images # Only upload on trusted events that land code on main if: ${{ github.event_name == 'schedule' || github.event_name == 'push' || github.event_name == 'workflow_dispatch' }} + shell: bash run: | + sid_failed_marker="${RUNNER_TEMP}/sid-build-failed" + # UBUNTU IMAGES docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:noble docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:resolute # DEBIAN IMAGES docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:trixie + docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:forky + if [[ -f "${sid_failed_marker}" ]]; then + echo "Skipping sid push because sid build/validation failed." + else + docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid + fi build-rpm-arm64: permissions: diff --git a/AGENTS.md b/AGENTS.md index e747e2b..d700eb4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,9 +29,9 @@ Dockerfiles/ | Suite | OS | sbuild backend | Chroot format | |-----------|--------|----------------|---------------| | noble | Ubuntu | unshare | `/root/.cache/sbuild/noble-arm64.tar` (mmdebstrap) | -| questing | Ubuntu | unshare | `/root/.cache/sbuild/questing-arm64.tar` (mmdebstrap) | | resolute | Ubuntu | unshare | `/root/.cache/sbuild/resolute-arm64.tar` (mmdebstrap) | | trixie | Debian | unshare | `/root/.cache/sbuild/trixie-arm64.tar` (mmdebstrap) | +| forky | Debian | unshare | `/root/.cache/sbuild/forky-arm64.tar` (mmdebstrap) | | sid | Debian | unshare | `/root/.cache/sbuild/sid-arm64.tar` (mmdebstrap) | ## Key Design Decisions @@ -46,7 +46,7 @@ Dockerfiles/ so that HTTPS APT repositories work inside the chroot at build time. - **Qualcomm APT sources**: - Ubuntu chroots include `qsc-deb-releases.sources` from `qartifactory-edge.qualcomm.com`. - - Debian `trixie` chroots include `qli.sources` from `deb.debusine.qualcomm.com`. + - Debian `trixie` and `forky` chroots include `qli.sources` from `deb.debusine.qualcomm.com`. - `sid` includes no default Qualcomm source and relies on caller-provided `--extra-repo` when needed. ## Common Commands diff --git a/Dockerfiles/Dockerfile.debian.forky b/Dockerfiles/Dockerfile.debian.forky new file mode 100644 index 0000000..4bde7aa --- /dev/null +++ b/Dockerfiles/Dockerfile.debian.forky @@ -0,0 +1,61 @@ +# Use an official Debian base image +FROM debian:forky +LABEL org.opencontainers.image.source=https://github.com/qualcomm-linux/docker-pkg-build + +# Prevent interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +COPY base-packages.txt /tmp/base-packages.txt + +RUN apt-get update && \ + BASE_PACKAGES=$(tr '\n' ' ' < /tmp/base-packages.txt) && \ + if ! apt-cache show gh >/dev/null 2>&1; then \ + echo "Package 'gh' is unavailable in forky; continuing without it."; \ + BASE_PACKAGES=$(grep -vx 'gh' /tmp/base-packages.txt | tr '\n' ' '); \ + fi && \ + apt-get install -y ${BASE_PACKAGES} && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# sbuild on forky is standardized on the "unshare" backend which requires: +# uidmap - newuidmap/newgidmap for user-namespace id mapping +# mmdebstrap - used below to create the chroot tarball +# Also provision subuid/subgid ranges for root so unshare can map ids. +RUN apt-get update && \ + apt-get install -y uidmap mmdebstrap && \ + if ! grep -q '^root:100000:65536$' /etc/subuid; then \ + echo 'root:100000:65536' | tee -a /etc/subuid; \ + fi && \ + if ! grep -q '^root:100000:65536$' /etc/subgid; then \ + echo 'root:100000:65536' | tee -a /etc/subgid; \ + fi + +COPY extra-packages.txt /tmp/extra-packages.txt +COPY keyrings/ /tmp/keyrings/ +COPY sources/forky/qli.sources /tmp/qli.sources + +RUN EXTRA_PACKAGES=$(tr -s '[:space:]' ',' < /tmp/extra-packages.txt) && \ + mkdir -p /root/.cache/sbuild && \ + mmdebstrap --variant=buildd \ + --arch=arm64 \ + --include="$EXTRA_PACKAGES" \ + --skip=output/mknod \ + --format=tar \ + --components=main,contrib,non-free,non-free-firmware \ + --customize-hook='mkdir -p "$1/etc/apt/keyrings" && cp /tmp/keyrings/*.asc "$1/etc/apt/keyrings/" && mkdir -p "$1/etc/apt/sources.list.d" && cp /tmp/qli.sources "$1/etc/apt/sources.list.d/"' \ + forky \ + /root/.cache/sbuild/forky-arm64.tar + +RUN mkdir -p /root/.config/sbuild && \ + printf '%s\n' \ + '$chroot_mode = "unshare";' \ + '$unshare_mmdebstrap_keep_tarball = 1;' \ + '$unshare_mmdebstrap_max_age = -1;' \ + '$unshare_mmdebstrap_auto_create = 0;' \ + > /root/.config/sbuild/config.pl + +# Set working directory +WORKDIR /workspace + +# Default command +CMD [ "bash" ] diff --git a/Dockerfiles/sources/forky/qli.sources b/Dockerfiles/sources/forky/qli.sources new file mode 100644 index 0000000..6e277be --- /dev/null +++ b/Dockerfiles/sources/forky/qli.sources @@ -0,0 +1,7 @@ +# Qualcomm Linux repository +Types: deb +URIs: https://deb.debusine.qualcomm.com/qualcomm/qli +Suites: forky +Components: main contrib non-free-firmware non-free +Signed-By: /etc/apt/keyrings/debusine.asc +Enabled: yes diff --git a/README.md b/README.md index 437a97c..e861a26 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # docker-pkg-build A Python-based tool for building Debian packages inside Docker containers, designed for Qualcomm Linux projects. -It supports ARM64 package builds across Ubuntu and Debian suites (`noble`, `questing`, `resolute`, `trixie`, and `sid`) to ensure consistent and reproducible package builds. +It supports ARM64 package builds across Ubuntu and Debian suites (`noble`, `resolute`, `trixie`, `forky`, and `sid`) to ensure consistent and reproducible package builds. This repo encompases two use cases: - Builder-agnostic local builds for a users wanting to build debian packages on their local machines in a repeatable way. @@ -57,9 +57,9 @@ You should then see the following: $ docker image ls REPOSITORY TAG IMAGE ID CREATED SIZE ghcr.io/qualcomm-linux/pkg-builder noble bdbf1ec3b9bf 2 hours ago 1.25GB -ghcr.io/qualcomm-linux/pkg-builder questing ac7b0936a006 About an hour ago 1.32GB ghcr.io/qualcomm-linux/pkg-builder resolute c41a1b076a1b About an hour ago 1.35GB ghcr.io/qualcomm-linux/pkg-builder trixie d00e2414b324 2 hours ago 1.47GB +ghcr.io/qualcomm-linux/pkg-builder forky 7dbd93bc2e4d About an hour ago 1.49GB ghcr.io/qualcomm-linux/pkg-builder sid 8090ef2d71cc About an hour ago 1.52GB ``` @@ -95,7 +95,7 @@ git clone git@github.com:qualcomm-linux/pkg-example.git mkdir build -debb --source-dir pkg-example --output-dir build --distro questing +debb --source-dir pkg-example --output-dir build --distro resolute ``` ## Usage @@ -110,7 +110,7 @@ docker_deb_build.py --help - **Docker-based Builds**: Packages are built inside isolated Docker containers to ensure reproducibility. - **Per-suite Builder Images**: Includes one Dockerfile and one prebuilt sbuild environment per supported suite. -- **Supported Suites**: Supports Ubuntu `noble`, `questing`, `resolute` and Debian `trixie`, `sid`. +- **Supported Suites**: Supports Ubuntu `noble`, `resolute` and Debian `trixie`, `forky`, `sid`. - **Unified sbuild Backend**: All Debian/Ubuntu suites use sbuild unshare tarballs created with mmdebstrap. - **Host-backed `/tmp` option for large builds**: `--host-tmp-dir` bind-mounts a host directory to container `/tmp`. - **Automated Workflows**: Integrates with GitHub Actions via the `qcom-container-build-and-upload.yml` workflow for CI/CD. @@ -131,7 +131,7 @@ For normal/smaller packages, this option is usually not necessary. docker_deb_build.py \ --source-dir pkg-camx \ --output-dir build \ - --distro questing \ + --distro resolute \ --host-tmp-dir /var/tmp/sbuild ``` @@ -140,23 +140,25 @@ docker_deb_build.py \ To add a new suite, copy an existing suite Dockerfile in `Dockerfiles/` and adapt it for the new release. Also add any suite-specific Qualcomm source files under `Dockerfiles/sources//`: - Ubuntu suites use `qsc-deb-releases.sources` -- Debian `trixie` uses `qli.sources` +- Debian `trixie` and `forky` use `qli.sources` The last step is to ensure the new image is also pushed to GHCR as part of the -`.github/workflows/qcom-container-build-and-upload.yml` workflow by adding a new line in the +`.github/workflows/container-build-and-upload.yml` workflow by adding a new line in the `Upload Debian Images` step: ``` docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:noble -docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:questing docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:resolute docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:trixie +docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:forky docker push ghcr.io/${{env.QCOM_ORG_NAME}}/${{env.IMAGE_NAME}}:sid # Add one more line for the new suite ``` +`sid` is best-effort in this workflow. If sid build or validation fails, CI +continues with the remaining suites and sid push is skipped. ### GitHub Workflow -The repository includes a `qcom-container-build-and-upload.yml` workflow (located in `.github/workflows/`) that automates building and uploading Docker containers for package builds. +The repository includes a `container-build-and-upload.yml` workflow (located in `.github/workflows/`) that automates building and uploading Docker containers for package builds. This workflow is automatically executed every week so that the GHCR registry where the images are stored contains a one-week-or-less old image. This keeps build time as small as possible for workflows relying on those images. This is because when building using sbuild, the first step is doing an apt update; the older the image, the longer it takes doing this apt upgrade. This also applies for non-github-workflow local builds; doing a **docker_deb_build.py --rebuild** periodically ensures a recent image and reduces the apt upgrade time at the start of every build. @@ -171,7 +173,7 @@ For whatever reason, you may have to enter the container in interactive mode. It Note: adapt the suite name and mounted paths for your scenario. ``` -docker run --rm -it --privileged -v /local/mnt/workspace/sbeaudoi/extra-repo/libdmabufheap-1.0.r1.03200:/workspace/src:Z -v /local/mnt/workspace/sbeaudoi/extra-repo/build:/workspace/output:Z -w /workspace/src --name pkg-builder-questing ghcr.io/qualcomm-linux/pkg-builder:questing bash +docker run --rm -it --privileged -v /local/mnt/workspace/sbeaudoi/extra-repo/libdmabufheap-1.0.r1.03200:/workspace/src:Z -v /local/mnt/workspace/sbeaudoi/extra-repo/build:/workspace/output:Z -w /workspace/src --name pkg-builder-resolute ghcr.io/qualcomm-linux/pkg-builder:resolute bash ``` ## Development diff --git a/docker_deb_build.py b/docker_deb_build.py index 96e7aec..6b64138 100755 --- a/docker_deb_build.py +++ b/docker_deb_build.py @@ -26,12 +26,11 @@ from color_logger import logger # Docker image name template -# suite_name: 'noble', 'resolute', 'trixie', 'sid' +# suite_name: 'noble', 'resolute', 'trixie', 'forky', 'sid' # Example: ghcr.io/qualcomm-linux/pkg-builder:noble DOCKER_IMAGE_NAME_FMT = "ghcr.io/qualcomm-linux/pkg-builder:{suite_name}" - -# Distros excluded from automatic rebuild (e.g. temporarily broken upstream) -SKIP_REBUILD_DISTROS = {"questing", "sid"} +# Debian/Ubuntu suites currently supported by this wrapper for package builds. +SUPPORTED_DEB_DISTROS = ("noble", "resolute", "trixie", "forky", "sid") def _discover_available_distros() -> list: """ @@ -96,7 +95,7 @@ def parse_arguments() -> argparse.Namespace: parser.add_argument("-d", "--distro", type=str, - choices=['noble', 'resolute', 'trixie', 'sid'], + choices=SUPPORTED_DEB_DISTROS, default=None, help="The target distribution for the package build (or rebuild if --rebuild is used). If not specified with --rebuild, all distros will be rebuilt.") @@ -223,7 +222,7 @@ def build_docker_image(distro: str) -> bool: Build a Docker image from the local Dockerfile. Args: - distro (str): The distribution (e.g., 'noble', 'questing'). + distro (str): The distribution (e.g., 'noble', 'forky'). Returns: bool: True if the build succeeded, False otherwise. @@ -296,11 +295,13 @@ def rebuild_docker_images(distro: str = None) -> None: raise Exception(f"No Dockerfile found for distro={distro}") logger.info(f"Rebuilding docker image for {distro}: {dockerfiles}") else: - # Rebuild all available debian-based distros + # Rebuild all currently supported debian-based distros. dockerfiles = [] for distro in _discover_available_distros(): - if distro in SKIP_REBUILD_DISTROS: - logger.warning(f"Skipping rebuild for '{distro}' (listed in SKIP_REBUILD_DISTROS)") + if distro not in SUPPORTED_DEB_DISTROS: + logger.warning( + f"Skipping rebuild for unsupported distro '{distro}'" + ) continue dockerfile_glob = os.path.join(docker_dir, f'Dockerfile.*.*{distro}') dockerfiles.extend(glob.glob(dockerfile_glob))