From 11e3fed99773a50be3cd1e77f6253c651d39d589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Gon=C3=A7alves?= Date: Thu, 6 Aug 2026 23:32:26 -0300 Subject: [PATCH 1/2] fix: build from a mirror URL with no empty path segment A stable BigLinux KDE build died an hour in, retrieving packages pacman had just resolved: error: failed retrieving file 'linux618-6.18.39-1-x86_64.pkg.tar.zst' ... 404 error: failed retrieving file 'vtk-9.6.2-3-x86_64.pkg.tar.zst' ... 404 ==> ERROR: Failed to install packages to new root BUILD_MIRROR defaulted to `http://mirrors.manjaro.org/repo/`, and manjaro-tools appends `/${target_branch}` to it (mkchroot -B "${build_mirror}/${target_branch}", util-iso.sh), so every request went to `repo//stable/...`. To the CDN in front of mirrors.manjaro.org that empty path segment is a cache key of its own, and nothing refreshes it: the core.db served under it was two weeks old and named kernel and vtk versions the origin no longer carries, while the package files themselves came back fresh. Measured at the time of the failure: /repo/stable/core/x86_64/core.db Last-Modified 06 Aug linux618-6.18.42-1 /repo//stable/core/x86_64/core.db Last-Modified 23 Jul linux618-6.18.39-1 read_inputs strips trailing slashes now, so no caller can reintroduce it. The validation pattern already rejected an empty segment anywhere else in the path. --- build-iso/README.md | 3 ++- build-iso/build-iso.sh | 11 ++++++++++- build-iso/tests/test_engine_inputs.py | 15 ++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/build-iso/README.md b/build-iso/README.md index 16c6c336..3510a671 100644 --- a/build-iso/README.md +++ b/build-iso/README.md @@ -164,7 +164,7 @@ release build: | `WORK_PATH` | `/output` | where the ISO ends up | | `DISTRONAME` | detected | `biglinux` or `bigcommunity`, from the profile dirs | | `PROFILES_ROOT` | the checkout | build profiles living in another checkout | -| `BUILD_MIRROR` | `http://mirrors.manjaro.org/repo/` | the one mirror the whole Manjaro package set comes from | +| `BUILD_MIRROR` | `http://mirrors.manjaro.org/repo` | the one mirror the whole Manjaro package set comes from; trailing slashes are stripped | | `BIGLINUX_REPO_HOST` | `repo.biglinux.com.br` | host of the BigLinux repositories the **build** installs from | | `COMMUNITY_REPO_HOST` | `repo.communitybig.org` | same for the community repositories; bigcommunity only | | `MESA_TKG` | `false` | swap mesa for the TKG builds, on `latest` / `xanmod*` only (see `configure_profile`) | @@ -305,5 +305,6 @@ shellcheck -x build-iso/*.sh # must be clean |:---|:---| | **Build dies on disk space** | The chroots need ~15 GB under `/var/lib/manjaro-tools`, plus the ISO under `/var/cache/manjaro-tools`. On GitHub the workflow already mounts the runner's big `/mnt` disk there. | | **Packages older than expected** | The entire Manjaro package set comes from `BUILD_MIRROR` — `mkchroot` rewrites the mirrorlist down to that single URL. If that mirror lags, everything lags. Point `BUILD_MIRROR` at a fresh one. | +| **`404` retrieving a package pacman just resolved** | The database and the package files disagreed: the database named a version the mirror no longer serves. The known cause was a trailing slash in `BUILD_MIRROR`, which turned every URL into `repo//stable/…` — behind a CDN, an empty path segment is a cache key of its own, and the `core.db` cached under it was weeks old. `read_inputs` strips trailing slashes now; check the `--> mirror:` line of the build log for any other `//`. | | **xanmod version in the ISO name** | Only knowable *after* the build, so it is read back from the `.pkgs` list, producing names like `…_xanmod71.iso`. Not a bug, just chronology. | | **Black screen in the live session** | See `patch-live-setup.sh` above. The build is supposed to *fail* rather than produce one — so if you are looking at a black screen, somebody bypassed the engine. | diff --git a/build-iso/build-iso.sh b/build-iso/build-iso.sh index e0ea6d35..979c886f 100755 --- a/build-iso/build-iso.sh +++ b/build-iso/build-iso.sh @@ -97,7 +97,16 @@ read_inputs() { # mkchroot replaces the mirrorlist Include with this one Server, so it decides the # whole Manjaro package set. Unset, manjaro-tools picks mirror.easyname.at, which # lags behind stable. - BUILD_MIRROR="${BUILD_MIRROR:-http://mirrors.manjaro.org/repo/}" + # + # manjaro-tools appends `/$branch/$repo/$arch` to this value, so a trailing + # slash produces `repo//stable/core/...`. Behind a CDN that empty path segment + # is a cache key of its own, holding a copy of core.db nothing refreshes: the + # build resolves package versions that were current weeks ago and then 404s + # fetching them, an hour in. Normalised here so no caller can reintroduce it. + BUILD_MIRROR="${BUILD_MIRROR:-http://mirrors.manjaro.org/repo}" + while [[ "$BUILD_MIRROR" == */ ]]; do + BUILD_MIRROR="${BUILD_MIRROR%/}" + done # Overridable so a fork builds from its own repositories. BIGLINUX_REPO_HOST="${BIGLINUX_REPO_HOST:-repo.biglinux.com.br}" diff --git a/build-iso/tests/test_engine_inputs.py b/build-iso/tests/test_engine_inputs.py index 55a1bb6a..b21a7576 100644 --- a/build-iso/tests/test_engine_inputs.py +++ b/build-iso/tests/test_engine_inputs.py @@ -72,11 +72,24 @@ def test_the_defaults_are_the_documented_ones(profiles): assert values["MANJARO_BRANCH"] == "stable" assert values["BIGLINUX_BRANCH"] == "stable" assert values["BIGCOMMUNITY_BRANCH"] == "stable" - assert values["BUILD_MIRROR"] == "http://mirrors.manjaro.org/repo/" + assert values["BUILD_MIRROR"] == "http://mirrors.manjaro.org/repo" assert values["BIGLINUX_REPO_HOST"] == "repo.biglinux.com.br" assert values["COMMUNITY_REPO_HOST"] == "repo.communitybig.org" +@pytest.mark.parametrize( + "given", + ["http://mirror.example.org/repo/", "http://mirror.example.org/repo///"], +) +def test_a_trailing_slash_is_stripped_from_the_build_mirror(profiles, given): + # manjaro-tools appends `/$branch/$repo/$arch`, so a trailing slash here + # becomes `repo//stable/core/...`. Behind a CDN that empty path segment caches + # a database of its own, which goes stale and makes the build resolve package + # versions the mirror no longer serves. + _, values, _ = run_stages(profiles, EDITION="kde", BUILD_MIRROR=given) + assert values["BUILD_MIRROR"] == "http://mirror.example.org/repo" + + def test_the_edition_can_come_from_the_first_argument(profiles): code, values, stderr = run_stages(profiles, args=("xivastudio",)) assert code == 0, stderr From 972b450a678c85b4a36b716e036d471d24df882a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bruno=20Gon=C3=A7alves?= Date: Thu, 6 Aug 2026 23:33:29 -0300 Subject: [PATCH 2/2] fix: give the build's repository list a single owner Every build logged this, a few seconds before spending four hours ignoring it: error: could not register 'biglinux-stable' database (database already registered) Two places named the repositories. append_build_repos writes them into /usr/share/manjaro-tools/pacman-.conf, and manjaro-tools then hands pacman `cat pacman-.conf user-repos.conf` (get_pacman_conf, util-iso.sh) -- so the [biglinux-stable] the profiles also declared arrived twice and pacman dropped the second copy. [biglinux-update-stable], which only the profiles declared, was left below stable instead of above it, where the packages held back from stable have to be to win. So append_build_repos declares update-stable for both distributions now -- it already did for bigcommunity -- and configure_build_repos removes the profile's user-repos.conf before buildiso reads it. The file is gone from sources/ and from the generated profiles too; the community profiles still ship an empty one, which is why the engine removes rather than forbids it. No package changes hands: update-stable holds kernel-alive and latte-dock, neither of which exists in biglinux-stable, biglinux-testing or the Manjaro repositories, so its rank was never able to decide anything. It is ordered for the next package that lands there, not for these two. test_engine_repos.py covers the resulting order per distribution and branch, including the no-duplicates property the error above was reporting. --- .github/workflows/make-profiles.yml | 6 +- README.md | 15 ++-- biglinux/kde/user-repos.conf | 7 -- biglinux/xivastudio/user-repos.conf | 7 -- build-iso/README.md | 3 +- build-iso/build-iso.sh | 16 ++++- build-iso/tests/test_engine_repos.py | 99 +++++++++++++++++++++++++++ build-iso/tests/test_engine_source.py | 10 +++ sources/common/user-repos.conf | 7 -- 9 files changed, 139 insertions(+), 31 deletions(-) delete mode 100644 biglinux/kde/user-repos.conf delete mode 100644 biglinux/xivastudio/user-repos.conf create mode 100644 build-iso/tests/test_engine_repos.py delete mode 100644 sources/common/user-repos.conf diff --git a/.github/workflows/make-profiles.yml b/.github/workflows/make-profiles.yml index 68d22876..8501f279 100644 --- a/.github/workflows/make-profiles.yml +++ b/.github/workflows/make-profiles.yml @@ -62,8 +62,10 @@ jobs: rm -rf "$commonBase" mkdir -p "$commonBase" - # Settings are ours outright: copied, not merged. - cp "$common/profile.conf" "$common/user-repos.conf" "$commonBase/" + # Settings are ours outright: copied, not merged. No user-repos.conf: + # build-iso.sh declares the build's repositories itself, and manjaro-tools + # concatenates the two, which registered [biglinux-stable] twice. + cp "$common/profile.conf" "$commonBase/" # Overlays keep the directory names manjaro-tools expects. cp -r "$common/overlays/root/" "$commonBase/root-overlay" diff --git a/README.md b/README.md index 01dc7c91..5069f336 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,6 @@ sources/ │ ├── Live-add Live-remove live session only │ ├── Mhwd-add Mhwd-remove driver packages │ ├── profile.conf hostname, live user, services -│ ├── user-repos.conf BigLinux pacman repositories │ └── overlays/root/ desktop/ live/ files shipped as-is └── editions/ ├── kde/ Desktop-add + special-commands.sh @@ -317,9 +316,15 @@ tree and the generated tree use different names for the same thing.
Where do the pacman repositories come from? -From `user-repos.conf`. A profile-level `pacman-default.conf` is **never read**, -which is precisely why this repository does not ship one — an unread config file -is worse than no config file, because people keep editing it and expecting -results. +From [`build-iso/build-iso.sh`](build-iso/build-iso.sh), and nowhere else. Its +`append_build_repos` writes them into the pacman config `buildiso` uses, in +priority order, and `set-biglinux-branch.sh` puts the matching list in the ISO's +own `pacman.conf` so the installed system updates from what it was built with. + +The profiles ship no `user-repos.conf` and no `pacman-default.conf`. The second +is **never read** by manjaro-tools at all, and the first *is* — it gets +concatenated onto the config the engine just wrote, which registered +`[biglinux-stable]` twice and had pacman drop one copy mid-build. The engine +removes any it finds. An unread config file is bad; a half-read one is worse.
diff --git a/biglinux/kde/user-repos.conf b/biglinux/kde/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/biglinux/kde/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch diff --git a/biglinux/xivastudio/user-repos.conf b/biglinux/xivastudio/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/biglinux/xivastudio/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch diff --git a/build-iso/README.md b/build-iso/README.md index 3510a671..dd4f9f72 100644 --- a/build-iso/README.md +++ b/build-iso/README.md @@ -108,7 +108,7 @@ Stage by stage, in the order `main()` runs them: |:---|:---|:---| | `resolve_kernel` | turns `lts` / `latest` / `xanmod` into a real package name (`linux612`, `linux-xanmod`) | the profiles carry the placeholder `KERNEL`, not a version, so a kernel bump needs no commit | | `prepare_host` | installs the build tools and keyrings, creates loop devices, writes `manjaro-tools.conf` | the container is disposable and ships none of this | -| `configure_build_repos` | adds the BigLinux repositories to the *build* pacman config, sets the compression | the chroots must be able to install our packages, not only Manjaro's | +| `configure_build_repos` | adds the BigLinux repositories to the *build* pacman config, drops the profile's `user-repos.conf`, sets the compression | the chroots must be able to install our packages, not only Manjaro's — from one list, since manjaro-tools concatenates the profile's onto this one and pacman rejects the repeats | | `patch_manjaro_tools` | edits `util-iso.sh` / `util-iso-image.sh`: profile path, volume label, kernel check, image cleanups, live-user fix | none of these have a configuration knob upstream | | `configure_profile` | rewrites the *profile* in place: branch mirrors, volume label, `KERNEL` placeholders, `/etc/big-release` | the profile in git is branch-agnostic; the build makes it concrete | | `run_build` | `buildiso -p -b -k ` | the actual four hours | @@ -306,5 +306,6 @@ shellcheck -x build-iso/*.sh # must be clean | **Build dies on disk space** | The chroots need ~15 GB under `/var/lib/manjaro-tools`, plus the ISO under `/var/cache/manjaro-tools`. On GitHub the workflow already mounts the runner's big `/mnt` disk there. | | **Packages older than expected** | The entire Manjaro package set comes from `BUILD_MIRROR` — `mkchroot` rewrites the mirrorlist down to that single URL. If that mirror lags, everything lags. Point `BUILD_MIRROR` at a fresh one. | | **`404` retrieving a package pacman just resolved** | The database and the package files disagreed: the database named a version the mirror no longer serves. The known cause was a trailing slash in `BUILD_MIRROR`, which turned every URL into `repo//stable/…` — behind a CDN, an empty path segment is a cache key of its own, and the `core.db` cached under it was weeks old. `read_inputs` strips trailing slashes now; check the `--> mirror:` line of the build log for any other `//`. | +| **`could not register '' database (database already registered)`** | Two sections declared the same repository, and pacman kept the first. `append_build_repos` is the only place that may name one; if a profile ships a `user-repos.conf`, manjaro-tools concatenates it onto that same file. `configure_build_repos` deletes it for that reason. | | **xanmod version in the ISO name** | Only knowable *after* the build, so it is read back from the `.pkgs` list, producing names like `…_xanmod71.iso`. Not a bug, just chronology. | | **Black screen in the live session** | See `patch-live-setup.sh` above. The build is supposed to *fail* rather than produce one — so if you are looking at a black screen, somebody bypassed the engine. | diff --git a/build-iso/build-iso.sh b/build-iso/build-iso.sh index 979c886f..b5d9cbb4 100755 --- a/build-iso/build-iso.sh +++ b/build-iso/build-iso.sh @@ -328,9 +328,13 @@ append_build_repos() { repo_section "$config_file" "bigiborg-${UNSTABLE_REPO_NAME:-}" \ "https://$UNSTABLE_REPO_MIRROR" fi + # The handful of packages held back from stable. Both distributions build + # with it; until it was here, biglinux got it from the profile's + # user-repos.conf instead -- see configure_build_repos for why that had to + # stop. + repo_section "$config_file" biglinux-update-stable \ + "https://$BIGLINUX_REPO_HOST/update-stable" if [[ "$DISTRONAME" == "bigcommunity" ]]; then - repo_section "$config_file" biglinux-update-stable \ - "https://$BIGLINUX_REPO_HOST/update-stable" append_community_repos "$config_file" fi if [[ "$BIGLINUX_BRANCH" == "testing" ]]; then @@ -351,6 +355,14 @@ configure_build_repos() { -e '/ParallelDownloads/s/ParallelDownloads =.*/ParallelDownloads = 10/' "$conf" done + # manjaro-tools hands pacman `cat pacman-.conf user-repos.conf`, so a + # repository the profile names as well as this function does is registered + # twice: `could not register 'biglinux-stable' database`, mid-build, with the + # profile's copy silently dropped. append_build_repos is the one owner of the + # build's repository list, and the profiles are a disposable clone, so the + # file goes rather than the two being kept in step forever. + rm -f "$PROFILE_PATH_EDITION/user-repos.conf" + # Development builds trade compression for speed; releases keep the # manjaro-tools default (level 20). Block size 1024K for both. if [[ "$DISTRO_BRANCH" != "stable" ]]; then diff --git a/build-iso/tests/test_engine_repos.py b/build-iso/tests/test_engine_repos.py new file mode 100644 index 00000000..afb94158 --- /dev/null +++ b/build-iso/tests/test_engine_repos.py @@ -0,0 +1,99 @@ +# The repository list the build installs from. +# +# append_build_repos is its single owner. It used to share the job with the +# profiles' own user-repos.conf, which manjaro-tools concatenates onto the very +# file this function writes -- so [biglinux-stable] was declared twice and pacman +# refused the second copy with `could not register 'biglinux-stable' database` +# in the middle of a four-hour build. +# +# Order is the whole point of the function: pacman serves a package from the +# earliest section that has it, so these tests assert the sequence, not just the +# presence, of each repository. + +import subprocess + +import pytest +from conftest import SCRIPTS + +ENGINE = SCRIPTS / "build-iso.sh" + + +def sections(tmp_path, **env): + """Run append_build_repos over an empty file and list the sections it wrote.""" + config = tmp_path / "pacman.conf" + config.touch() + script = f'source "{ENGINE}"\nappend_build_repos "{config}"\n' + proc = subprocess.run( + ["bash", "-c", script], + check=False, + capture_output=True, + text=True, + env={ + "PATH": "/usr/bin:/bin", + "DISTRONAME": "biglinux", + "BIGLINUX_BRANCH": "stable", + "BIGCOMMUNITY_BRANCH": "stable", + "BIGLINUX_REPO_HOST": "repo.biglinux.com.br", + "COMMUNITY_REPO_HOST": "repo.communitybig.org", + **env, + }, + ) + assert proc.returncode == 0, proc.stderr + written = config.read_text(encoding="utf-8") + return [line.strip("[]") for line in written.splitlines() if line.startswith("[")], written + + +def test_biglinux_stable_builds_from_update_stable_above_stable(tmp_path): + # update-stable holds the packages held back from stable, so it has to + # outrank it. biglinux used to receive it from user-repos.conf, which landed + # it *below* stable and duplicated stable on the way. + listed, _ = sections(tmp_path) + assert listed == ["biglinux-update-stable", "biglinux-stable"] + + +def test_biglinux_testing_is_inserted_above_stable(tmp_path): + # BigLinux branches are additive: testing wins where it has a package and + # stable still answers for everything else. + listed, _ = sections(tmp_path, BIGLINUX_BRANCH="testing") + assert listed == ["biglinux-update-stable", "biglinux-testing", "biglinux-stable"] + + +def test_bigcommunity_keeps_its_own_repositories_between_the_two(tmp_path): + listed, _ = sections(tmp_path, DISTRONAME="bigcommunity", BIGCOMMUNITY_BRANCH="testing") + assert listed == [ + "biglinux-update-stable", + "community-testing", + "community-stable", + "community-extra", + "biglinux-stable", + ] + + +def test_the_development_repository_outranks_everything(tmp_path): + listed, _ = sections(tmp_path, UNSTABLE_REPO_MIRROR="repo.example.org/x", UNSTABLE_REPO_NAME="pr9") + assert listed[0] == "bigiborg-pr9" + + +def test_no_repository_is_declared_twice(tmp_path): + # The defect this file exists for: a name appearing in two sections is a + # database pacman declines to register, and the build carries on without it. + listed, _ = sections(tmp_path, DISTRONAME="bigcommunity", BIGLINUX_BRANCH="testing", + BIGCOMMUNITY_BRANCH="testing") + assert len(listed) == len(set(listed)) + + +@pytest.mark.parametrize("host_var,host", [ + ("BIGLINUX_REPO_HOST", "mirror.example.org"), + ("COMMUNITY_REPO_HOST", "community.example.org"), +]) +def test_a_fork_builds_from_its_own_host(tmp_path, host_var, host): + # So an ISO built from a fork's repositories also updates from them. + _, written = sections(tmp_path, DISTRONAME="bigcommunity", **{host_var: host}) + assert f"https://{host}/" in written + + +def test_the_arch_variable_reaches_pacman_unexpanded(tmp_path): + # `$arch` is pacman's own variable; expanded here every Server line would + # point at a directory that does not exist. + _, written = sections(tmp_path) + assert written.count("/$arch") == 2 diff --git a/build-iso/tests/test_engine_source.py b/build-iso/tests/test_engine_source.py index 12dcbae2..2d579b10 100644 --- a/build-iso/tests/test_engine_source.py +++ b/build-iso/tests/test_engine_source.py @@ -63,6 +63,16 @@ def test_the_build_mirror_is_configured_not_patched(): assert "manjaro-tools.conf" in ENGINE +def test_the_profiles_user_repos_conf_is_removed_before_the_build(): + # manjaro-tools hands pacman `cat pacman-.conf user-repos.conf`, and + # append_build_repos already wrote that first file: any repository named in + # both is registered twice and the second copy is dropped. Nothing in this + # checkout ships the file any more, but the community profiles still do, so + # the engine removes it rather than trusting every profile repository. + assert 'rm -f "$PROFILE_PATH_EDITION/user-repos.conf"' in ENGINE + assert not list(SCRIPTS.parent.glob("*/*/user-repos.conf")) + + def test_the_build_checkout_is_trusted_for_git_inside_the_container(): # GitHub Actions mounts the checkout with the runner's owner, while this # engine runs as root in the build container. buildiso validates the diff --git a/sources/common/user-repos.conf b/sources/common/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/sources/common/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch