diff --git a/.github/workflows/make-profiles.yml b/.github/workflows/make-profiles.yml index 68d22876..8501f279 100644 --- a/.github/workflows/make-profiles.yml +++ b/.github/workflows/make-profiles.yml @@ -62,8 +62,10 @@ jobs: rm -rf "$commonBase" mkdir -p "$commonBase" - # Settings are ours outright: copied, not merged. - cp "$common/profile.conf" "$common/user-repos.conf" "$commonBase/" + # Settings are ours outright: copied, not merged. No user-repos.conf: + # build-iso.sh declares the build's repositories itself, and manjaro-tools + # concatenates the two, which registered [biglinux-stable] twice. + cp "$common/profile.conf" "$commonBase/" # Overlays keep the directory names manjaro-tools expects. cp -r "$common/overlays/root/" "$commonBase/root-overlay" diff --git a/README.md b/README.md index 01dc7c91..5069f336 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,6 @@ sources/ │ ├── Live-add Live-remove live session only │ ├── Mhwd-add Mhwd-remove driver packages │ ├── profile.conf hostname, live user, services -│ ├── user-repos.conf BigLinux pacman repositories │ └── overlays/root/ desktop/ live/ files shipped as-is └── editions/ ├── kde/ Desktop-add + special-commands.sh @@ -317,9 +316,15 @@ tree and the generated tree use different names for the same thing.
Where do the pacman repositories come from? -From `user-repos.conf`. A profile-level `pacman-default.conf` is **never read**, -which is precisely why this repository does not ship one — an unread config file -is worse than no config file, because people keep editing it and expecting -results. +From [`build-iso/build-iso.sh`](build-iso/build-iso.sh), and nowhere else. Its +`append_build_repos` writes them into the pacman config `buildiso` uses, in +priority order, and `set-biglinux-branch.sh` puts the matching list in the ISO's +own `pacman.conf` so the installed system updates from what it was built with. + +The profiles ship no `user-repos.conf` and no `pacman-default.conf`. The second +is **never read** by manjaro-tools at all, and the first *is* — it gets +concatenated onto the config the engine just wrote, which registered +`[biglinux-stable]` twice and had pacman drop one copy mid-build. The engine +removes any it finds. An unread config file is bad; a half-read one is worse.
diff --git a/biglinux/kde/user-repos.conf b/biglinux/kde/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/biglinux/kde/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch diff --git a/biglinux/xivastudio/user-repos.conf b/biglinux/xivastudio/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/biglinux/xivastudio/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch diff --git a/build-iso/README.md b/build-iso/README.md index 16c6c336..dd4f9f72 100644 --- a/build-iso/README.md +++ b/build-iso/README.md @@ -108,7 +108,7 @@ Stage by stage, in the order `main()` runs them: |:---|:---|:---| | `resolve_kernel` | turns `lts` / `latest` / `xanmod` into a real package name (`linux612`, `linux-xanmod`) | the profiles carry the placeholder `KERNEL`, not a version, so a kernel bump needs no commit | | `prepare_host` | installs the build tools and keyrings, creates loop devices, writes `manjaro-tools.conf` | the container is disposable and ships none of this | -| `configure_build_repos` | adds the BigLinux repositories to the *build* pacman config, sets the compression | the chroots must be able to install our packages, not only Manjaro's | +| `configure_build_repos` | adds the BigLinux repositories to the *build* pacman config, drops the profile's `user-repos.conf`, sets the compression | the chroots must be able to install our packages, not only Manjaro's — from one list, since manjaro-tools concatenates the profile's onto this one and pacman rejects the repeats | | `patch_manjaro_tools` | edits `util-iso.sh` / `util-iso-image.sh`: profile path, volume label, kernel check, image cleanups, live-user fix | none of these have a configuration knob upstream | | `configure_profile` | rewrites the *profile* in place: branch mirrors, volume label, `KERNEL` placeholders, `/etc/big-release` | the profile in git is branch-agnostic; the build makes it concrete | | `run_build` | `buildiso -p -b -k ` | the actual four hours | @@ -164,7 +164,7 @@ release build: | `WORK_PATH` | `/output` | where the ISO ends up | | `DISTRONAME` | detected | `biglinux` or `bigcommunity`, from the profile dirs | | `PROFILES_ROOT` | the checkout | build profiles living in another checkout | -| `BUILD_MIRROR` | `http://mirrors.manjaro.org/repo/` | the one mirror the whole Manjaro package set comes from | +| `BUILD_MIRROR` | `http://mirrors.manjaro.org/repo` | the one mirror the whole Manjaro package set comes from; trailing slashes are stripped | | `BIGLINUX_REPO_HOST` | `repo.biglinux.com.br` | host of the BigLinux repositories the **build** installs from | | `COMMUNITY_REPO_HOST` | `repo.communitybig.org` | same for the community repositories; bigcommunity only | | `MESA_TKG` | `false` | swap mesa for the TKG builds, on `latest` / `xanmod*` only (see `configure_profile`) | @@ -305,5 +305,7 @@ shellcheck -x build-iso/*.sh # must be clean |:---|:---| | **Build dies on disk space** | The chroots need ~15 GB under `/var/lib/manjaro-tools`, plus the ISO under `/var/cache/manjaro-tools`. On GitHub the workflow already mounts the runner's big `/mnt` disk there. | | **Packages older than expected** | The entire Manjaro package set comes from `BUILD_MIRROR` — `mkchroot` rewrites the mirrorlist down to that single URL. If that mirror lags, everything lags. Point `BUILD_MIRROR` at a fresh one. | +| **`404` retrieving a package pacman just resolved** | The database and the package files disagreed: the database named a version the mirror no longer serves. The known cause was a trailing slash in `BUILD_MIRROR`, which turned every URL into `repo//stable/…` — behind a CDN, an empty path segment is a cache key of its own, and the `core.db` cached under it was weeks old. `read_inputs` strips trailing slashes now; check the `--> mirror:` line of the build log for any other `//`. | +| **`could not register '' database (database already registered)`** | Two sections declared the same repository, and pacman kept the first. `append_build_repos` is the only place that may name one; if a profile ships a `user-repos.conf`, manjaro-tools concatenates it onto that same file. `configure_build_repos` deletes it for that reason. | | **xanmod version in the ISO name** | Only knowable *after* the build, so it is read back from the `.pkgs` list, producing names like `…_xanmod71.iso`. Not a bug, just chronology. | | **Black screen in the live session** | See `patch-live-setup.sh` above. The build is supposed to *fail* rather than produce one — so if you are looking at a black screen, somebody bypassed the engine. | diff --git a/build-iso/build-iso.sh b/build-iso/build-iso.sh index e0ea6d35..b5d9cbb4 100755 --- a/build-iso/build-iso.sh +++ b/build-iso/build-iso.sh @@ -97,7 +97,16 @@ read_inputs() { # mkchroot replaces the mirrorlist Include with this one Server, so it decides the # whole Manjaro package set. Unset, manjaro-tools picks mirror.easyname.at, which # lags behind stable. - BUILD_MIRROR="${BUILD_MIRROR:-http://mirrors.manjaro.org/repo/}" + # + # manjaro-tools appends `/$branch/$repo/$arch` to this value, so a trailing + # slash produces `repo//stable/core/...`. Behind a CDN that empty path segment + # is a cache key of its own, holding a copy of core.db nothing refreshes: the + # build resolves package versions that were current weeks ago and then 404s + # fetching them, an hour in. Normalised here so no caller can reintroduce it. + BUILD_MIRROR="${BUILD_MIRROR:-http://mirrors.manjaro.org/repo}" + while [[ "$BUILD_MIRROR" == */ ]]; do + BUILD_MIRROR="${BUILD_MIRROR%/}" + done # Overridable so a fork builds from its own repositories. BIGLINUX_REPO_HOST="${BIGLINUX_REPO_HOST:-repo.biglinux.com.br}" @@ -319,9 +328,13 @@ append_build_repos() { repo_section "$config_file" "bigiborg-${UNSTABLE_REPO_NAME:-}" \ "https://$UNSTABLE_REPO_MIRROR" fi + # The handful of packages held back from stable. Both distributions build + # with it; until it was here, biglinux got it from the profile's + # user-repos.conf instead -- see configure_build_repos for why that had to + # stop. + repo_section "$config_file" biglinux-update-stable \ + "https://$BIGLINUX_REPO_HOST/update-stable" if [[ "$DISTRONAME" == "bigcommunity" ]]; then - repo_section "$config_file" biglinux-update-stable \ - "https://$BIGLINUX_REPO_HOST/update-stable" append_community_repos "$config_file" fi if [[ "$BIGLINUX_BRANCH" == "testing" ]]; then @@ -342,6 +355,14 @@ configure_build_repos() { -e '/ParallelDownloads/s/ParallelDownloads =.*/ParallelDownloads = 10/' "$conf" done + # manjaro-tools hands pacman `cat pacman-.conf user-repos.conf`, so a + # repository the profile names as well as this function does is registered + # twice: `could not register 'biglinux-stable' database`, mid-build, with the + # profile's copy silently dropped. append_build_repos is the one owner of the + # build's repository list, and the profiles are a disposable clone, so the + # file goes rather than the two being kept in step forever. + rm -f "$PROFILE_PATH_EDITION/user-repos.conf" + # Development builds trade compression for speed; releases keep the # manjaro-tools default (level 20). Block size 1024K for both. if [[ "$DISTRO_BRANCH" != "stable" ]]; then diff --git a/build-iso/tests/test_engine_inputs.py b/build-iso/tests/test_engine_inputs.py index 55a1bb6a..b21a7576 100644 --- a/build-iso/tests/test_engine_inputs.py +++ b/build-iso/tests/test_engine_inputs.py @@ -72,11 +72,24 @@ def test_the_defaults_are_the_documented_ones(profiles): assert values["MANJARO_BRANCH"] == "stable" assert values["BIGLINUX_BRANCH"] == "stable" assert values["BIGCOMMUNITY_BRANCH"] == "stable" - assert values["BUILD_MIRROR"] == "http://mirrors.manjaro.org/repo/" + assert values["BUILD_MIRROR"] == "http://mirrors.manjaro.org/repo" assert values["BIGLINUX_REPO_HOST"] == "repo.biglinux.com.br" assert values["COMMUNITY_REPO_HOST"] == "repo.communitybig.org" +@pytest.mark.parametrize( + "given", + ["http://mirror.example.org/repo/", "http://mirror.example.org/repo///"], +) +def test_a_trailing_slash_is_stripped_from_the_build_mirror(profiles, given): + # manjaro-tools appends `/$branch/$repo/$arch`, so a trailing slash here + # becomes `repo//stable/core/...`. Behind a CDN that empty path segment caches + # a database of its own, which goes stale and makes the build resolve package + # versions the mirror no longer serves. + _, values, _ = run_stages(profiles, EDITION="kde", BUILD_MIRROR=given) + assert values["BUILD_MIRROR"] == "http://mirror.example.org/repo" + + def test_the_edition_can_come_from_the_first_argument(profiles): code, values, stderr = run_stages(profiles, args=("xivastudio",)) assert code == 0, stderr diff --git a/build-iso/tests/test_engine_repos.py b/build-iso/tests/test_engine_repos.py new file mode 100644 index 00000000..afb94158 --- /dev/null +++ b/build-iso/tests/test_engine_repos.py @@ -0,0 +1,99 @@ +# The repository list the build installs from. +# +# append_build_repos is its single owner. It used to share the job with the +# profiles' own user-repos.conf, which manjaro-tools concatenates onto the very +# file this function writes -- so [biglinux-stable] was declared twice and pacman +# refused the second copy with `could not register 'biglinux-stable' database` +# in the middle of a four-hour build. +# +# Order is the whole point of the function: pacman serves a package from the +# earliest section that has it, so these tests assert the sequence, not just the +# presence, of each repository. + +import subprocess + +import pytest +from conftest import SCRIPTS + +ENGINE = SCRIPTS / "build-iso.sh" + + +def sections(tmp_path, **env): + """Run append_build_repos over an empty file and list the sections it wrote.""" + config = tmp_path / "pacman.conf" + config.touch() + script = f'source "{ENGINE}"\nappend_build_repos "{config}"\n' + proc = subprocess.run( + ["bash", "-c", script], + check=False, + capture_output=True, + text=True, + env={ + "PATH": "/usr/bin:/bin", + "DISTRONAME": "biglinux", + "BIGLINUX_BRANCH": "stable", + "BIGCOMMUNITY_BRANCH": "stable", + "BIGLINUX_REPO_HOST": "repo.biglinux.com.br", + "COMMUNITY_REPO_HOST": "repo.communitybig.org", + **env, + }, + ) + assert proc.returncode == 0, proc.stderr + written = config.read_text(encoding="utf-8") + return [line.strip("[]") for line in written.splitlines() if line.startswith("[")], written + + +def test_biglinux_stable_builds_from_update_stable_above_stable(tmp_path): + # update-stable holds the packages held back from stable, so it has to + # outrank it. biglinux used to receive it from user-repos.conf, which landed + # it *below* stable and duplicated stable on the way. + listed, _ = sections(tmp_path) + assert listed == ["biglinux-update-stable", "biglinux-stable"] + + +def test_biglinux_testing_is_inserted_above_stable(tmp_path): + # BigLinux branches are additive: testing wins where it has a package and + # stable still answers for everything else. + listed, _ = sections(tmp_path, BIGLINUX_BRANCH="testing") + assert listed == ["biglinux-update-stable", "biglinux-testing", "biglinux-stable"] + + +def test_bigcommunity_keeps_its_own_repositories_between_the_two(tmp_path): + listed, _ = sections(tmp_path, DISTRONAME="bigcommunity", BIGCOMMUNITY_BRANCH="testing") + assert listed == [ + "biglinux-update-stable", + "community-testing", + "community-stable", + "community-extra", + "biglinux-stable", + ] + + +def test_the_development_repository_outranks_everything(tmp_path): + listed, _ = sections(tmp_path, UNSTABLE_REPO_MIRROR="repo.example.org/x", UNSTABLE_REPO_NAME="pr9") + assert listed[0] == "bigiborg-pr9" + + +def test_no_repository_is_declared_twice(tmp_path): + # The defect this file exists for: a name appearing in two sections is a + # database pacman declines to register, and the build carries on without it. + listed, _ = sections(tmp_path, DISTRONAME="bigcommunity", BIGLINUX_BRANCH="testing", + BIGCOMMUNITY_BRANCH="testing") + assert len(listed) == len(set(listed)) + + +@pytest.mark.parametrize("host_var,host", [ + ("BIGLINUX_REPO_HOST", "mirror.example.org"), + ("COMMUNITY_REPO_HOST", "community.example.org"), +]) +def test_a_fork_builds_from_its_own_host(tmp_path, host_var, host): + # So an ISO built from a fork's repositories also updates from them. + _, written = sections(tmp_path, DISTRONAME="bigcommunity", **{host_var: host}) + assert f"https://{host}/" in written + + +def test_the_arch_variable_reaches_pacman_unexpanded(tmp_path): + # `$arch` is pacman's own variable; expanded here every Server line would + # point at a directory that does not exist. + _, written = sections(tmp_path) + assert written.count("/$arch") == 2 diff --git a/build-iso/tests/test_engine_source.py b/build-iso/tests/test_engine_source.py index 12dcbae2..2d579b10 100644 --- a/build-iso/tests/test_engine_source.py +++ b/build-iso/tests/test_engine_source.py @@ -63,6 +63,16 @@ def test_the_build_mirror_is_configured_not_patched(): assert "manjaro-tools.conf" in ENGINE +def test_the_profiles_user_repos_conf_is_removed_before_the_build(): + # manjaro-tools hands pacman `cat pacman-.conf user-repos.conf`, and + # append_build_repos already wrote that first file: any repository named in + # both is registered twice and the second copy is dropped. Nothing in this + # checkout ships the file any more, but the community profiles still do, so + # the engine removes it rather than trusting every profile repository. + assert 'rm -f "$PROFILE_PATH_EDITION/user-repos.conf"' in ENGINE + assert not list(SCRIPTS.parent.glob("*/*/user-repos.conf")) + + def test_the_build_checkout_is_trusted_for_git_inside_the_container(): # GitHub Actions mounts the checkout with the runner's owner, while this # engine runs as root in the build container. buildiso validates the diff --git a/sources/common/user-repos.conf b/sources/common/user-repos.conf deleted file mode 100644 index e34a68a2..00000000 --- a/sources/common/user-repos.conf +++ /dev/null @@ -1,7 +0,0 @@ -[biglinux-update-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/update-stable/$arch - -[biglinux-stable] -SigLevel = PackageRequired -Server = https://repo.biglinux.com.br/stable/$arch