diff --git a/.editorconfig b/.editorconfig index 1dd1eec2..dcb32673 100644 --- a/.editorconfig +++ b/.editorconfig @@ -104,7 +104,3 @@ end_of_line = unset insert_final_newline = unset trim_trailing_whitespace = unset charset = unset - -[lib/tests/*.plist] -indent_style = tab -insert_final_newline = unset diff --git a/.gitignore b/.gitignore index a02a713f..d9d8b923 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,7 @@ .sessions .vscode .zed + +# Generated by `nix flake check` / any nix command in this repo — +# deliberately NOT pinned (library flake; see AGENTS.md "Core lib tests"). +flake.lock diff --git a/AGENTS.md b/AGENTS.md index 00b8896d..eec6d741 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,7 +54,7 @@ needed regardless. And **never run a plain `icedos rebuild`** (that is a `switch ``` /config.toml? (+ configs/*.toml) (config.toml optional; root marked by flake.nix) - │ lib/load-user-config.nix (parse TOML, strict-merge) + │ lib/config/load-user-config.nix (parse TOML, strict-merge) ▼ icedos.* options modules/options.nix declares the schema │ lib/genflake.nix (evalModules → validate every value) @@ -77,7 +77,7 @@ nh os path:. `["modules"]`) are imported the same way. - User config beyond `config.toml` is autoloaded from the `icedos.system.extraConfigs` dirs (default `["configs"]`): every `*.toml` (including hidden `.*.toml`) is enumerated - by `lib/config-files.nix` and strict-merged in, with `config.toml` as the base. Hidden + by `lib/config/config-files.nix` and strict-merged in, with `config.toml` as the base. Hidden `.*.toml` are a **gitignore-only** channel — their values are plaintext in the store (and rollback snapshots), so treat them as private, not secret. Both options are bootstrap paths — read from `config.toml` only (like `system.arch`), or their @@ -104,11 +104,18 @@ Exposed to every module as **`icedosLib`**. |---|---| | `lib/options/helpers.nix` | The `mk*Option` family: `mkBoolOption`, `mkStrOption`, `mkStrListOption`, `mkNumberOption`, `mkEnumOption`, `mkIntBetweenOption`, `mkFloatBetweenOption`, `mkNullableOption`, `mkListOption`, `mkAttrsOfOption`, `mkSubmodule{,List,Attrs}Option`, `mkRecordOption`, `mkUsersOption`. | | `lib/options/validate.nix` | `validate.{int,float,enum,str,nonEmpty,list,requires,abort}` — rich, path-aware error messages. | -| `lib/helpers.nix` | `getModules`, `scanModules`, `hasModule`, `moduleInputName` (top-level generated-flake name of a module-declared input — the string-context twin of `_getModuleInputs`), `bash.prelude`, `bash.{blue,green,dim*}String`, `bash.requireConfigOwner` (permission guard for executing the baked `configurationLocation`; capture `ORIG_ARGS=("$@")` before arg parsing and only use where `$0` is the leaf command script), `toolset.mk{Dispatcher,BashCompletion,ZshCompletion,FishCompletion}`, `users.{getNormal,genDefaults,mkGroupInjector}`, `pkgs.{mapper,mkConfig,overlaysFromChannel}`, `packaging.{extractAppImage,installDesktopEntry}`, `mkInputName`, flake-revision helpers. `hasModule` aborts on a malformed call (no `name` and no `modules`, or an empty `modules = []`) — always pass a `name` or a non-empty `modules` list. | +| `lib/bash.nix` | `bash.{prelude,exportSystemPath,genHelpFlags,mkFlags,blueString,dimBlueString,greenString,dimGreenString,purpleString,dimPurpleString,redString,dimRedString,yellowString,dimYellowString,configSet,gcTimerCheckSnippet,requireConfigOwner}` — runtime shell helpers shared between Nix-embedded scripts and `prelude.sh` (color vars + the `*String` builders that emit `$(...)`-interpolated escape sequences — the **only** way the dispatcher/completions add color to command help text; `log_*`/`die`/`is_help_flag`; `bash.requireConfigOwner` is the permission guard for executing the baked `configurationLocation` — capture `ORIG_ARGS=("$@")` before arg parsing and only use where `$0` is the leaf command script). Also `injectIfExists` (emits `()` when a path exists — used by genflake for `/etc/nixos/extras.nix`). | +| `lib/toolset.nix` | `toolset.mk{Dispatcher,BashCompletion,ZshCompletion,FishCompletion}` — the CLI dispatcher generator (used to build `icedos` itself and every subcommand attrset that has children) + the per-shell completion generators. | +| `lib/users.nix` | `users.{getNormal,genDefaults,mkGroupInjector}`. | +| `lib/color.nix` | `color.hexToRgbInts`. | +| `lib/pkgs.nix` | `pkgs.{mapper,mkConfig,overlaysFromChannel}`. | +| `lib/packaging.nix` | `packaging.{extractAppImage,installDesktopEntry}` — shell-snippet builders for `installPhase`/`postFixup` bodies in icedos `package.nix` files. | +| `lib/scan.nix` | `getModules`, `scanModules`, `hasModule` — module discovery. `hasModule` aborts on a malformed call (no `name` and no `modules`, or an empty `modules = []`) — always pass a `name` or a non-empty `modules` list. | +| `lib/inputs.nix` | `moduleInputName` (sub-flake-relative path of a module-declared input — `"/"` — the string-context twin of `_getModuleInputs`; a **breaking change** from the old top-level name), `moduleSubFlakeName`, `mkInputName`, flake-revision helpers (`_resolveFlakeRevisionLocked` — pure tail given a lock + node key — and `_resolveFlakeRevisionNested`/`_resolveFlakeRevisionNestedLocked` — two-hop lookup for a sub-flake input, the latter driven directly by the tests), `_parseFlakeUrl`, `_getModuleKey`, `freshInputs`. | | `lib/icedos.nix` | `fetchModulesRepository`, `resolveExternalDependencyRecursively`, `modulesFromConfig` — the external-repo/dependency engine + input masking. Stamps every module's emitted NixOS config with `#` provenance (`setDefaultModuleLocation`) so nixpkgs eval/type/conflict errors name the source module instead of an anonymous generated location. Emitted module values are deduplicated (`_dedupeNixosModules`): each arrives wrapped in a `setDefaultModuleLocation` shim (`{ _file; imports = [ m ]; }`), and nixpkgs keys modules by `_file`/position, so two IceDOS modules emitting the SAME value would load it twice — core unwraps the shim (only pure `{ _file; imports = [ m ]; }` shims), keys the payload with `_opaqueOrKey` (a structural key, every shape tagged by `kind` (list/attrs/path/str/bool/int/float/null) so `{ }`≠`[ ]`, a path≠a plain string, and `42`≠`42.0`; functions, derivations, and `_type`-bearing property wrappers — `mkIf`/`mkMerge`/`mkForce`/option types — and anything containing them are opaque `null` and never merged; derivations are detected via `type` alone and never forced, since a derivation is cyclic and `drvPath` access can trigger instantiation, and `_type` wrappers are never descended into because the module system drops their unforced branches (`mkIf false`); depth-capped so any other cyclic value degrades to opaque instead of `max-call-depth exceeded`; and wrapped in `tryEval`, which degrades values that `throw`/`assert` when forced — an `abort`, missing attribute, or type error still propagates), and keeps the first occurrence per key at both the per-source flatten (`_extractNixosModules`) and the final external+extra combine (`modulesFromConfig.nixosModules`). Only the `nixosModules` output is deduplicated — `modulesFromConfig.options` (the option-doc index) is intentionally left as-is, and option-declaring payloads are opaque (`lib.mkOption` produces `{ _type = "option"; … }`), so duplicate option declarations still fail loudly rather than being silently merged. The common `inputs..nixosModules.default` case (a path) was already handled by nixpkgs' own identical-path dedup; this closes the identical-attrset-config-value gap (e.g. a shared function-free module emitted by two modules, or a future `nixosModules.default` that is a pure attrset). `modulesFromConfig` also exports `loadedModules` (repo url → module names, the fully-resolved set) which `genflake.nix` injects into the module system as the read-only `icedos.system.loadedModules`. Extra-modules share this: an `icedos.nix` extra-module is labeled `config#`; a plain `default.nix` extra-module is imported by path, so it already carries its real on-disk location. **Module `lib` field contributions:** any `icedos.nix` module — a configured repo's module or a config-root extra module — may extend `icedosLib` with a top-level `lib` field, usually `lib = import ./lib.nix { inherit icedosLib lib; };`. Core folds every contribution into the module-facing lib via `_mergeModuleLibs` (guarded: non-attrset contribution or a duplicate name = a named error). The merge is **two-phase**: during dependency resolution module files are imported with the **base** lib (phase 1 — only `meta` + contributions are forced); once the closure is known, `modulesFromConfig` computes `closureLib = _mergeModuleLibs (deduped ++ extraModulesP1)` over the **fully-resolved closure** and re-imports each module file's outputs (`externalOutputs`) plus the extra modules (`extraModulesP2`) with that merged lib (phase 2). The generated flake's `outputs.icedosLib` **and** `specialArgs.icedosLib` both reuse `modulesFromConfig.closureLib`, so module files and the module system share one merged lib within a single flake evaluation; `repl-context.nix` reads `flake.icedosLib`. A repo pulled in as a dependency — e.g. desktop, a **required** dep of every DE repo — still contributes its helpers because its always-loaded `default` module carries the `lib` field. A contribution file must live inside the kept set of `genflake.nix`'s `configRootKeep`/`configRootKeepDirs` (extra-module/config dirs, declared patches — a `builtins.path` keep-list, **not** git tracking): genflake imports config live, the build stage from the filtered snapshot, so an import that escapes the kept set evaluates at genflake and then fails at build with a bare missing-path error. Upgrade note: the old magic auto-discovery of a config-root `lib.nix` is gone — a user extends `icedosLib` from their own config by adding a `lib` field to one of their extra modules instead. Tradeoffs (inherent): each external module file is imported twice per stage (meta + contributions in phase 1, outputs in phase 2); the merge is evaluated at genflake stage, build-stage `specialArgs`, and repl (fresh per-stage evaluations, but `flake.icedosLib` shares one value with `specialArgs`). The bare `icedosLib` name stays a static set — the merge is a lazy member, so the `default.nix` probe (`attrNames (import icedos.nix …)`) never forces it. A contribution sees only the base lib (passing the merged lib would recurse); repo-to-repo composition happens at the module layer. | -| `lib/load-user-config.nix` | Parse `config.toml` + every `configs/*.toml` (enumerated by `lib/config-files.nix`), strict-merge (duplicate scalar key across files = error; lists concatenated). Top-level `icedos` is schema-validated by `modules/options.nix`; **every other top-level table is applied as raw NixOS config** (see passthrough below). | -| `lib/extra-options.nix` | `extraOptions.{marker,declare,inject}` — translates a user's `[extraOptions]` TOML table into real NixOS option declarations + genflake-stage value injection (see §6). | -| `lib/config-files.nix` | Bare `configRoot: [{rel;content;}]` — the ordered, pre-parsed config set (`config.toml` + each enabled `configs/*.toml`), shared by `load-user-config.nix` and `modules/options.nix` so both load the identical set. Applies the per-file `enable = false` opt-out and strips the `enable` key. | +| `lib/config/load-user-config.nix` | Parse `config.toml` + every `configs/*.toml` (enumerated by `lib/config/config-files.nix`), strict-merge (duplicate scalar key across files = error; lists concatenated). Top-level `icedos` is schema-validated by `modules/options.nix`; **every other top-level table is applied as raw NixOS config** (see passthrough below). | +| `lib/config/extra-options.nix` | `extraOptions.{marker,declare,inject}` — translates a user's `[extraOptions]` TOML table into real NixOS option declarations + genflake-stage value injection (see §6). | +| `lib/config/config-files.nix` | Bare `configRoot: [{rel;content;}]` — the ordered, pre-parsed config set (`config.toml` + each enabled `configs/*.toml`), shared by `load-user-config.nix` and `modules/options.nix` so both load the identical set. Applies the per-file `enable = false` opt-out and strips the `enable` key. | | `lib/common.nix` | `abortIf`, `filterByAttrs`, `findFirst`, `flatMap`, `generateAttrPath`, … | | `lib/constants.nix` | `ICEDOS_*` env/stage constants, `INPUTS_PREFIX`, `ENABLE_LOGGING` (either `ICEDOS_LOGGING=1` in the env **or** the `enableLogging` flag baked into the generated flake's lib import at genflake time — so `--logs` stays active for the whole nixos build even though the env var doesn't reach it). | | `lib/logger.nix` | `log`/`logValue`/`logAttrKeys` — active when `ENABLE_LOGGING` is set. | @@ -165,12 +172,39 @@ speedInBytes = true Optional module fields: - `inputs = { foo = { url = "…"; patches = [ … ]; }; };` — extra flake - inputs the module needs (merged into the generated state flake). The generated - top-level input name is namespaced to the declaring module — computed by - `icedosLib.moduleInputName { repo; module; input; }` — but the input is exposed to - every enabled module's `outputs.nixosModules` under the bare declared name `foo`. - A legacy `override = true` key is accepted and ignored (naming is now always - namespaced). + inputs the module needs. Each declaring module gets one thin **input-namespace + sub-flake** (`icedos-_`, a **content-addressed** `/nix/store/-` + flake.nix — no per-sub-flake files are written to disk; the sub-flake's text is + `pkgs.writeTextDir`'d + `builtins.path`'d at genflake time, so a decl change flips the + store path and a plain `nix flake lock` re-locks the sub root — and declared as a single + `path:` store input of the generated state flake); the module's declared inputs live inside it. The input's + sub-flake-relative path is computed by `icedosLib.moduleInputName { repo; module; input; }` + (`"/"` — the string-context twin of `_getModuleInputs`), and + each input is exposed to every enabled module's `outputs.nixosModules` under the + bare declared name `foo`. A module input's `follows` — at ANY depth: the + input's own `follows`, or anywhere in its `inputs` tree (a nested two-level + follows is legal flake syntax) — may only target an ambient top-level input + of the generated flake (nixpkgs, home-manager, icedos-config, + icedos-core, icedos-state, a configured channel, a url-mode overlay, an + extraFlake, or a sibling input of the same module); cross-module `follows` — + e.g. built from `moduleInputName`, which now yields a sub-flake-relative path — + abort at genflake naming the declarer. An input that declares both `url` and + `follows` also aborts (nix rejects a flake input with both a flake reference + and a follows attribute, so the sub-flake could not lock); a url-less + follows-only input stays legal. Two modules declaring the same bare input + name with **different** urls also abort (naming both declarers); the same url but + **different** patch sets aborts too, since the two would realise different trees + and the masked set (`listToAttrs` keyed by the bare name) would silently pick one. + (Two byte-identical but separately-vendored patch files still count as different + patch sets — the store paths differ — so an author should share one patch file, + not copy it.) A legacy + `override = true` key is accepted and ignored (naming is now always namespaced). + **Shadowing:** a `follows` whose first segment is also a declared sibling input + of the same module resolves to that declared input, *not* the ambient one — e.g. + a module that declares its own `nixpkgs` input makes any `follows = "nixpkgs"` + on its other inputs pull in the module's `nixpkgs`, not the generated flake's + (a slot is not emitted for a name the module already declares, so the follows + targets the sibling inside the sub-flake). - `meta.dependencies = [ { url?; modules = [ … ]; } ];` and `meta.optionalDependencies` — other modules this one needs (pulled automatically). @@ -241,7 +275,7 @@ the path must fill every normal user, in its `outputs.nixosModules` config: icedos..users = icedosLib.users.genDefaults { inherit (config.icedos) users; }; ``` -`genDefaults` (`lib/helpers.nix`, `users.genDefaults`) writes `{ = {}; … }` +`genDefaults` (`lib/users.nix`, `users.genDefaults`) writes `{ = {}; … }` for every `isNormalUser`, which triggers each submodule's own field defaults; explicit `[icedos..users.]` TOML stanzas still merge on top (submodule attrs merge). **Without it**, a user must hand-write an empty per-user stanza just to get defaults, and @@ -300,16 +334,35 @@ modules = [ "btop", "steam", "me3" ] # which modules to enable - `lib/icedos.nix:resolveExternalDependencyRecursively` walks each module's `meta.dependencies` so you only list what you directly want; deps come along. -- A module's declared `inputs` become flake inputs of the generated state flake - (patched via `pkgs.applyPatches` if `patches` is set). **Input masking** gives modules - stable names (`inputs.`, `inputs.self`) regardless of how the repo was fetched. +- A module's declared `inputs` live inside its per-module **input-namespace + sub-flake** (a **content-addressed** `/nix/store/-/flake.nix` — the + sub-flake text is `pkgs.writeTextDir`'d + `builtins.path`'d at genflake time, never + written to disk under `.state/`, and declared as a single `path:` store input of the + generated state flake — rewired to + the parent's ambient inputs via `inputs..inputs..follows`; patched + inputs become a `path:` node for the realised tree plus an upstream `_source` + node — see §5). `genflake.nix` emits the sub-flakes directly as the generated + `flake.nix`'s root `path:` inputs (no `subflakes.json` export — build.sh derives + sub-flake roots and their declared inputs from the resulting `flake.lock`), and + `--update-repos-inputs` refreshes + sub-flake inputs via `nix flake update "/" --refresh`. All lock steps run + against a **detached** copy of the state flake — `build.sh` rsyncs `.state` into a temp + dir (`mktemp`) and copies only the resulting `flake.lock` back, because a git flake + refuses to lock/refresh untracked `path:` inputs (`nix` says "git add ..."): locking in + `.state` would force every new or changed module input to be staged/committed. Sub-flakes + are never written as files and never committed; a changed sub-flake materializes a new + content-addressed store path, so a plain `nix flake lock` picks up new/edited module + inputs (preserving unchanged nested pins). Their nested inputs are inlined into the parent + `flake.lock`, so only that one file needs syncing. **Input masking** + gives modules stable names (`inputs.`, `inputs.self`) regardless of how the + repo was fetched. - Channels/overlays: `[[icedos.system.channels]]` and `[[icedos.system.overlays.fromChannel]]` add extra nixpkgs instances/overlays. ### Declaring user options from TOML: `[extraOptions]` A user can declare their **own** typed NixOS options without writing a Nix module — -purely from `config.toml` / `configs/*.toml`. `lib/extra-options.nix` translates the +purely from `config.toml` / `configs/*.toml`. `lib/config/extra-options.nix` translates the `[extraOptions]` table into real option declarations: - A node **with a `type` key** is a **leaf** — one declared option at its full dotted @@ -388,7 +441,7 @@ at your checkout, and enable/configure the module you touched) → run `icedos r **from wherever you are**. No `cd`, no `sudo`, no activation. You never switch — the user does. Core lib tests run as a flake check: `nix flake check` in the core repo evaluates -`lib/tests/tests.nix` and fails if any result is not "ok" (or the eval throws). +`tests/tests.nix` and fails if any result is not "ok" (or the eval throws). Core's `flake.lock` is gitignored and generated on demand (it is a library flake consumed via flake inputs, and a committed lock would pin core's own inputs — `nixpkgs`, `cache-server` — for consumers without `follows`). @@ -566,7 +619,7 @@ Environment a hook can rely on: | `ICEDOS_ROOT` | build app | the core store path. | | `ICEDOS_BUILD_DIR` | `build.sh` | temp build dir — set **after** `build.sh` starts, so **not** available in `preRebuild`/`preUpdate` (they run before it). | | `ICEDOS_HOOKS_ONLY=1` | `--update-hooks` only | tells `pre/postUpdate` that no HM activation follows, so they must complete standalone. | -| `ICEDOS_LOGGING` / `ICEDOS_STAGE` / `ICEDOS_UPDATE` | eval-internal | don't depend on these in runtime hooks. | +| `ICEDOS_LOGGING` / `ICEDOS_STAGE` / `ICEDOS_UPDATE` / `ICEDOS_UPDATE_MODULE_INPUTS` | eval-internal | don't depend on these in runtime hooks. | Order (`modules/rebuild.nix`): `--update-hooks` short-circuit (pre+postUpdate, then exit) → `preRebuild` → `preUpdate` (only with `--update`) → `build.sh` → `postUpdate` diff --git a/README.md b/README.md index dfc631ab..e06cb891 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ url = "github:numtide/flake-utils" inputs = { nixpkgs.follows = "nixpkgs" } ``` -A `name` must not collide with a module-declared input, a repository input, a `[[icedos.system.channels]]`/overlay input, or the framework-reserved set (`nixpkgs`, `home-manager`, `self`, `icedos-config`, `icedos-core`, `icedos-state`) — all of those become top-level inputs too, and a duplicate would silently overwrite. +A `name` must not collide with a module-declared input (its sub-flake name or its masked bare names), a repository input, a `[[icedos.system.channels]]`/overlay input, or the framework-reserved set (`nixpkgs`, `home-manager`, `self`, `icedos-config`, `icedos-core`, `icedos-state`) — extraFlakes become top-level inputs, module-declared inputs live as bare names inside their module's sub-flake, and a duplicate would silently overwrite. ### Hardware configuration @@ -480,8 +480,8 @@ With no flags this is a `switch`: it builds your configuration and activates it | `--update` | Update everything (core, nixpkgs, module repos, and module-declared inputs) in one blanket bump. | Full update. | | `--update-core` | Update IceDOS core, then re-run the command once. | Update IceDOS itself. | | `--update-nixpkgs` | Update the nixpkgs channel only. | Newer packages without touching modules. | -| `--update-repos` | Pull new revisions of the IceDOS module repos (e.g. `apps`, `hardware`). Does **not** re-lock inputs declared *inside* those modules. | Get the latest modules. | -| `--update-repos-inputs` | Re-lock every module-declared dependency. The only way to bump inputs defined inside module files. | Bump module dependencies without bumping nixpkgs. | +| `--update-repos` | Pull new revisions of the IceDOS module repos (e.g. `apps`, `hardware`). Does **not** re-lock inputs declared *inside* those modules — the sub-flake texts are generated before the repo bump in the same run, so if the bumped rev changes a module's declared inputs, those land on the **next** build (that build's genflake re-reads the new decls and the plain lock re-locks the changed sub-flake; one-build lag, self-healing). | Get the latest modules. | +| `--update-repos-inputs` | Re-lock every module-declared dependency (each module's inputs live in its own input-namespace sub-flake — a content-addressed store path; this bumps them via `nix flake update "/"`). The only way to bump inputs defined inside module files. | Bump module dependencies without bumping nixpkgs. | | `--update-hooks` | Run only the `preUpdate`/`postUpdate` hooks and exit — no build, no activation. | Refresh non-Nix things (e.g. `flatpak update`). | #### Behavior flags diff --git a/build.sh b/build.sh index 4a9442db..e17df5fe 100644 --- a/build.sh +++ b/build.sh @@ -128,7 +128,7 @@ if [ "$export_search_index" == "1" ]; then jsonfmt .cache/modules-doc.json -w # Full merged user config as JSON (config.toml + every enabled - # configs/*.toml — see lib/load-user-config.nix). The webui editor reads + # configs/*.toml — see lib/config/load-user-config.nix). The webui editor reads # this next to options-doc.json to tell which keys the user actually set # and to recover submodule-list values (repositories, users) the options # doc doesn't expand. @@ -170,40 +170,180 @@ if [ "$update_repos" == "1" ]; then refresh="--refresh" fi -# Generate flake -ICEDOS_UPDATE="$update_repos" ICEDOS_STAGE="genflake" nix eval $refresh $trace --file "$ICEDOS_ROOT/lib/genflake.nix" --raw flakeFinal >"$ICEDOS_STATE_DIR/$FLAKE" +# Genflake bakes the locked rev into patched `_source` urls so the realised +# patched tree matches the lock — but only when nothing is being updated. The +# repo bake (`_resolveFlakeRevision`, gated on ICEDOS_UPDATE) and the sub-flake +# bake (`_resolveFlakeRevisionNested`, gated on ICEDOS_UPDATE_MODULE_INPUTS) are +# cleared by SEPARATE flags: while --update-repos (resp. --update-repos-inputs) +# runs, the baked rev would pin the very input that flag is about to bump +# (`nix flake update "/_source"` on a ?rev=/rev-pinned url is a no-op). +# --update-all sets both. +update_flag="$update_repos" +update_module_inputs_flag="$update_repos_inputs" + +# Generate flake — one genflake eval producing the state flake and the +# sub-flake export together (evaluating the file per-attr would redo the whole +# config load + module resolution, and re-run patched-input store builds). +flake_final=$(ICEDOS_UPDATE="$update_flag" ICEDOS_UPDATE_MODULE_INPUTS="$update_module_inputs_flag" \ + ICEDOS_STAGE="genflake" nix eval --json $refresh $trace \ + --file "$ICEDOS_ROOT/lib/genflake.nix" \ + --apply 'g: g.flakeFinal') +jq -r . <<< "$flake_final" >"$ICEDOS_STATE_DIR/$FLAKE" nixfmt "$ICEDOS_STATE_DIR/$FLAKE" +# The sub-flake flake.nix texts live ONLY as content-addressed store paths +# (`/nix/store/-/flake.nix`) embedded in the generated flake's root +# inputs — no materialized `subflakes/` dir, no `subflakes.json` export. A +# changed sub-flake (decl added/edited, patch edited) changes its store path, +# so the plain `nix flake lock` below re-locks that one root node (preserving +# every unchanged nested pin). `flake.lock` is the single source of truth for +# which inputs belong to which sub-flake: the loops below derive sub-flake +# roots and their declared inputs from it. + +# Run every lock step against a DETACHED copy of the state flake, not the +# tracked tree: nix treats `.state` as a git flake, and a git flake refuses +# to lock/refresh an untracked `path:` input ("not tracked by Git ... git add +# ..."). In a plain directory outside any git work tree path inputs lock with +# no tracking requirement; the resulting `flake.lock` (which inlines every +# sub-flake's nested inputs) is copied back into `.state` once the lock steps +# below are done. The build itself already runs from a detached dir +# ($ICEDOS_BUILD_DIR), so a non-git state never leaks tracking requirements +# into the final `nh` call either. +lock_dir="$(mktemp -d -t icedos-lock-XXXXXXX-0)" +trap 'rm -rf "$lock_dir" 2>/dev/null || true' EXIT +rsync -a --exclude=".cache" "$ICEDOS_STATE_DIR/" "$lock_dir/" + +sync_lock() { + if [ -f "$lock_dir/flake.lock" ]; then + cp "$lock_dir/flake.lock" "$ICEDOS_STATE_DIR/flake.lock" + else + echo "warning: no flake.lock in detached lock dir — nothing to sync" >&2 + fi +} + +# Sub-flake root NAMES from a state lock: content-addressed `path:` root inputs +# whose store path ends with their own input name (genflake emits them as +# `builtins.path { name = subFlakeName; }` → `/nix/store/-`), +# excluding the framework's own icedos-config (store name is literally +# "icedos-config", so the suffix test would match it) and icedos-state (store +# name is "icedos", already rejected by the suffix test — kept explicit). +# Node keys are resolved via `nodes.root.inputs` — nix appends `_2`/`_3` to +# colliding node names, so a root input's key is not guaranteed to equal its +# name. A PATCHED repository is also a store-path `path:` root input, but its +# path ends `-patched` (not `-$k`), so the suffix test keeps it out — it is a +# repo, refreshed by the repo loops, and its own declared inputs are NOT +# module-declared inputs. Single source of truth for both the plain-build +# refresh and the --update-repos-inputs bump. +subflakes_from_lock() { + jq -r ' + . as $doc | $doc.nodes.root.inputs | to_entries[] + | .key as $k | .value as $key + | select(($key | type) == "string") + | select($doc.nodes[$key].locked.type == "path") + | select($doc.nodes[$key].locked.path | startswith("/nix/store/")) + | select($doc.nodes[$key].locked.path | endswith("-" + $k)) + | select($k != "icedos-config" and $k != "icedos-state") + | $k + ' "$1" 2>/dev/null +} ( set -e - cd "$ICEDOS_STATE_DIR" + cd "$lock_dir" + + # Create or refresh the lock. Sub-flake roots are content-addressed store + # paths: a changed sub-flake (module author added/edited an input, or edited + # patches) materializes a new `/nix/store/-` and this plain lock + # recompute re-locks that single root node — pulling in any new nested input + # — while PRESERVING every unchanged nested pin (`nix flake update ` + # would re-resolve the whole subtree to latest; a plain lock only updates + # nodes whose url changed). On a first build there is no lock file and this + # step creates it. When nothing changed it is a fast no-op. + nix flake lock - if [ ! -f flake.lock ] || [ -n "$update_core$update_nixpkgs$update_repos$update_repos_inputs" ]; then + # `nix flake prefetch-inputs` is only a store-warming optimization (the lock + # steps below fetch the same inputs on demand). It requires a lock file, so + # it runs after the plain lock above. Sub-flake roots are absolute store + # paths, so they prefetch fine. + if [ -n "$update_core$update_nixpkgs$update_repos$update_repos_inputs" ]; then nix flake prefetch-inputs fi - # Refresh every `type: "path"` input on each build so local sibling- - # repo edits (e.g. overrideUrl = "path:..." in config.toml) land - # without requiring --update-repos. github / git inputs stay pinned - # to their lock entries so we don't pay a network roundtrip per - # rebuild. - for input in $(jq -r '.nodes | to_entries[] | select(.value.locked.type == "path") | .key' flake.lock 2>/dev/null); do + # Refresh every non-store `type: "path"` ROOT input on each build so local + # sibling-repo edits (e.g. overrideUrl = "path:..." in config.toml) land + # without requiring --update-repos. github / git inputs stay pinned to their + # lock entries so we don't pay a network roundtrip per rebuild. Store-path + # root inputs are content-addressed — sub-flake roots, the masked + # icedos-config, and icedos-state (`path:${...}` interpolation copies the + # source at genflake): their path changes with their content, so the plain + # `nix flake lock` above re-locks them exactly when the content changed, and + # refreshing one here would `nix flake update` a sub-flake root and + # re-resolve its whole nested subtree to latest, silently unpinning every + # module-declared input on an ordinary rebuild. The node key for a root + # input name is resolved via `nodes.root.inputs` — nix appends `_2`/`_3` to + # colliding node names, so the input name is not guaranteed to be its key. + for input in $(jq -r '. as $doc | $doc.nodes.root.inputs | to_entries[] | .key as $k | .value as $key | select(($key | type) == "string") | select($doc.nodes[$key].locked.type == "path") | $k' flake.lock 2>/dev/null); do + locked_path=$(jq -r --arg k "$input" '.nodes.root.inputs[$k] as $key | select(($key | type) == "string") | .nodes[$key].locked.path // ""' flake.lock) + case "$locked_path" in + /nix/store/*) continue ;; + esac nix flake update "$input" 2>/dev/null || true done + # Same guarantee for module-declared local `path:` inputs, which live NESTED + # inside sub-flake roots: the root-input loop above cannot see them, and a + # plain `nix flake lock` does not re-copy a local dir whose url string is + # unchanged — the nested node keeps its stale narHash until an explicit + # update. Walk every sub-flake root's string-valued inputs and refresh the + # non-store `path:` ones so local module-input edits land on every build — + # restoring the pre-refactor all-nodes loop's behaviour. github / git nested + # inputs stay pinned; store-path nested inputs (patched trees, `_source` + # bakes) are content-addressed and the plain lock already re-locks them on + # change. + for sub in $(subflakes_from_lock flake.lock); do + for input in $(jq -r --arg sub "$sub" ' + . as $doc + | $doc.nodes.root.inputs[$sub] as $key + | select(($key | type) == "string") + | $doc.nodes[$key].inputs | to_entries[] + | select(.value | type == "string") + | .key as $in | .value as $lk + | select($doc.nodes[$lk].locked.type == "path") + | select(($doc.nodes[$lk].locked.path | startswith("/nix/store/")) | not) + | $in + ' flake.lock 2>/dev/null); do + nix flake update "$sub/$input" 2>/dev/null || true + done + done + [ "$update_core" == "1" ] && nix flake update icedos-core --refresh 2>/dev/null || true ) if [ "$update_all" == "1" ]; then ( set -e - cd "$ICEDOS_STATE_DIR" + cd "$lock_dir" nix flake update --refresh ) elif [ "$update_repos_inputs" == "1" ]; then ( set -e - cd "$ICEDOS_STATE_DIR" + cd "$lock_dir" + # Repo inputs (icedos-) and sub-flake roots are both `icedos-` + # prefixed root inputs. Update the repos, then bump each module-declared + # input nested inside its sub-flake (`/`); a path-typed + # (pinned) entry relocks trivially. Sub-flake roots come from + # `subflakes_from_lock` (same helper the plain-build refresh uses): the + # store-path `path:` root inputs whose store path ENDS with their own + # input name, excluding icedos-config/icedos-state and patched repos — + # everything derived from the lock, the single source of truth now that + # subflakes.json is gone. A sub's declared inputs are its lock node's + # `inputs` entries with a STRING value (a direct node reference); array + # values are `follows` — the sub rewires ambient slots (nixpkgs, + # home-manager, ...) to the parent's own inputs, and bumping one would + # re-resolve the followed chain and silently unpin e.g. nixpkgs. + mapfile -t subflakes < <(subflakes_from_lock flake.lock) + declare -A subflake_set + for sub in "${subflakes[@]}"; do subflake_set["$sub"]=1; done for input in $(jq -r ' .nodes.root.inputs | to_entries[] @@ -211,11 +351,62 @@ elif [ "$update_repos_inputs" == "1" ]; then | .key | select(startswith("icedos-")) ' flake.lock 2>/dev/null); do - nix flake update "$input" --refresh 2>/dev/null || true + # An associative-array membership test, not a `printf | grep` pipeline: + # under `set -o pipefail`, grep exiting on its first match SIGPIPEs printf + # (status 141) and turns the `!` into a false positive that updates anyway. + if [ -z "${subflake_set[$input]:-}" ]; then + nix flake update "$input" --refresh 2>/dev/null || true + fi done + for sub in "${subflakes[@]}"; do + for input in $(jq -r --arg sub "$sub" '.nodes.root.inputs[$sub] as $key | select(($key | type) == "string") | .nodes[$key].inputs | to_entries[] | select(.value | type == "string") | .key' flake.lock 2>/dev/null); do + nix flake update "$sub/$input" --refresh 2>/dev/null || true + done + done + ) +fi + +# Convergence pass after a module-input (or --update-all) bump. The genflake +# above ran with the sub-flake bake suppressed (`ICEDOS_UPDATE_MODULE_INPUTS`) +# so `nix flake update "/_source"` could actually move the pin — which +# means a PATCHED input's sub-flake embeds the realised tree built at genflake +# time (pre-bump rev), while the update loop just bumped the `_source` node in +# the lock. Without a re-genflake the system built right after the update would +# evaluate the pre-update patched tree while the lock records the new rev (one- +# build lag). Re-run genflake WITH the bakes re-enabled now that the lock holds +# the fresh revs: the patched `_source` urls bake the new rev, the patched +# trees are re-realised from it, the sub-flake texts change, their store paths +# flip, and the plain `nix flake lock` below re-locks exactly those roots — +# preserving every other pin. For unpatched inputs the sub-flake texts are +# verbatim decls, so this pass is a no-op apart from one extra eval. +if [ "$update_repos_inputs" == "1" ] || [ "$update_all" == "1" ]; then + # Copy the freshly-updated lock into `.state` FIRST so the re-genflake bakes + # the new revs (genflake reads the lock from ICEDOS_STATE_DIR). + sync_lock + flake_final=$(ICEDOS_UPDATE="" ICEDOS_UPDATE_MODULE_INPUTS="" \ + ICEDOS_STAGE="genflake" nix eval --json $trace \ + --file "$ICEDOS_ROOT/lib/genflake.nix" \ + --apply 'g: g.flakeFinal') + jq -r . <<< "$flake_final" >"$ICEDOS_STATE_DIR/$FLAKE" + nixfmt "$ICEDOS_STATE_DIR/$FLAKE" + # Re-sync the state flake into the detached dir and re-lock the changed + # sub-flake roots (new content-addressed store paths) with the same plain + # `nix flake lock` — unchanged nodes keep their pins. + rsync -a --exclude=".cache" "$ICEDOS_STATE_DIR/" "$lock_dir/" + ( + set -e + cd "$lock_dir" + nix flake lock ) fi +# Persist the locks written in the detached copy (only the state `flake.lock` +# changes — nested sub-flake inputs are inlined there, so no per-sub-flake +# lock files need copying). A failed lock step above exits via the trap +# WITHOUT syncing, leaving `.state` on its previous lock; the next run's plain +# lock step re-attempts any re-lock. +sync_lock + # Stop after the flake (and its lock) have been generated, without # building anything. Lets callers evaluate the generated flake (e.g. to # query per-package output paths) without realising the system closure. @@ -225,9 +416,12 @@ fi [ "$update_nixpkgs" == "1" ] && [ "$update_all" != "1" ] && ( set -e - cd "$ICEDOS_STATE_DIR" + cd "$lock_dir" nix flake update nixpkgs ) +sync_lock +rm -rf "$lock_dir" +trap - EXIT # Created here, not earlier: every path that exits before the build # (--genflake-only, --export-search-index) would otherwise leave an empty temp diff --git a/flake.nix b/flake.nix index 921c7ff3..8db5280c 100644 --- a/flake.nix +++ b/flake.nix @@ -36,7 +36,7 @@ _stateDir = if (isString stateDir) then stateDir else (throw "The value of `stateDir` should be a string."); - inherit (import ./lib/load-user-config.nix _configRoot) icedos; + inherit (import ./lib/config/load-user-config.nix _configRoot) icedos; system = icedos.system.arch or "x86_64-linux"; pkgs = nixpkgs.legacyPackages.${system}; @@ -116,7 +116,7 @@ }; }; - # Eval-only lib tests (`lib/tests/tests.nix`) as a flake check. Any result + # Eval-only lib tests (`tests/tests.nix`) as a flake check. Any result # value other than "ok" fails the derivation. checks = let @@ -135,7 +135,7 @@ r = builtins.tryEval value; in if r.success then r.value else "FAIL: ${name} threw during evaluation" - ) (import ./lib/tests/tests.nix { inherit (pkgs) lib; }); + ) (import ./tests/tests.nix { inherit (pkgs) lib; }); failures = lib.filterAttrs (_: value: value != "ok") results; in diff --git a/lib/bash.nix b/lib/bash.nix new file mode 100644 index 00000000..9cae855b --- /dev/null +++ b/lib/bash.nix @@ -0,0 +1,502 @@ +{ + icedosLib, + lib, + self, + ... +}: + +let + inherit (builtins) + foldl' + pathExists + stringLength + ; + + inherit (lib) + concatStringsSep + escapeShellArg + fileContents + max + ; +in +rec { + # Runtime bash helpers shared between Nix-embedded scripts (via the + # auto-prepended `prelude` from toolset.nix:41) and standalone .sh files + # (which `source` core/lib/prelude.sh directly). Both layers see the + # same color vars, log_* / die / is_help_flag functions. + bash = { + prelude = builtins.readFile ./prelude.sh; + + # PATH export used by icedos systemd user services that shell out to + # binaries from the host (e.g. systemctl, loginctl) and the user's + # per-user system profile (`/etc/profiles/per-user/$USER`, where home-manager + # installs packages under `useUserPackages`), in addition to whatever + # derivation the unit ships. The legacy `~/.nix-profile/bin` is kept as a + # harmless fallback (empty once `home.packages` move to the per-user profile). + # Spliced into writeShellScript bodies via `${icedosLib.bash.exportSystemPath}`. + exportSystemPath = '' + base_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + nix_system_path="/run/current-system/sw/bin" + nix_peruser_path="/etc/profiles/per-user/''${USER}/bin" + nix_user_path="''${HOME}/.nix-profile/bin" + export PATH="''${base_path}:''${nix_system_path}:''${nix_peruser_path}:''${nix_user_path}:$PATH" + ''; + + genHelpFlags = + { + excludeNoArgs ? false, + }: + let + base = ''"$1" == "--help" || "$1" == "-h" || "$1" == "help" || "$1" == "h"''; + in + if excludeNoArgs then base else ''"$1" == "" || '' + base; + + # Generate a bash arg parser from a Nix flag spec. Returns bash source that + # must be spliced into a script body (e.g. via `${icedosLib.bash.mkFlags {}}`). + # Toolset leaves get the prelude auto-injected, so `die` is always available; + # standalone consumers get a `die` polyfill at the top of the emitted block. + mkFlags = + { + prefix, + flags, + passthroughUnknown ? false, + }: + let + # Map flag name to bash variable name (e.g. "gpu-layers" → "LLAMACPP_GPU_LAYERS") + toVarName = name: "${prefix}_${lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] name)}"; + + # Map flag name to "was set" tracking variable + toSetVar = name: "${toVarName name}_SET"; + + # Short flag pattern for case arm, e.g. "-H|" or "" + shortPat = f: if f ? short then "-${f.short}|" else ""; + + # Flag spec for help text alignment + flagSpec = + f: + let + shortPart = if f ? short then "-${f.short}, " else " "; + flagPart = "--${f.name}"; + typePart = + if f.type == "string" then + " " + else if f.type == "int" then + " " + else if f.type == "bool" then + "" + else if f.type == "enum" then + " <${lib.concatStringsSep "|" f.choices}>" + else + ""; + in + "${shortPart}${flagPart}${typePart}"; + + # Compute max flag spec width for alignment + maxSpecLen = foldl' max 0 (map (f: stringLength (flagSpec f)) flags); + + # Help text line for one flag + helpLine = + f: + let + spec = flagSpec f; + pad = maxSpecLen - stringLength spec + 2; + defaultHelp = + if f.type == "bool" then + if f.default then "true" else "false" + else if f.type == "int" then + toString (builtins.floor f.default) + else + toString f.default; + in + " ${spec}${ + lib.concatStringsSep "" (lib.genList (_: " ") pad) + }${f.description} (default: ${defaultHelp})"; + + # Full help text with real newlines + helpText = "Flags:\n${lib.concatStringsSep "\n" (map helpLine flags)}"; + + # Escaped default value for bash + escDefault = + f: + if f.type == "bool" then + if f.default then "true" else "false" + else if f.type == "int" then + lib.escapeShellArg (toString (builtins.floor f.default)) + else + lib.escapeShellArg (toString f.default); + + # Variable declarations for one flag + varDecl = f: "${toVarName f.name}=${escDefault f}\n${toSetVar f.name}=0"; + + # Generate case arms for one flag + genCaseArm = + f: + let + var = toVarName f.name; + svar = toSetVar f.name; + long = "--${f.name}"; + sp = shortPat f; + in + if f.type == "bool" then + '' + ${sp}${long}) + ${var}="true"; ${svar}=1 + shift + ;; + --no-${f.name}) + ${var}="false"; ${svar}=1 + shift + ;; + ${long}=true|${long}=false) + ${var}="''${1#${long}=}"; ${svar}=1 + shift + ;; + '' + else + '' + ${sp}${long}) + [[ $# -ge 2 ]] || die "${long} requires a value" + ${lib.optionalString ( + f.type == "int" + ) ''[[ "$2" =~ ^-?[0-9]+$ ]] || die "${long} must be an integer"''} + ${lib.optionalString (f.type == "enum") '' + case "$2" in + ${lib.concatStringsSep "|" f.choices}) ;; + *) die "invalid value for ${long}: $2 (choose: ${lib.concatStringsSep ", " f.choices})" ;; + esac + ''} + ${var}="$2"; ${svar}=1 + shift 2 + ;; + ${long}=*) + v="''${1#${long}=}" + ${lib.optionalString ( + f.type == "int" + ) ''[[ "$v" =~ ^-?[0-9]+$ ]] || die "${long} must be an integer"''} + ${lib.optionalString (f.type == "enum") '' + case "$v" in + ${lib.concatStringsSep "|" f.choices}) ;; + *) die "invalid value for ${long}: $v (choose: ${lib.concatStringsSep ", " f.choices})" ;; + esac + ''} + ${var}="$v"; ${svar}=1 + shift + ;; + ''; + + shorts = lib.filter (s: s != null) (map (f: if f ? short then f.short else null) flags); + in + assert lib.assertMsg (lib.all (n: builtins.match "^[a-z0-9-]+$" n != null) ( + map (f: f.name) flags + )) "mkFlags (${prefix}): flag names must match [a-z0-9-]+"; + assert lib.assertMsg (lib.all ( + s: builtins.match "^[a-zA-Z0-9-]+$" s != null + ) shorts) "mkFlags (${prefix}): short flags must match [a-zA-Z0-9-]+"; + assert lib.assertMsg ( + lib.length (lib.unique (map (f: toVarName f.name) flags)) == lib.length flags + ) "mkFlags (${prefix}): duplicate variable names generated"; + assert lib.assertMsg ( + lib.length (lib.unique shorts) == lib.length shorts + ) "mkFlags (${prefix}): duplicate short flags"; + assert lib.assertMsg ( + !(lib.elem "help" (map (f: f.name) flags)) + ) "mkFlags (${prefix}): 'help' is a reserved flag name"; + assert lib.assertMsg (!(lib.elem "h" shorts)) "mkFlags (${prefix}): 'h' is a reserved short flag"; + '' + if ! declare -F die >/dev/null 2>&1; then + die() { printf 'error: %s\n' "$*" >&2; exit 1; } + fi + + ${lib.concatStringsSep "\n" (map varDecl flags)} + + _HELP_TEXT=$(cat <<'__ICEDOS_MKFLAGS_EOF__' + ${helpText} + __ICEDOS_MKFLAGS_EOF__ + ) + + _REST=() + while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + echo "$_HELP_TEXT" + exit 0 + ;; + ${lib.concatStringsSep "\n" (map genCaseArm flags)} + --) + shift + _REST+=("$@") + break + ;; + ${ + if passthroughUnknown then + '' + -*) + _REST+=("$1") + shift + ;; + '' + else + '' + -*) + die "unknown flag: $1" + ;; + '' + } + *) + _REST+=("$1") + shift + ;; + esac + done + set -- "''${_REST[@]}" + ''; + + blueString = s: "\${BLUE}${s}\${NC}"; + greenString = s: "\${GREEN}${s}\${NC}"; + purpleString = s: "\${PURPLE}${s}\${NC}"; + redString = s: "\${RED}${s}\${NC}"; + yellowString = s: "\${YELLOW}${s}\${NC}"; + + dimBlueString = s: "\${DIM_BLUE}${s}\${NC}"; + dimGreenString = s: "\${DIM_GREEN}${s}\${NC}"; + dimPurpleString = s: "\${DIM_PURPLE}${s}\${NC}"; + dimRedString = s: "\${DIM_RED}${s}\${NC}"; + dimYellowString = s: "\${DIM_YELLOW}${s}\${NC}"; + + # Config-set paths + the shell walker shared by the `icedos configuration` + # diff / rollback / history commands: all three pair a snapshot folder + # (`.cache//`, written by rebuild.nix's snapshot_config_set) + # against the working tree, over the same file set — config.toml plus every + # *.toml, hidden .*.toml included, under each `icedos.system.extraConfigs` + # dir. Takes the NixOS `config`. + configSet = + config: + let + inherit (config.icedos) configurationLocation; + configRoot = "${configurationLocation}/.."; + workingConfig = "${configRoot}/config.toml"; + configDirsArgs = concatStringsSep " " (map escapeShellArg config.icedos.system.extraConfigs); + in + { + inherit configRoot workingConfig configDirsArgs; + cacheDir = "${configurationLocation}/.cache"; + + # Defines CONFIG_DIRS + `walk_config_set `, which + # calls `