diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d96aa4e..fde80aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,14 +14,18 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - STACK: stacks/observability - # Image versions are NOT duplicated here. They are resolved from compose.yaml - # at run time by scripts/image-for.sh, because Dependabot only updates - # compose.yaml — hardcoded copies went stale silently and CI ended up - # validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the - # last exception to that and is now a profile-gated service in compose.yaml - # like every other image (#65). +# There is no `env: STACK:` here any more. It pinned every step below to +# stacks/observability, which is why stacks/lab landed as a stack CI had never +# seen (#263, #264). Each step loops over ./scripts/stacks.sh instead — the one +# place that defines what a stack is, and the thing that fails when a directory +# under stacks/ has no compose.yaml. +# +# Image versions are NOT duplicated here either. They are resolved from +# compose.yaml at run time by scripts/image-for.sh, because Dependabot only +# updates compose.yaml — hardcoded copies went stale silently and CI ended up +# validating v3.1.0 configs against a stack running v3.13.2. gitleaks was the +# last exception to that and is now a profile-gated service in compose.yaml +# like every other image (#65). jobs: # --------------------------------------------------------------------------- @@ -64,8 +68,14 @@ jobs: # needs a decryption key. The list of variables lives in the script, which # scripts/validate.sh also calls — inlining it here is what let CI and the # local run drift apart. The .env written here stays gitignored. + # Per stack, because the guard list is derived from each stack's own + # compose.yaml — seeding one from another's guards proves nothing about + # the file being validated. - name: Seed a validation-only .env - run: ./scripts/seed-validation-env.sh "$STACK/.env" + run: | + for stack in $(./scripts/stacks.sh); do + ./scripts/seed-validation-env.sh "stacks/$stack/.env" "$stack" + done # Single source of truth: whatever compose.yaml pins is what gets tested. - name: Resolve pinned images from compose.yaml @@ -80,7 +90,11 @@ jobs: ./scripts/image-for.sh alloy - name: docker compose config - run: docker compose -f "$STACK/compose.yaml" config -q + run: | + for stack in $(./scripts/stacks.sh); do + echo "== $stack" + docker compose -f "stacks/$stack/compose.yaml" config -q + done # Guard against the duplication coming back. Any image: pin outside # compose.yaml is drift waiting to happen, since Dependabot cannot see it. @@ -125,7 +139,9 @@ jobs: *@sha256:*) ;; *) echo "::error::$ref is not pinned by digest — run make pin-digests"; missing=1 ;; esac - done < <(awk '$1 == "image:" { print $2 }' "$STACK/compose.yaml") + done < <(./scripts/stacks.sh --paths | while read -r sd; do + awk '$1 == "image:" { print $2 }' "$sd/compose.yaml" + done) exit "$missing" # The three checks above are all pattern matches, and #65 walked past all @@ -141,33 +157,57 @@ jobs: - name: Verify every docker image comes from compose.yaml run: python3 scripts/check_image_pins.py - - name: promtool check config + # Every stack that runs a Prometheus. A stack with rules and no unit + # tests fails, which is stricter than the single-stack version was: rules + # that cannot fire pass `check rules` (#63), so the moment to notice a + # stack has none is when its rules land. + - name: promtool check config, check rules, test rules run: | - docker run --rm --entrypoint promtool \ - -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ - check config "$STACK/prometheus/prometheus.yaml" - - - name: promtool check rules - run: | - docker run --rm --entrypoint promtool \ - -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ - check rules "$STACK"/prometheus/rules/*.rules.yaml - - # check rules only parses PromQL. It passed for months against a rule that - # could not fire for any input (#63); these are the tests that catch that. - - name: promtool test rules - run: | - docker run --rm --entrypoint promtool \ - -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ - test rules "$STACK"/prometheus/tests/*.test.yaml + fail=0 + for stack in $(./scripts/stacks.sh); do + sd="stacks/$stack" + [ -f "$sd/prometheus/prometheus.yaml" ] || { echo "== $stack: no prometheus.yaml"; continue; } + echo "== $stack" + docker run --rm --entrypoint promtool \ + -v "$PWD:/repo" -w /repo "$PROM_IMAGE" \ + check config "$sd/prometheus/prometheus.yaml" || fail=1 + + rules=$(find "$sd/prometheus/rules" -name '*.rules.yaml' 2>/dev/null | sort) + tests=$(find "$sd/prometheus/tests" -name '*.test.yaml' 2>/dev/null | sort) + if [ -z "$rules" ]; then + echo " no alert rules" + continue + fi + # shellcheck disable=SC2086 + docker run --rm --entrypoint promtool \ + -v "$PWD:/repo" -w /repo "$PROM_IMAGE" check rules $rules || fail=1 + if [ -z "$tests" ]; then + echo "::error::$stack has alert rules and no promtool tests —" + echo "::error::a rule that cannot fire still passes check rules (#63)" + fail=1 + continue + fi + # shellcheck disable=SC2086 + docker run --rm --entrypoint promtool \ + -v "$PWD:/repo" -w /repo "$PROM_IMAGE" test rules $tests || fail=1 + done + exit $fail # No secret needed: the receiver URL comes from url_file, which # Alertmanager reads at notify time rather than at config load time. + # Only stacks that run one. stacks/lab has no Alertmanager by decision + # (ADR-0020), and the ROUTES table below describes the estate's tree + # specifically. - name: amtool check-config run: | - docker run --rm --entrypoint amtool \ - -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ - check-config "$STACK/alertmanager/alertmanager.yaml" + fail=0 + for f in $(./scripts/stacks.sh --paths | sed 's|$|/alertmanager/alertmanager.yaml|'); do + [ -f "$f" ] || continue + echo "== $f" + docker run --rm --entrypoint amtool \ + -v "$PWD:/repo" -w /repo "$AM_IMAGE" check-config "$f" || fail=1 + done + exit $fail # check-config proves the tree parses and that every route names a # receiver that exists. It does not say WHICH receiver an alert reaches, @@ -181,13 +221,17 @@ jobs: - name: amtool config routes test run: | fail=0 + # The estate's tree. Asserted against the one stack that has an + # Alertmanager rather than against every stack, because these eight + # rows are that tree's routing and not a property all stacks share. + cfg=stacks/observability/alertmanager/alertmanager.yaml while read -r expected labels; do [ -n "$expected" ] || continue # shellcheck disable=SC2086 docker run --rm --entrypoint amtool \ -v "$PWD:/repo" -w /repo "$AM_IMAGE" \ config routes test \ - --config.file="$STACK/alertmanager/alertmanager.yaml" \ + --config.file="$cfg" \ --verify.receivers="$expected" $labels \ || { echo "::error::expected $expected for $labels"; fail=1; } done <<'ROUTES' @@ -209,8 +253,14 @@ jobs: - name: alloy fmt --test # Every file in the directory: the agent loads the directory, and the # deploy script ships a subset of it, so each file must stand alone. + # Every *.alloy under stacks/, found rather than assumed to live under + # one of them. stacks/lab has no alloy/ directory — it mounts the + # estate's two files rather than copying them (ADR-0007) — so a `find` + # here covers both today and any stack that grows its own tomorrow. run: | - for f in "$STACK"/alloy/*.alloy; do + found=$(find stacks -path '*/alloy/*.alloy' | sort) + [ -n "$found" ] || { echo "::error::no *.alloy under stacks/"; exit 1; } + for f in $found; do docker run --rm --entrypoint alloy \ -v "$PWD:/repo" -w /repo "$ALLOY_IMAGE" \ fmt --test "$f" @@ -233,7 +283,16 @@ jobs: # the static check. scripts/validate.sh may honestly skip it on a host # without docker; CI may not. - name: Verify health dependencies are satisfiable and probe the images - run: python3 scripts/check_compose_health.py --probe + run: | + fail=0 + for f in $(./scripts/stacks.sh --paths | sed 's|$|/compose.yaml|'); do + echo "== $f" + python3 scripts/check_compose_health.py --probe "$f" || fail=1 + done + # The half no single file can answer: a reloaded or claimed service + # that exists in no stack at all. + python3 scripts/check_compose_health.py --cross-stack || fail=1 + exit $fail # promtool cannot check these — it parses PromQL and rejects every LogQL # stream selector. Loki itself is the only thing that understands them, so @@ -245,7 +304,12 @@ jobs: # so a typo in one rendered an empty panel and looked like quiet traffic # rather than a broken query (#82). - name: Validate Loki rules and dashboard LogQL - run: ./scripts/check_loki_rules.sh + run: | + fail=0 + for stack in $(./scripts/stacks.sh); do + ./scripts/check_loki_rules.sh --stack "$stack" || fail=1 + done + exit $fail # The device list is spread across snmp.yaml, generator.yaml, # render-config.sh's REQUIRED array and the example secrets file. Drift @@ -283,7 +347,12 @@ jobs: run: ./scripts/install-timers.sh --check --require-all - name: Validate Grafana dashboards - run: python3 scripts/check_dashboards.py + run: | + fail=0 + for stack in $(./scripts/stacks.sh); do + python3 scripts/check_dashboards.py --stack "$stack" || fail=1 + done + exit $fail # Dashboard queries are as easy to typo as alert rules, and a broken one # shows up as an empty panel rather than an error. Parse them all — the @@ -291,9 +360,22 @@ jobs: # can parse it. - name: Parse every dashboard PromQL expression run: | - python3 scripts/check_dashboards.py --emit-promql > /tmp/dashboard-exprs.yaml - docker run --rm --entrypoint promtool \ - -v /tmp:/tmp "$PROM_IMAGE" check rules /tmp/dashboard-exprs.yaml + fail=0 + for stack in $(./scripts/stacks.sh); do + # --emit-promql fails on a stack with no dashboards, deliberately: + # its output is what proves the panel queries parse, so emitting an + # empty file would pass promtool over nothing (#68). The guard is + # here, where "this stack ships none" is knowable. + if [ -z "$(find "stacks/$stack/grafana/dashboards" -name '*.json' 2>/dev/null)" ]; then + echo "== $stack: no dashboards" + continue + fi + python3 scripts/check_dashboards.py --stack "$stack" --emit-promql \ + > "/tmp/dashboard-exprs-$stack.yaml" + docker run --rm --entrypoint promtool \ + -v /tmp:/tmp "$PROM_IMAGE" check rules "/tmp/dashboard-exprs-$stack.yaml" || fail=1 + done + exit $fail # The step above reads the dashboard JSON; this one makes Grafana serve it # back. Grafana does not return what it was given — it sorts keys, HTML- diff --git a/Makefile b/Makefile index c1bbc2b..a6ca77b 100644 --- a/Makefile +++ b/Makefile @@ -179,12 +179,25 @@ check-timers: ## Verify the schedule and its staleness thresholds agree ./scripts/install-timers.sh --check .PHONY: pin-digests -pin-digests: ## Re-resolve image digests in compose.yaml (--write applies) - ./scripts/pin-digests.sh --write +pin-digests: ## Re-resolve image digests in every stack's compose.yaml (--write applies) + @# Every stack, not just the estate's. pin-digests.sh takes one compose file + @# and rewrites it in place, which is the right shape for the work it does — + @# so the loop lives here rather than inside it, driven by the same + @# scripts/stacks.sh that validate.sh and ci.yml read. Left single-stack, + @# `stacks/lab`'s digests would be re-resolved by nothing and verified by + @# nothing, which is the #263 defect in the one place it costs a supply-chain + @# guarantee rather than a test. + @set -e; for sd in $$(./scripts/stacks.sh --paths); do \ + printf '\033[0;34m--\033[0m %s\n' "$$sd"; \ + COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh --write; \ + done .PHONY: check-digests check-digests: ## Verify pinned digests still match the registry - ./scripts/pin-digests.sh + @set -e; for sd in $$(./scripts/stacks.sh --paths); do \ + printf '\033[0;34m--\033[0m %s\n' "$$sd"; \ + COMPOSE_FILE="$$sd/compose.yaml" ./scripts/pin-digests.sh; \ + done .PHONY: scan scan: ## Scan the working tree and history for secrets diff --git a/docs/roadmap.md b/docs/roadmap.md index 93c03ad..af72ade 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -306,11 +306,17 @@ what left this one unfireable for months. (`render-config.sh` derives its required keys per stack rather than demanding the estate's ten, `reload-config.sh` skips services a stack does not declare, `bootstrap.sh` refuses to give one age key both stacks) and gave `.sops.yaml` - the lab rule ADR-0020 asked for. It did **not** touch - [#263](https://github.com/Gerrrt/HomeLab/issues/263): `STACK ?=` reaches the - lifecycle targets and stops there, every checker is still pinned to - `stacks/observability`, and so the new stack is one CI has never seen — - validated only by hand and by the checks that already follow `STACK`. + the lab rule ADR-0020 asked for. [#263](https://github.com/Gerrrt/HomeLab/issues/263) + followed it: `scripts/stacks.sh` is now the single definition of what a stack + is, and `validate.sh`, `ci.yml`, `pin-digests.sh` and the Python checkers all + read it instead of carrying `stacks/observability`. Both stacks are checked, + each line says which, and a directory under `stacks/` with no compose.yaml + fails rather than being skipped — a stack nothing checks being the defect the + list exists to prevent. Two guards got stronger on the way: rules without + `promtool` unit tests are now a failure rather than an absence nobody + measured (#63), and the reload/ABSENT_BINARIES cross-checks gained a + cross-stack mode, because "not in this compose file" stopped meaning "in no + stack at all" the moment there were two. [#265](https://github.com/Gerrrt/HomeLab/issues/265) the domain is what everything else is pointed at, and blocks both [#266](https://github.com/Gerrrt/HomeLab/issues/266) Wazuh — the heaviest diff --git a/scripts/check_compose_health.py b/scripts/check_compose_health.py index e671c38..b198cdb 100755 --- a/scripts/check_compose_health.py +++ b/scripts/check_compose_health.py @@ -32,7 +32,25 @@ reads the array back out and requires each entry to match the healthcheck it is standing on (#80). +That cross-check changed shape when a second stack arrived (#263, #264). +SERVICES is the union across every stack, and reload-config.sh now skips +entries the stack it is reloading does not declare — so "not defined in this +compose file" stopped being a defect on its own. It is split in two instead, +and the pair is strictly stronger than the single check it replaces: + + * per file, an entry that stack DOES declare must carry the healthcheck the + probe stands on — unchanged, and now applied to every stack rather than + only to the estate's; + * across the complete set of stacks, every entry must be declared SOMEWHERE. + That is what still catches an array naming a service nothing has, which is + the case the old "not defined" message existed for. + +The completeness half is only claimed when this script discovered the stacks +itself, which it does when given no paths. Explicit paths mean the caller chose +the scope, and no claim about the whole repository can follow from a subset. + Usage: scripts/check_compose_health.py [--probe] [compose.yaml] + scripts/check_compose_health.py --cross-stack """ from __future__ import annotations @@ -162,20 +180,21 @@ def healthcheck_binary(test: object) -> tuple[str | None, str | None]: ) -def reload_probe_problems(services: dict, compose_name: str) -> list[str]: - """Where reload-config.sh's SERVICES array disagrees with the healthchecks. +def reload_entries() -> tuple[list[tuple[str, str]], list[str]]: + """The SERVICES array from reload-config.sh, as (name, port) pairs. - Returns the problems, or a single problem if the array itself could not be - read — a scraper that has silently stopped matching must fail rather than - report nothing to check, which is the shape of bug this whole file is about. + Split out of reload_probe_problems() so the completeness check across + stacks and the per-stack healthcheck check read the identical parse. Two + readers of one hand-rolled scraper is how they come to disagree about what + the array says. """ if not RELOAD_SCRIPT.exists(): - return [f"{RELOAD_SCRIPT.name} is missing; nothing reloads these services"] + return [], [f"{RELOAD_SCRIPT.name} is missing; nothing reloads these services"] source = RELOAD_SCRIPT.read_text(encoding="utf-8") block = RELOAD_SERVICES.search(source) if not block: - return [ + return [], [ f"could not find the SERVICES=( ... ) array in {RELOAD_SCRIPT.name} — " f"it was reshaped, and this check has been reading nothing ever since" ] @@ -195,15 +214,27 @@ def reload_probe_problems(services: dict, compose_name: str) -> list[str]: entries.append((entry.group(1), entry.group(2))) if not entries and not problems: - return [f"{RELOAD_SCRIPT.name} SERVICES is empty"] + problems.append(f"{RELOAD_SCRIPT.name} SERVICES is empty") + return entries, problems + + +def reload_probe_problems(services: dict, compose_name: str) -> list[str]: + """Where reload-config.sh's SERVICES array disagrees with the healthchecks. + + Scoped to the entries THIS compose file declares. An entry it does not + declare is not a defect here: SERVICES is the union across stacks, and + reload-config.sh skips what a stack does not define, so `stacks/lab` having + no alertmanager is the arrangement rather than a fault. That an entry + exists in no stack at all is still caught — by the completeness check in + main(), which is the only place that knows it is looking at every stack. + """ + entries, problems = reload_entries() + if not entries: + return problems for name, port in entries: svc = services.get(name) if svc is None: - problems.append( - f"{RELOAD_SCRIPT.name} reloads {name}, which is not defined in " - f"{compose_name}" - ) continue expected = f"http://localhost:{port}{RELOAD_PROBE_PATH}" @@ -342,10 +373,69 @@ def probe_binary( return "present", "" +def cross_stack_problems() -> list[str]: + """Names this file asserts about, checked against every stack at once. + + Both per-file checks had to stop treating "not in this compose file" as a + defect when a second stack arrived: reload-config.sh skips services a stack + does not declare, and a stack need not run everything ABSENT_BINARIES + describes. What must still be true is that each name exists SOMEWHERE — an + entry naming a service no stack has is a scraper that has silently stopped + matching, which is the failure this whole file was written for (#80). + + Only answerable with the complete set of stacks, which is why it is its own + mode rather than something the per-file path could do. + """ + try: + listed = subprocess.run( + [str(REPO / "scripts/stacks.sh"), "--paths"], + capture_output=True, text=True, check=True, + ).stdout.split() + except (OSError, subprocess.CalledProcessError) as exc: + err = (getattr(exc, "stderr", "") or str(exc)).strip() + return [f"could not list stacks: {err}"] + + declared: set[str] = set() + for entry in listed: + compose_path = REPO / entry / "compose.yaml" + compose = yaml.safe_load(compose_path.read_text(encoding="utf-8")) + declared |= set(compose.get("services") or {}) + + problems: list[str] = [] + entries, problems_from_parse = reload_entries() + problems += problems_from_parse + for name, _port in entries: + if name not in declared: + problems.append( + f"{RELOAD_SCRIPT.name} reloads {name}, which no stack defines — " + f"it was renamed or removed, and nothing has reloaded it since" + ) + for name in ABSENT_BINARIES: + if name not in declared: + problems.append( + f"ABSENT_BINARIES names {name}, which no stack defines — the " + f"claim it stands for has gone with the service" + ) + return problems + + def main() -> int: argv = sys.argv[1:] probe = "--probe" in argv argv = [arg for arg in argv if arg != "--probe"] + + if "--cross-stack" in argv: + argv = [arg for arg in argv if arg != "--cross-stack"] + if argv: + print("--cross-stack takes no compose file", file=sys.stderr) + return 1 + problems = cross_stack_problems() + for problem in problems: + print(f" {problem}", file=sys.stderr) + if problems: + return 1 + print("cross-stack OK — every reloaded and claimed service exists in some stack") + return 0 if argv and argv[0].startswith("-"): print(f"unknown option {argv[0]}", file=sys.stderr) print(__doc__.strip().splitlines()[-1], file=sys.stderr) @@ -384,11 +474,11 @@ def main() -> int: ) # The reload script's copy of these ports, checked against the originals. - # Only meaningful against the file it actually reloads, so it is skipped for - # an explicitly-passed compose file that is not the default one. - reload_checked = path.resolve() == DEFAULT.resolve() - if reload_checked: - problems += reload_probe_problems(services, path.name) + # Runs for EVERY stack now, not just the default one: it is scoped to the + # entries this file declares, so it is meaningful against any of them. The + # matching "no stack declares this entry at all" half is --reload-completeness + # below, which is the only mode that looks at every stack at once. + problems += reload_probe_problems(services, path.name) # Every healthcheck the compose file declares, decoded to the one binary the # image has to contain for it to run at all. Profiles are deliberately not @@ -440,11 +530,13 @@ def main() -> int: for name, binaries in ABSENT_BINARIES.items(): svc = services.get(name) if svc is None: - problems.append( - f"ABSENT_BINARIES names {name}, which is not defined in " - f"{path.name} — it was renamed or removed, and the claim it " - f"stands for has gone with it" - ) + # Not a defect for THIS file. ABSENT_BINARIES is a claim about + # images across the repository, and a stack is allowed not to run + # the service it names — `loki` happens to be in both stacks today, + # but nothing says the next one must be. The rename-or-removal + # catch this message existed for now lives in --cross-stack, which + # is the only mode that can tell "absent from this stack" from + # "absent from every stack". continue svc = svc or {} if svc.get("healthcheck"): @@ -559,8 +651,7 @@ def main() -> int: f"{path.name} OK — {checks} healthcheck(s), " f"{healthy_deps} service_healthy dependency/dependencies, all satisfiable" ) - if reload_checked: - summary += f"; {RELOAD_SCRIPT.name} probes agree with them" + summary += f"; {RELOAD_SCRIPT.name} probes agree with them" if probe: # The count is the point. A probe loop that silently stopped matching # anything would otherwise print this same green line having done diff --git a/scripts/check_dashboards.py b/scripts/check_dashboards.py index f34cc76..7416b88 100755 --- a/scripts/check_dashboards.py +++ b/scripts/check_dashboards.py @@ -34,6 +34,11 @@ import sys REPO = pathlib.Path(__file__).resolve().parent.parent + +# Rebound by main() from --stack. They stay module-level because +# declared_datasources() and the emit paths read them, and threading a stack +# through six call sites to avoid two `global` statements would be the worse +# trade. DASHBOARDS = REPO / "stacks/observability/grafana/dashboards" DATASOURCES = REPO / "stacks/observability/grafana/provisioning/datasources/datasources.yaml" @@ -276,8 +281,26 @@ def main() -> int: help="print dashboard PromQL as a rules file for promtool") emit.add_argument("--emit-logql", action="store_true", help="print dashboard LogQL as a rules file for Loki's ruler") + parser.add_argument("--stack", default="observability", + help="stack under stacks/ to check (default: observability)") args = parser.parse_args() + global DASHBOARDS, DATASOURCES + stack_dir = REPO / "stacks" / args.stack + if not (stack_dir / "compose.yaml").is_file(): + print(f"no such stack: {stack_dir}", file=sys.stderr) + return 1 + DASHBOARDS = stack_dir / "grafana/dashboards" + DATASOURCES = stack_dir / "grafana/provisioning/datasources/datasources.yaml" + + # ALLOY_DIR is deliberately NOT stack-relative. The agent config is one + # directory shared by every stack — stacks/lab mounts config.alloy and + # docker.alloy straight out of stacks/observability/alloy/ rather than + # copying them (ADR-0007's "reused unchanged") — so the level vocabulary it + # defines is a property of the repository, not of whichever stack is being + # checked. Making it stack-relative would look tidier and would check the + # lab's dashboards against an alloy directory that does not exist. + declared = declared_datasources() known = set(declared) | BUILTIN_UIDS problems: list[str] = [] @@ -289,8 +312,27 @@ def main() -> int: files = sorted(DASHBOARDS.glob("*.json")) if not files: - print(f"no dashboards found in {DASHBOARDS}", file=sys.stderr) - return 1 + # Not an error. `stacks/lab` ships no dashboards on purpose — copying + # the estate's seven would render four rows of empty panels, and an + # empty panel is indistinguishable from a broken collector. A stack is + # allowed to have none. + # + # This does not weaken the guard against the estate's dashboards + # disappearing: check_docs.py counts them and asserts the count against + # the prose that claims seven, so observability reaching zero fails + # there, in the check that owns that claim. + # The emit paths stay STRICT, and that is the half that matters. Their + # callers hand the output to promtool or to Loki's ruler as the file + # that proves the panel queries parse, so emitting an empty file over a + # vanished dashboard directory would pass those checks over nothing — + # which is the #68 shape. check_loki_rules.sh therefore only asks for an + # emit when the stack actually has dashboards, and a request that + # arrives anyway is a bug worth failing on. + if args.emit_promql or args.emit_logql: + print(f"no dashboards found in {DASHBOARDS}", file=sys.stderr) + return 1 + print(f"no dashboards in {DASHBOARDS.relative_to(REPO)} — nothing to check") + return 0 for path in files: name = path.name diff --git a/scripts/check_docs.py b/scripts/check_docs.py index ae18080..2978e59 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -86,7 +86,14 @@ "docs/network.md", "docs/observability.md", "docs/security.md", - "stacks/observability/README.md", + # Every stack's README, globbed rather than listed. This was the single + # literal "stacks/observability/README.md", so stacks/lab's README was + # prose nothing checked — its image table could have carried a version pin + # and gone stale silently, which is the #73 defect the whole PROSE list + # exists to prevent (#263). + *sorted( + str(p.relative_to(REPO)) for p in REPO.glob("stacks/*/README.md") + ), *sorted( str(p.relative_to(REPO)) for p in (REPO / "docs/runbooks").glob("*.md") ), diff --git a/scripts/check_loki_rules.sh b/scripts/check_loki_rules.sh index 94b18e8..bc5da05 100755 --- a/scripts/check_loki_rules.sh +++ b/scripts/check_loki_rules.sh @@ -18,14 +18,14 @@ # of the panels to promtool for exactly that reason; LogQL had no equivalent, # so a typo in a Loki panel reached production unchallenged. # -# Usage: scripts/check_loki_rules.sh +# Usage: scripts/check_loki_rules.sh [--stack NAME] [--skips-file PATH] # -e is on: a failed cp or config rewrite must not produce a cheerful PASS. # The one command allowed to fail is the timeout below, which is guarded. set -euo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -STACK="${REPO_ROOT}/stacks/observability" +STACK_NAME="observability" # Resolved from compose.yaml — see scripts/image-for.sh. LOKI_IMAGE="$("${REPO_ROOT}/scripts/image-for.sh" loki)" BOOT_SECONDS="${BOOT_SECONDS:-45}" @@ -40,13 +40,42 @@ SKIPS_FILE="" while (($#)); do case "$1" in --skips-file) SKIPS_FILE="${2:?--skips-file needs a path}"; shift ;; + --stack) STACK_NAME="${2:?--stack needs a name}"; shift ;; *) die "unknown argument: $1" ;; esac shift done +STACK="${REPO_ROOT}/stacks/${STACK_NAME}" +[[ -f "${STACK}/compose.yaml" ]] || die "no such stack: ${STACK}" RULES_DIR="${STACK}/loki/rules" -[[ -d "${RULES_DIR}" ]] || die "no rules directory at ${RULES_DIR}" + +# A stack may legitimately have neither Loki rules nor dashboards. `stacks/lab` +# has both absences on purpose: its Loki carries no ruler, because a ruler needs +# an Alertmanager to deliver to and that stack has none (ADR-0020), and it ships +# no dashboards yet. With nothing to parse there is nothing this check can say, +# and it exits 0 saying so. +# +# NOT a skip. A skip means "this could not run and therefore proved nothing", +# and validate.sh counts skips precisely so a run cannot claim to have checked +# what it did not (#68). This ran, and there was nothing to check — a different +# statement, and mislabelling it would inflate the skip count on every run +# forever until it stopped being read. +n_committed=0 +if [[ -d "${RULES_DIR}" ]]; then + shopt -s nullglob + rule_files=("${RULES_DIR}"/*.yaml) + shopt -u nullglob + n_committed=${#rule_files[@]} +fi +shopt -s nullglob +dash_files=("${STACK}/grafana/dashboards"/*.json) +shopt -u nullglob +if ((n_committed == 0 && ${#dash_files[@]} == 0)); then + printf '\033[0;32m PASS\033[0m %s\n' \ + "${STACK_NAME}: no Loki rules and no dashboards — nothing to parse" + exit 0 +fi # Whether this run can happen at all is decided FIRST, before anything with a # side effect or a failure mode of its own. @@ -81,20 +110,34 @@ WORK="$(mktemp -d)" trap 'rm -rf "${WORK}" 2>/dev/null || true' EXIT # auth_enabled is false, so Loki's local ruler looks under /fake/. mkdir -p "${WORK}/rules/fake" "${WORK}/data" -cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" +# An `if` and not `((n_committed)) && cp ...`: under `set -e` that one-liner +# exits the script when the count is zero, because the && chain's status +# becomes the failed arithmetic. A stack with dashboards but no rules would +# have died here reporting nothing. +if ((n_committed)); then + cp "${RULES_DIR}"/*.yaml "${WORK}/rules/fake/" +fi # Dashboard LogQL, as an extra rule file so it takes precisely the same path # through the ruler as the committed rules — same parser, same failure output, -# no second code path to keep honest. check_dashboards.py refuses to emit an -# empty file, so a run that finds no panels fails here rather than passing over -# nothing. +# no second code path to keep honest. +# +# Asked for only when the stack HAS dashboards. check_dashboards.py --emit-logql +# still fails on an empty directory, deliberately: its output is the file that +# proves the panel queries parse, so emitting nothing over a vanished dashboard +# directory would pass this check over nothing (#68). The guard belongs here, +# where "this stack ships no dashboards" is known, rather than in the emitter, +# where it cannot be told apart from "the dashboards are gone". DASH_RULES="${WORK}/rules/fake/dashboard-expressions.rules.yaml" -if ! python3 "${REPO_ROOT}/scripts/check_dashboards.py" --emit-logql \ - > "${DASH_RULES}" 2> "${WORK}/emit.err"; then - cat "${WORK}/emit.err" >&2 - die "could not emit dashboard LogQL" +n_dash=0 +if ((${#dash_files[@]})); then + if ! python3 "${REPO_ROOT}/scripts/check_dashboards.py" --stack "${STACK_NAME}" --emit-logql \ + > "${DASH_RULES}" 2> "${WORK}/emit.err"; then + cat "${WORK}/emit.err" >&2 + die "could not emit dashboard LogQL" + fi + n_dash="$(grep -c '^ *- alert:' "${DASH_RULES}" || true)" fi -n_dash="$(grep -c '^ *- alert:' "${DASH_RULES}" || true)" # The Loki image runs as uid 10001, while mktemp -d creates a 0700 directory # owned by the invoking user. Without this the container cannot read its own diff --git a/scripts/seed-validation-env.sh b/scripts/seed-validation-env.sh index e420a1d..9d0f28c 100755 --- a/scripts/seed-validation-env.sh +++ b/scripts/seed-validation-env.sh @@ -13,19 +13,33 @@ # Nothing written here is a secret and nothing here is ever deployed. The values # only need to exist. # -# Usage: scripts/seed-validation-env.sh +# Usage: scripts/seed-validation-env.sh [stack] (default: observability) # # The caller owns the output path and its cleanup: CI writes the gitignored -# stacks/observability/.env, validate.sh writes an mktemp file it removes on -# exit. Keeping lifetime out here is what lets one script serve both. +# stacks//.env, validate.sh writes an mktemp file it removes on exit. +# Keeping lifetime out here is what lets one script serve both. +# +# The stack argument matters because the guard list below is derived from that +# stack's compose.yaml. Seeding `lab` from `observability`'s guards would prove +# nothing about the file being validated — and a guard added to one stack and +# not the other would pass here and fail in compose, which is the drift this +# script was written to stop (#263). set -euo pipefail -OUT="${1:?usage: seed-validation-env.sh }" +OUT="${1:?usage: seed-validation-env.sh [stack]}" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -STACK="${REPO_ROOT}/stacks/observability" +STACK="${REPO_ROOT}/stacks/${2:-observability}" +[[ -f "${STACK}/compose.yaml" ]] || { + printf 'no such stack: %s\n' "${STACK}" >&2 + exit 1 +} -cat "${STACK}/.env.example" > "${OUT}" +# A stack may legitimately have no .env.example — every tunable in its +# compose.yaml can carry a default. The guards below are what must be satisfied, +# and they are read from compose.yaml, not from here. +: > "${OUT}" +[[ -f "${STACK}/.env.example" ]] && cat "${STACK}/.env.example" > "${OUT}" { echo "GRAFANA_ADMIN_PASSWORD=validation-only" echo "GRAFANA_RENDERER_TOKEN=validation-only" diff --git a/scripts/stacks.sh b/scripts/stacks.sh new file mode 100755 index 0000000..8980d2e --- /dev/null +++ b/scripts/stacks.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +# +# The list of stacks, defined once. +# +# scripts/stacks.sh names, one per line -> observability +# scripts/stacks.sh --paths repo-relative paths -> stacks/observability +# +# Why this exists +# --------------- +# `STACK ?= observability` in the Makefile parameterised the lifecycle and +# secrets targets and stopped there. Every validator carried its own +# `stacks/observability` — validate.sh, four Python checkers, check_loki_rules.sh, +# seed-validation-env.sh and ci.yml — so `stacks/lab` landed as a stack CI had +# never seen: compose not `config`-checked, rules not `promtool`-tested, images +# pinned by nothing (#263, #264). +# +# The fix is one list, not eight. Anything that iterates stacks reads it from +# here, including the Python checkers — a second implementation in Python would +# be a second definition, and two definitions of "what a stack is" drift the +# same way two copies of a device list do. That is the argument snmp-targets.sh +# makes about the SNMP inventory, applied one level up. +# +# What counts as a stack +# ---------------------- +# A directory under stacks/ holding a compose.yaml. Nothing else is consulted — +# not a manifest, not a list in this file — because a stack IS its compose file +# (ADR-0004), and any register kept beside the directory tree is a register that +# can disagree with it. +# +# A directory WITHOUT a compose.yaml is an error, not a skip. That is the whole +# point rather than strictness for its own sake: the failure this guards against +# is a stack nothing checks, and silently skipping a malformed directory is +# indistinguishable from it. `mkdir stacks/foo` now fails `make validate` with a +# sentence saying why, instead of passing and quietly covering nothing. +# +# Hidden directories are skipped — `.rendered/` and friends live under a stack, +# never beside one. + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +STACKS_DIR="${REPO_ROOT}/stacks" + +MODE="names" +case "${1:-}" in + "") ;; + --paths) MODE="paths" ;; + -h | --help) + sed -n '3,6p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//' + exit 0 + ;; + *) + printf '\033[0;31merror:\033[0m unknown argument: %s\n' "$1" >&2 + exit 2 + ;; +esac + +[[ -d "${STACKS_DIR}" ]] || { + printf '\033[0;31merror:\033[0m no stacks/ directory at %s\n' "${STACKS_DIR}" >&2 + exit 1 +} + +found=0 +malformed=() +for dir in "${STACKS_DIR}"/*/; do + name="$(basename "${dir}")" + [[ "${name}" == .* ]] && continue + # A glob that matches nothing expands to itself; without this an empty + # stacks/ would report a stack literally called `*`. + [[ -d "${dir}" ]] || continue + if [[ ! -f "${dir}compose.yaml" ]]; then + malformed+=("${name}") + continue + fi + found=1 + if [[ "${MODE}" == "paths" ]]; then + printf 'stacks/%s\n' "${name}" + else + printf '%s\n' "${name}" + fi +done + +if ((${#malformed[@]})); then + printf '\033[0;31merror:\033[0m no compose.yaml in: %s\n' "${malformed[*]}" >&2 + printf 'A directory under stacks/ is a stack, and a stack is its compose file +(ADR-0004). Nothing validates, deploys or pins images for a directory without +one, so this fails rather than skipping it — a stack nothing checks is the +defect this list exists to prevent (#263). + +Add a compose.yaml, or remove the directory.\n' >&2 + exit 1 +fi + +((found)) || { + printf '\033[0;31merror:\033[0m stacks/ holds no stack with a compose.yaml\n' >&2 + exit 1 +} diff --git a/scripts/validate.sh b/scripts/validate.sh index 29e248e..531ab38 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -14,7 +14,23 @@ set -uo pipefail REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${REPO_ROOT}" || exit 1 -STACK="stacks/observability" +# Every stack, from the one place that defines what a stack is. This was +# `STACK="stacks/observability"`, which is why `stacks/lab` landed as a stack +# CI had never seen — its compose was not `config`-checked, its rules were not +# promtool-tested, its images were pinned by nothing (#263). +# +# A failure here is fatal rather than an empty loop: stacks.sh exits non-zero +# on a directory under stacks/ with no compose.yaml, and silently validating +# zero stacks is the precise defect this is fixing. +# Command substitution and not `mapfile < <(...)`: a process substitution's +# exit status is not mapfile's, so `if ! mapfile ...` succeeds even when +# stacks.sh has just refused — which would turn "a stack nothing checks" into +# "no stacks checked at all", reported as a pass. +if ! STACK_LIST="$(./scripts/stacks.sh)"; then + printf '\033[0;31merror:\033[0m could not list the stacks — see above\n' >&2 + exit 1 +fi +mapfile -t STACKS <<< "${STACK_LIST}" # Resolved from compose.yaml so Dependabot's bumps reach the checks. Hardcoding # these meant CI validated Prometheus v3.1.0 configs while the stack ran v3.13.2. PROM_IMAGE="$(./scripts/image-for.sh prometheus)" @@ -36,6 +52,21 @@ head_() { printf '\n\033[1m%s\033[0m\n' "$*"; } have() { command -v "$1" >/dev/null 2>&1; } have_docker() { have docker && docker info >/dev/null 2>&1; } +# Whether ANY stack is up on this machine. Used to decide whether this is a +# deployment host, which was previously "is the observability stack running". +# The lab stack runs on its own guest, and a host running only that one is +# still a deployment host with timers to install. +stack_running() { + local stack + for stack in "${STACKS[@]}"; do + if docker compose -f "stacks/${stack}/compose.yaml" ps --status running -q 2>/dev/null \ + | grep -q .; then + return 0 + fi + done + return 1 +} + # One trap for every temporary file this script owns. A second `trap ... EXIT` # replaces the first rather than adding to it, so they are registered together # here instead of next to the code that creates them. @@ -60,17 +91,24 @@ head_ "Compose" if have docker; then # A .env is required for the ${VAR:?} guards; seeded by the same script CI # uses, so a variable added there cannot pass locally and fail in CI. Written - # to a temp file rather than ${STACK}/.env so a local run never leaves an .env + # to a temp file rather than ${sd}/.env so a local run never leaves an .env # sitting next to a real one. + # + # Seeded PER STACK, because the guard list is derived from that stack's own + # compose.yaml. Seeding every stack from the estate's guards would prove + # nothing about the file being validated. TMP_ENV="$(mktemp)" - if ! ./scripts/seed-validation-env.sh "${TMP_ENV}"; then - fail "seed a validation-only .env" - elif docker compose --env-file "${TMP_ENV}" -f "${STACK}/compose.yaml" config -q 2>/dev/null; then - pass "docker compose config" - else - docker compose --env-file "${TMP_ENV}" -f "${STACK}/compose.yaml" config -q - fail "docker compose config" - fi + for stack in "${STACKS[@]}"; do + sd="stacks/${stack}" + if ! ./scripts/seed-validation-env.sh "${TMP_ENV}" "${stack}"; then + fail "${stack}: seed a validation-only .env" + elif docker compose --env-file "${TMP_ENV}" -f "${sd}/compose.yaml" config -q 2>/dev/null; then + pass "${stack}: docker compose config" + else + docker compose --env-file "${TMP_ENV}" -f "${sd}/compose.yaml" config -q + fail "${stack}: docker compose config" + fi + done else skip "docker not installed" fi @@ -87,30 +125,61 @@ else fi if ((${#PROMTOOL[@]})); then - if "${PROMTOOL[@]}" check config "${STACK}/prometheus/prometheus.yaml" 2>&1 \ - | grep -qE '^\s*SUCCESS'; then - pass "promtool check config" - else - "${PROMTOOL[@]}" check config "${STACK}/prometheus/prometheus.yaml" - fail "promtool check config" - fi + for stack in "${STACKS[@]}"; do + sd="stacks/${stack}" + # A stack need not run Prometheus at all. Absence is reported and moved + # past rather than skipped: a skip means "could not check", and this ran + # and found nothing to check. Inflating SKIPPED with by-design absences is + # how the count stops being read (#68). + if [[ ! -f "${sd}/prometheus/prometheus.yaml" ]]; then + pass "${stack}: no prometheus.yaml — nothing to check" + continue + fi - if "${PROMTOOL[@]}" check rules "${STACK}"/prometheus/rules/*.rules.yaml >/dev/null 2>&1; then - pass "promtool check rules" - else - "${PROMTOOL[@]}" check rules "${STACK}"/prometheus/rules/*.rules.yaml - fail "promtool check rules" - fi + if "${PROMTOOL[@]}" check config "${sd}/prometheus/prometheus.yaml" 2>&1 \ + | grep -qE '^\s*SUCCESS'; then + pass "${stack}: promtool check config" + else + "${PROMTOOL[@]}" check config "${sd}/prometheus/prometheus.yaml" + fail "${stack}: promtool check config" + fi - # `check rules` only parses PromQL; it cannot tell whether an expression can - # ever be true. ContainerHighMemory passed it for months while being - # unfireable (#63). The unit tests are what actually assert the rules fire. - if "${PROMTOOL[@]}" test rules "${STACK}"/prometheus/tests/*.test.yaml >/dev/null 2>&1; then - pass "promtool test rules" - else - "${PROMTOOL[@]}" test rules "${STACK}"/prometheus/tests/*.test.yaml - fail "promtool test rules" - fi + # nullglob so an absent rules/ or tests/ directory does not hand promtool + # the literal glob, which it reports as a missing file — a failure that + # reads as a broken rule rather than as a stack that has none. + shopt -s nullglob + rule_files=("${sd}"/prometheus/rules/*.rules.yaml) + test_files=("${sd}"/prometheus/tests/*.test.yaml) + shopt -u nullglob + + if ((${#rule_files[@]} == 0)); then + pass "${stack}: no alert rules — nothing to check" + elif "${PROMTOOL[@]}" check rules "${rule_files[@]}" >/dev/null 2>&1; then + pass "${stack}: promtool check rules (${#rule_files[@]} file(s))" + else + "${PROMTOOL[@]}" check rules "${rule_files[@]}" + fail "${stack}: promtool check rules" + fi + + # `check rules` only parses PromQL; it cannot tell whether an expression can + # ever be true. ContainerHighMemory passed it for months while being + # unfireable (#63). The unit tests are what actually assert the rules fire. + # + # Rules with no tests is a FAIL and not a pass-with-a-note, and that is the + # one place this loop is stricter than the old single-stack version. A new + # stack arriving with rules and no tests is #63 waiting to happen, and the + # moment to say so is when the rules land. + if ((${#rule_files[@]} == 0)); then + : + elif ((${#test_files[@]} == 0)); then + fail "${stack}: ${#rule_files[@]} rule file(s) and no promtool tests — a rule that cannot fire passes 'check rules' (#63)" + elif "${PROMTOOL[@]}" test rules "${test_files[@]}" >/dev/null 2>&1; then + pass "${stack}: promtool test rules (${#test_files[@]} file(s))" + else + "${PROMTOOL[@]}" test rules "${test_files[@]}" + fail "${stack}: promtool test rules" + fi + done else skip "no promtool and no docker daemon" fi @@ -129,12 +198,24 @@ fi # The receiver URL comes from url_file, which Alertmanager reads at notify time # rather than at load time — so this validates without any secret present. -if ((${#AMTOOL[@]})); then +# Which stacks have an Alertmanager at all. `stacks/lab` has none by decision +# (ADR-0020), so there is no config to parse and no routing tree to assert +# against — the ROUTES table below describes the estate's tree specifically. +am_stacks=() +for stack in "${STACKS[@]}"; do + [[ -f "stacks/${stack}/alertmanager/alertmanager.yaml" ]] && am_stacks+=("${stack}") +done + +if ((${#am_stacks[@]} == 0)); then + pass "no stack runs an Alertmanager — nothing to check" +elif ((${#AMTOOL[@]})); then + for stack in "${am_stacks[@]}"; do + STACK="stacks/${stack}" if "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" >/dev/null 2>&1; then - pass "amtool check-config" + pass "${stack}: amtool check-config" else "${AMTOOL[@]}" check-config "${STACK}/alertmanager/alertmanager.yaml" - fail "amtool check-config" + fail "${stack}: amtool check-config" fi # check-config proves the tree parses and that every route names a receiver @@ -175,10 +256,11 @@ null severity=info category=correctness ROUTES if ((routes_ok)); then - pass "amtool config routes test (8 assertions)" + pass "${stack}: amtool config routes test (8 assertions)" else - fail "amtool config routes test" + fail "${stack}: amtool config routes test" fi + done else skip "no amtool and no docker daemon" fi @@ -201,13 +283,25 @@ fi # monitoring host mounts all of them, and deploy-agent.sh ships a subset. One # unformatted file used to be impossible to have; now it is one `fmt -w` away # from being missed, and this is the check that misses nothing. -if ((${#ALLOY[@]})); then - for alloy_file in "${STACK}"/alloy/*.alloy; do +# Every *.alloy in the repository, found rather than assumed to live under one +# stack. `stacks/lab` has no alloy/ directory — it mounts config.alloy and +# docker.alloy straight out of stacks/observability/alloy/ (ADR-0007's "reused +# unchanged"), so those files are checked once here and are the same bytes both +# stacks run. A stack that grows its own agent config is picked up with no +# change to this block. +shopt -s nullglob globstar +alloy_files=(stacks/**/alloy/*.alloy) +shopt -u nullglob globstar + +if ((${#alloy_files[@]} == 0)); then + fail "no *.alloy anywhere under stacks/ — the agent config has gone missing" +elif ((${#ALLOY[@]})); then + for alloy_file in "${alloy_files[@]}"; do if "${ALLOY[@]}" fmt --test "${alloy_file}" >/dev/null 2>&1; then - pass "alloy fmt --test ${alloy_file##*/}" + pass "alloy fmt --test ${alloy_file#stacks/}" else "${ALLOY[@]}" fmt --test "${alloy_file}" - fail "alloy fmt --test ${alloy_file##*/}" + fail "alloy fmt --test ${alloy_file#stacks/}" fi done else @@ -230,10 +324,22 @@ if have python3; then else skip "no docker daemon — healthcheck binaries not probed inside their images" fi - if "${HEALTH[@]}"; then - pass "compose health dependencies" + for stack in "${STACKS[@]}"; do + if "${HEALTH[@]}" "stacks/${stack}/compose.yaml"; then + pass "${stack}: compose health dependencies" + else + fail "${stack}: compose health dependencies" + fi + done + + # The half no single file can answer: a SERVICES entry, or an + # ABSENT_BINARIES claim, naming a service that exists in NO stack. Both + # checks had to stop treating "absent from this compose file" as a defect + # once a second stack existed, and this is where that catch went. + if python3 scripts/check_compose_health.py --cross-stack; then + pass "reloaded and claimed services all exist in some stack" else - fail "compose health dependencies" + fail "reloaded and claimed services all exist in some stack" fi else skip "python3 not installed" @@ -265,22 +371,26 @@ head_ "Loki rules and dashboard LogQL" # a pass here and left SKIPPED untouched — so this script could sign off with an # unqualified "all checks passed" over rules it never validated (#68). LOKI_SKIPS="$(mktemp)" -if ./scripts/check_loki_rules.sh --skips-file "${LOKI_SKIPS}"; then - : -else - FAILED=1 -fi +for stack in "${STACKS[@]}"; do + if ./scripts/check_loki_rules.sh --stack "${stack}" --skips-file "${LOKI_SKIPS}"; then + : + else + FAILED=1 + fi +done SKIPPED=$((SKIPPED + $(wc -l < "${LOKI_SKIPS}"))) # --------------------------------------------------------------------------- head_ "Grafana dashboards" # --------------------------------------------------------------------------- if have python3; then - if python3 scripts/check_dashboards.py; then - pass "dashboard JSON and datasource references" - else - fail "dashboard JSON and datasource references" - fi + for stack in "${STACKS[@]}"; do + if python3 scripts/check_dashboards.py --stack "${stack}"; then + pass "${stack}: dashboard JSON and datasource references" + else + fail "${stack}: dashboard JSON and datasource references" + fi + done else skip "python3 not installed" fi @@ -395,8 +505,8 @@ if ! have systemctl; then skip "systemctl absent — cannot tell whether the schedule is installed" elif ! have_docker; then skip "docker unavailable — cannot tell whether this is the deployment host" -elif ! docker compose -f "${STACK}/compose.yaml" ps --status running -q 2>/dev/null | grep -q .; then - skip "the stack is not running here — this is not the deployment host" +elif ! stack_running; then + skip "no stack is running here — this is not a deployment host" elif systemctl list-unit-files 'homelab-*' --no-legend 2>/dev/null | grep -q .; then pass "the schedule is installed on this host" else @@ -445,8 +555,12 @@ fi # certificates/ is in that list because its contents were committed once and # had to be removed by rewriting every commit in the repository. The cheapest # possible check is that it never becomes tracked again. -if git ls-files --error-unmatch "${STACK}/.env" >/dev/null 2>&1 \ - || git ls-files "${STACK}/snmp-exporter/.rendered" | grep -q . \ +# Every stack's, not one stack's: a second stack means a second .env holding a +# decrypted Grafana password, and the check that it never becomes tracked has +# to grow with it. `git ls-files 'stacks/*/.env'` rather than a loop, so a +# stack added without touching this file is still covered. +if git ls-files 'stacks/*/.env' | grep -q . \ + || git ls-files 'stacks/*/.rendered' 'stacks/*/*/.rendered' | grep -q . \ || git ls-files | grep -q '\.purge-secrets\.txt' \ || git ls-files | grep -q '^certificates/' \ || git ls-files | grep -q '^backups/'; then diff --git a/stacks/lab/README.md b/stacks/lab/README.md index fa96fbe..cac1f7f 100644 --- a/stacks/lab/README.md +++ b/stacks/lab/README.md @@ -63,11 +63,11 @@ thing entirely on this segment. ## Things worth knowing before editing -- **Nothing here is validated by CI yet.** Every checker in this repository is - pinned to `stacks/observability` — `validate.sh`, the four Python checkers, - `check_loki_rules.sh`, `seed-validation-env.sh` and `ci.yml`. Making them - multi-stack is [#263]. Until it lands, run the checks against this stack by - hand; the commands are at the bottom of this file. +- **CI validates this stack.** It did not when the directory first landed — + every checker was pinned to `stacks/observability` — and [#263] fixed that: + `scripts/stacks.sh` is now the one definition of what a stack is, and + `validate.sh`, `ci.yml`, `pin-digests.sh` and the Python checkers all read it. + What that does *not* cover is stated below. - **Nothing converges this stack, either.** [#99] replaced deploying over SSH with `scripts/converge.sh` on an hourly timer, and that script runs a bare `make up` — which is `STACK=observability`, on the monitoring host. This @@ -102,18 +102,26 @@ thing entirely on this segment. ## Validate before deploying -`make validate` does **not** cover this stack ([#263]). Until it does: - ```bash -make check-rules STACK=lab +make validate ``` -That target already follows `STACK`, because it uses `$(STACK_DIR)`. The rest -needs the image directly: - -```bash -docker compose -f stacks/lab/compose.yaml config -q -``` +Covers this stack and the estate's together, and names which is which on every +line. `make check-rules STACK=lab` narrows it to this stack's Prometheus rules +and their unit tests. + +What `make validate` still does **not** prove about this stack, in the order it +matters: + +- **That it runs.** Nothing here has ever been deployed — the guest is [#262]. + Every check is static: configs parse, images resolve, healthcheck binaries + exist inside their pinned images. None of it says the four services come up + and talk to each other. +- **That its Grafana serves.** `check_dashboard_roundtrip.sh` boots the pinned + Grafana against the estate's dashboards; this stack has none to round-trip, + so that check has nothing to say here. +- **That the retention figures are right.** They are a bound, not a + measurement — see above. [#99]: https://github.com/Gerrrt/HomeLab/issues/99 [#88]: https://github.com/Gerrrt/HomeLab/issues/88 diff --git a/stacks/lab/grafana/dashboards/README.md b/stacks/lab/grafana/dashboards/README.md index 98fdeca..5ff244c 100644 --- a/stacks/lab/grafana/dashboards/README.md +++ b/stacks/lab/grafana/dashboards/README.md @@ -31,10 +31,12 @@ they are what `scripts/check_dashboards.py` enforces. Two things about this stack specifically: -- **`check_dashboards.py` does not enforce them here.** That checker is pinned - to `stacks/observability`, along with every other validator in the - repository. Making them multi-stack is [#263]. Until it lands, a dashboard - committed to this directory is checked by nothing, so check it by hand. +- **`check_dashboards.py` does enforce them here**, since [#263] made the + validators multi-stack. A dashboard committed to this directory is checked by + `make validate` and by CI the same way the estate's seven are: JSON validity, + unique uid, provisioned datasource references, and its PromQL and LogQL + parsed by promtool and by Loki's own ruler. Today it reports "no dashboards + — nothing to check", which is a pass and not a skip. - **`make dashboards-export STACK=lab` works, and nothing reminds you to run it.** `scripts/export-dashboards.sh` takes a stack argument, and the provider here sets `allowUiUpdates: true` so that it can. What the lab does not have