diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 174843d..409bc57 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,6 +19,28 @@ updates: prometheus-stack: patterns: ["prom/*"] + # The lab stack. A second entry rather than a second directory on the one + # above, because Dependabot takes a single directory per entry — and without + # it the lab's pins would rot exactly the way the comment above says pinned + # tags do, while the estate's stayed current. The two run the same four + # images, so they are expected to move together; they get separate PRs + # because they are separate deploys to separate hosts, and the lab's is a + # host the operator has to be sitting in front of. + - package-ecosystem: docker-compose + directory: /stacks/lab + schedule: + interval: weekly + day: sunday + open-pull-requests-limit: 5 + commit-message: + prefix: "chore(deps)" + labels: ["dependencies", "lab"] + groups: + grafana-stack: + patterns: ["grafana/*"] + prometheus-stack: + patterns: ["prom/*"] + - package-ecosystem: github-actions directory: / schedule: diff --git a/.sops.yaml b/.sops.yaml index 9dee59a..70f5198 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -24,6 +24,31 @@ # public half into this file. creation_rules: + # The lab stack, FIRST — SOPS takes the first rule whose path_regex matches, + # so this has to sit above the catch-all below or it would never apply. + # + # It exists because the rule below matches all of secrets/, which means any + # recipient added to it can decrypt every file there. The lab guest needs to + # decrypt secrets/lab.sops.yaml on `make render STACK=lab`; adding its key to + # the general rule to achieve that would also hand it the estate's SNMP + # communities and Grafana admin password — a lab host holding the credentials + # of the estate it is supposed to be isolated from, which inverts the trust + # direction ADR-0007 exists to protect. So it gets a rule, and a key, of its + # own. See docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md. + # + # This is NOT the "a second rule would be a second copy of the key" case the + # note above warns about: that argument is against two rules sharing one + # recipient. These two hold different recipients on purpose, which is the + # whole point. + # + # Placeholder until the lab guest exists. `make secrets-init STACK=lab`, run + # ON that guest, fills it in — and refuses if the key it would write is + # already a recipient below, because that would collapse the two rules back + # into one. + - path_regex: secrets/lab\..*\.sops\.ya?ml$ + age: >- + REPLACE_WITH_LAB_AGE_PUBLIC_KEY + - path_regex: (secrets/.*|backups/firewall/.*)\.sops\.ya?ml$ age: >- age1yrdu996u5mhdh0qf93l7s8zz8stneqnqxpncrcarrmgxvsy264rqmkcs6x diff --git a/README.md b/README.md index f62584e..a1e90d2 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,9 @@ Full topology and data flow in [`docs/architecture.md`](docs/architecture.md). │ ├── alloy/ # the agent config directory, shipped to every host │ ├── snmp-exporter/ # generator.yaml is the source of truth │ └── grafana/ # provisioning + 7 dashboards +├── stacks/lab/ # the lab's own stack — four services, not yet deployed +│ # runs on a guest on Saruman, never remote-writes +│ # to VLAN 99. See its README and ADR-0020 ├── secrets/ # SOPS-encrypted; see secrets/README.md ├── scripts/ # bootstrap, render, validate, pin-digests, purge ├── SECURITY.md # disclosure policy and known exposure diff --git a/docs/architecture.md b/docs/architecture.md index 89f1bc7..31bdbaf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -181,7 +181,8 @@ hole from the monitoring VLAN into the monitored one. | Host | VLAN | Stack | Contents | | --- | --- | --- | --- | | `prometheus` (10.0.99.20) | 🔴 99 | [`stacks/observability`](../stacks/observability) | Prometheus, Alertmanager, Loki, Grafana, snmp-exporter, blackbox-exporter, Alloy | -| `Saruman` (10.0.30.110) | 🟢 30 | *(none yet)* | Proxmox VE 9, no guests — see [roadmap](roadmap.md); Alloy agent (native package) | +| `Saruman` (10.0.30.110) | 🟢 30 | *(none — and none intended)* | Proxmox VE 9, no guests yet — see [roadmap](roadmap.md); Alloy agent (native package). It runs no compose stack by decision, not by omission: Docker would rewrite the iptables its own firewall relies on ([ADR-0014](adr/0014-put-ifrit-on-imaginationlan-and-give-the-targets-no-route.md)), which is why the agent here is the native package and why `stacks/lab` runs in a guest | +| `alexander` (10.0.30.40) | 🟢 30 | [`stacks/lab`](../stacks/lab) | **Not built yet** — the guest is [#262](https://github.com/Gerrrt/HomeLab/issues/262), the stack is committed and deployable. Prometheus, Loki, Grafana, Alloy: the lab's own observability, which never remote-writes to VLAN 99 ([ADR-0007](adr/0007-defensive-estate-and-offensive-range.md), [ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md)) | | `oracle` (10.0.99.30) | 🔴 99 | *(none — hand-run containers)* | The Lemmiwinks wiki and its Postgres, since 2025-11-12 ([ADR-0011](adr/0011-keep-the-wiki-internal.md)); Alloy agent (Docker, `scripts/deploy-agent.sh`); the off-host copy of the firewall export (`make backup-firewall`). The estate's host for small off-host jobs — [ADR-0015](adr/0015-give-oracle-the-off-host-jobs.md) | One directory per stack, not one per service. A stack is the unit that gets diff --git a/docs/network.md b/docs/network.md index cf18be9..ae9eb97 100644 --- a/docs/network.md +++ b/docs/network.md @@ -278,7 +278,21 @@ Where things get broken on purpose. dedicated port, `Saruman` is the Proxmox install. They are separate addresses and separate names, and conflating them is a mistake this document previously made. -- `Saruman` currently runs no guests. +- `Saruman` currently runs no guests. The first will be `alexander`, at + `10.0.30.40` — a static below `.100` with a reservation, single-homed on this + segment like its host — which runs + [`stacks/lab`](../stacks/lab): the lab's own Prometheus, Loki, Grafana and + Alloy. The stack is built and committed; the guest is + [#262](https://github.com/Gerrrt/HomeLab/issues/262), which is why it is + described here and not in the table above. **It is a guest and not the + hypervisor for a reason**: a compose stack is Docker, and Docker would + rewrite the iptables of the box whose own firewall ADR-0014 relies on — the + same fact that put the native `.deb` agent on `Saruman` rather than a + container + ([ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md)). + It gets **no** pass into Winterfell: the rule below is the hypervisor's, and + ADR-0007's "guests get no such rule" covers this one too. Nothing in that + stack remote-writes off the segment. - `Saruman` runs an Alloy agent and is the one host on this segment with a path into Winterfell: a single pass, `10.0.30.110 → 10.0.99.20` on 9090 and 3100 TCP, unlogged and above the ADR-0014 tripwire. The hypervisor's own telemetry diff --git a/docs/observability.md b/docs/observability.md index 4275d80..93c910d 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -2,6 +2,29 @@ What is collected, where it goes, and how to change it. +**This document describes the estate's stack, on `prometheus` (10.0.99.20).** +There is a second one. [`stacks/lab`](../stacks/lab) is the lab's own +Prometheus, Loki, Grafana and Alloy, and it is deliberately not part of any of +what follows: no series it holds reaches this Prometheus, no log line reaches +this Loki, and none of the alert rules or dashboards below can see it. That is +ADR-0007's decision — lab telemetry stays in the lab, so that deliberately +hostile data never lands in the store the estate is actually run from — and +[ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md) +settles its shape. It is built but not yet deployed; the guest that runs it is +[#262](https://github.com/Gerrrt/HomeLab/issues/262). + +The one path that does cross belongs to the hypervisor and not to any guest: +`Saruman`'s own agent remote-writes here over a single unlogged pass +([#88](https://github.com/Gerrrt/HomeLab/issues/88)). A DL360 with an ageing +mirrored pair is estate hardware, and its health belongs with the rest of the +estate's. + +The consequence worth carrying into everything below: **nothing here can tell a +quiet lab from a dead one.** `RemoteWriteJobStale` keys on jobs that arrive on +this Prometheus, so by construction it can never cover a stack that never +arrives. That gap is [#257](https://github.com/Gerrrt/HomeLab/issues/257), and +it is not closed by anything in this document. + ## What is collected | Source | Via | Interval | Examples | diff --git a/docs/roadmap.md b/docs/roadmap.md index 4ace329..93c03ad 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -298,12 +298,19 @@ what left this one unfireable for months. `*_URL` variables changed, and that file has two sinks, so a lab without a Prometheus points the second one at `10.0.99.20` and inverts the isolation the ADR exists for. - In order — [#262](https://github.com/Gerrrt/HomeLab/issues/262) the guest, - then [#263](https://github.com/Gerrrt/HomeLab/issues/263) the validators, - because `STACK ?=` reaches the lifecycle targets and stops there and every - checker in this repository is pinned to `stacks/observability`, so a second - stack today would be one CI has never seen; then - [#264](https://github.com/Gerrrt/HomeLab/issues/264) the stack itself. + [#264](https://github.com/Gerrrt/HomeLab/issues/264) is built: + `stacks/lab/` holds the compose file, both configs, four alert rules and + their unit tests, and the secrets template. What is left of it is a deploy, + which needs [#262](https://github.com/Gerrrt/HomeLab/issues/262) — the guest + on `Saruman` — to exist first. Building it made the tooling stack-aware + (`render-config.sh` derives its required keys per stack rather than demanding + the estate's ten, `reload-config.sh` skips services a stack does not declare, + `bootstrap.sh` refuses to give one age key both stacks) and gave `.sops.yaml` + the lab rule ADR-0020 asked for. It did **not** touch + [#263](https://github.com/Gerrrt/HomeLab/issues/263): `STACK ?=` reaches the + lifecycle targets and stops there, every checker is still pinned to + `stacks/observability`, and so the new stack is one CI has never seen — + validated only by hand and by the checks that already follow `STACK`. [#265](https://github.com/Gerrrt/HomeLab/issues/265) the domain is what everything else is pointed at, and blocks both [#266](https://github.com/Gerrrt/HomeLab/issues/266) Wazuh — the heaviest diff --git a/scripts/backup-firewall.sh b/scripts/backup-firewall.sh index 09f51c7..8f44b0f 100755 --- a/scripts/backup-firewall.sh +++ b/scripts/backup-firewall.sh @@ -137,8 +137,26 @@ need() { command -v "$1" >/dev/null 2>&1 || { red "missing dependency: $1"; exit # The age recipient is read from .sops.yaml rather than duplicated here. One # source of truth for the key; rotating it in .sops.yaml rotates it here too. +# +# Read from the rule that actually covers backups/firewall/, not the first key +# in the file. This was `grep ... | head -1` while .sops.yaml held exactly one +# creation_rule, which made "first key" and "the right key" the same string. +# ADR-0020 added a second rule above it for the lab stack, and the moment that +# rule's placeholder is replaced with a real key, `head -1` would encrypt every +# firewall backup to the LAB guest — silently, since sops would happily do it +# and the file would still look like a backup. That is the same trust inversion +# the second rule was created to prevent, arriving through the back door. +# +# Anchored on the path_regex naming backups/firewall, so it follows the rule +# rather than the ordering. Exits at the first key after that line: `age:` uses +# a folded scalar, so the key is on the line following the one that matches. recipient() { - grep -oE 'age1[0-9a-z]{50,}' "$SOPS_POLICY" | head -1 + awk ' + /path_regex:.*backups\/firewall/ { inrule = 1 } + inrule && match($0, /age1[0-9a-z]{50,}/) { + print substr($0, RSTART, RLENGTH); exit + } + ' "$SOPS_POLICY" } # Newest first, sorted by NAME and not by mtime. The stamp is UTC ISO-8601 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index a91c812..cad2d96 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -51,9 +51,40 @@ info "public key: ${PUBLIC_KEY}" # --------------------------------------------------------------------------- # 2. register it in .sops.yaml # --------------------------------------------------------------------------- -if grep -q "REPLACE_WITH_YOUR_AGE_PUBLIC_KEY" "${SOPS_CONFIG}"; then - info "writing public key into .sops.yaml" - sed -i.bak "s|REPLACE_WITH_YOUR_AGE_PUBLIC_KEY|${PUBLIC_KEY}|" "${SOPS_CONFIG}" +# Which placeholder belongs to THIS stack. +# +# There is no longer one placeholder to fill. .sops.yaml carries a creation_rule +# per stack that needs its own recipient — ADR-0020 gives `lab` one, because the +# single rule that used to match all of secrets/ meant any recipient added to it +# could decrypt every other stack's credentials too. Filling in "the first +# placeholder found" would write the lab guest's key into the estate's rule, or +# the estate's into the lab's, which is the failure that decision exists to +# prevent. +# +# The generic name is kept as the fallback so a fresh clone bootstrapping +# `observability` behaves exactly as it always has. +PLACEHOLDER="REPLACE_WITH_${STACK^^}_AGE_PUBLIC_KEY" +grep -q "${PLACEHOLDER}" "${SOPS_CONFIG}" 2>/dev/null \ + || PLACEHOLDER="REPLACE_WITH_YOUR_AGE_PUBLIC_KEY" + +if grep -q "${PLACEHOLDER}" "${SOPS_CONFIG}"; then + # Refused rather than warned about. Reaching here means this host's key is + # already a recipient of some other rule in this file, and is now being asked + # to become ${STACK}'s as well — one key that decrypts both stacks, which is + # the exact collapse the separate rules exist to stop. It is also the easy + # mistake: `make secrets-init STACK=lab` typed on the monitoring host rather + # than on the lab guest does precisely this, and the result would look like a + # successful bootstrap. + if grep -q "${PUBLIC_KEY}" "${SOPS_CONFIG}"; then + die "this host's key is already a recipient in ${SOPS_CONFIG##*/}, and +making it ${STACK}'s recipient as well would give one key both stacks. + +Run this on the host that will run ${STACK}, so that stack gets a key of its +own. If one key for both is genuinely what you want, edit .sops.yaml by hand — +it should be a decision, not a side effect of where you happened to type this." + fi + info "writing public key into .sops.yaml (${PLACEHOLDER})" + sed -i.bak "s|${PLACEHOLDER}|${PUBLIC_KEY}|" "${SOPS_CONFIG}" rm -f "${SOPS_CONFIG}.bak" elif grep -q "${PUBLIC_KEY}" "${SOPS_CONFIG}"; then info ".sops.yaml already lists this key" @@ -61,6 +92,7 @@ else warn ".sops.yaml lists a different age recipient." warn "Add this key as an additional recipient by hand, then run:" warn " sops updatekeys ${SECRETS_FILE}" + warn "Add it to the rule matching secrets/${STACK}. — NOT to another stack's." fi # --------------------------------------------------------------------------- diff --git a/scripts/check_docs.py b/scripts/check_docs.py index 62aafe4..ae18080 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -178,6 +178,16 @@ def strip_md(cell: str) -> str: return cell.strip() +# A host-and-stack row for something that does not exist yet. See the block +# above check_host_stack_table() for what it does there; count_alloy_agents() +# below reads it too, because a row describing an undeployed host describes an +# undeployed agent. +# +# Matched against the RAW cell, never strip_md()'s output: that helper removes +# every `*`, which takes the emphasis with it and leaves the marker unfindable. +NOT_BUILT = re.compile(r"\*\*not built yet\*\*", re.I) + + # --------------------------------------------------------------------------- # Facts, computed from the configs # --------------------------------------------------------------------------- @@ -253,6 +263,13 @@ def count_alloy_agents() -> int: is the machine-readable side. A row whose Contents cell names Alloy is an agent; "two Alloy agents" in hardware.md was unguarded and stale for as long as it took to deploy a third (#88). + + A row marked NOT_BUILT is not counted, because an agent on a host that does + not exist is not an agent. `stacks/lab` declares one for `alexander`, and it + collects nothing until that guest is racked (#262). The exclusion is not a + convenience: dropping the marker on the commit that builds the host pushes + this count to four and fails hardware.md's "three Alloy agents" in the same + run, which is exactly when that sentence should be forced to change. """ tables = tables_under( ARCH_MD.read_text(encoding="utf-8"), @@ -262,7 +279,9 @@ def count_alloy_agents() -> int: return 0 return sum( 1 for row in tables[0][1:] - if len(row) > 3 and "alloy" in strip_md(row[3]).lower() + if len(row) > 3 + and "alloy" in strip_md(row[3]).lower() + and not NOT_BUILT.search(row[3]) ) @@ -407,6 +426,27 @@ def check_snmp_targets() -> list[str]: # --------------------------------------------------------------------------- # 3. Host and stack mapping # --------------------------------------------------------------------------- +# A stack directory can legitimately exist before the host that runs it does. +# `stacks/lab` was committed complete — compose file, configs, rules, unit +# tests — while the guest that will run it, `alexander`, was still an issue +# (#262, #264). ADR-0004 puts the host-to-stack mapping in this document, so the +# row has to exist; but docs/network.md is the inventory of what is actually on +# the wire, and writing an unbuilt guest into it would be the precise kind of +# false claim this file exists to catch. It would also mean inventing a MAC, a +# device and an OS for a machine whose distribution is explicitly undecided. +# +# So the row is marked, and the marker INVERTS the check rather than switching +# it off. A normal row's host must APPEAR in network.md; a row marked "not built +# yet" must be ABSENT from it. That is what makes the marker self-clearing — +# rack the host, add its network.md row, and this fails saying the marker is +# stale, instead of quietly tolerating a row that claims both things at once. +# The address is still required and still checked for collisions, so a plan is +# held to the same standard as a deployment. +# +# The marker itself is defined next to strip_md(), because count_alloy_agents() +# reads it too. + + def check_host_stack_table() -> list[str]: text = ARCH_MD.read_text(encoding="utf-8") tables = tables_under(text, re.compile(r"^##\s+Host and stack mapping")) @@ -424,6 +464,7 @@ def check_host_stack_table() -> list[str]: ip_match = re.search(r"(\d+\.\d+\.\d+\.\d+)", host_cell) vlan_match = re.search(r"(\d+)", strip_md(row[1])) named_stacks.update(re.findall(r"stacks/([a-z0-9-]+)", row[2])) + planned = bool(len(row) > 3 and NOT_BUILT.search(row[3])) if not (ip_match and vlan_match): problems.append( @@ -438,7 +479,35 @@ def check_host_stack_table() -> list[str]: r for r in rows_for_vlan if strip_md(r[0]).lower() == host.lower() and ip in strip_md(r[1]) ] - if not hit: + + if planned: + # The VLAN must be one network.md actually describes, or a typo'd + # segment would make every assertion below vacuously true. + if not rows_for_vlan: + problems.append( + f"docs/architecture.md plans {host} on VLAN {vlan}, which " + f"docs/network.md has no table for" + ) + if hit: + problems.append( + f"docs/architecture.md still marks {host} 'not built yet', " + f"and docs/network.md now lists it at {ip} on VLAN {vlan} — " + f"it has been built, so drop the marker" + ) + # An unbuilt host planned onto an address something else already + # holds is a real conflict, and the cheapest possible moment to + # find it is before anyone racks it. + clash = [ + r for r in rows_for_vlan + if ip in strip_md(r[1]) and strip_md(r[0]).lower() != host.lower() + ] + if clash: + problems.append( + f"docs/architecture.md plans {host} at {ip}, which " + f"docs/network.md already gives to " + f"{strip_md(clash[0][0])} on VLAN {vlan}" + ) + elif not hit: problems.append( f"docs/architecture.md places {host} at {ip} on VLAN {vlan}; " f"docs/network.md does not list it there" diff --git a/scripts/reload-config.sh b/scripts/reload-config.sh index cf986b0..e6391cf 100755 --- a/scripts/reload-config.sh +++ b/scripts/reload-config.sh @@ -328,8 +328,37 @@ restart_alloy() { ok "alloy (restarted)" } +# SERVICES above is the union across every stack, not the contents of this one. +# `stacks/lab` runs Prometheus, Loki, Grafana and Alloy and no Alertmanager, +# snmp-exporter or blackbox-exporter (ADR-0020), so reloading the array as +# written died on the first service that stack does not have — and it died +# through reload_one's "is not running, so there is nothing to reload", which is +# the correct message for a stopped service and a wrong one for an absent one. +# +# The two cases must stay separate, which is why this filters on what the +# compose file DEFINES rather than on what is running. A service this stack +# declares and is not running is still a failed deploy and still dies below; +# only one it never declared is skipped. +# +# `compose config --services` rather than grepping for two-space-indented keys: +# the answer is compose's own, so a service list this script disagrees with is +# not a shape it can be tricked by. It needs the ${VAR:?} guards satisfied, +# which is true wherever this runs — `make up` renders .env immediately before +# calling this, and `make reload` acts on a stack that is already up. If it +# cannot answer at all, fall back to the full array: that is exactly today's +# behaviour, so the failure mode is the one that has always been there rather +# than a new one. +declared="" +if ! declared="$("${COMPOSE[@]}" config --services 2>/dev/null)"; then + declared="" +fi + for entry in "${SERVICES[@]}"; do - reload_one "${entry%%:*}" "${entry##*:}" + svc="${entry%%:*}" + if [[ -n "${declared}" ]] && ! grep -qx "${svc}" <<<"${declared}"; then + continue + fi + reload_one "${svc}" "${entry##*:}" done restart_alloy diff --git a/scripts/render-config.sh b/scripts/render-config.sh index 6d00ad4..ae92d25 100755 --- a/scripts/render-config.sh +++ b/scripts/render-config.sh @@ -113,18 +113,69 @@ unset absent clobbered cert info "decrypting $(basename "${SECRETS_FILE}")" load_secrets "${STACK}" -REQUIRED=( - GRAFANA_ADMIN_PASSWORD - GRAFANA_RENDERER_TOKEN - ALERTMANAGER_WEBHOOK_URL - ALERTMANAGER_URGENT_WEBHOOK_URL - ALERTMANAGER_SECURITY_WEBHOOK_URL - ALERTMANAGER_HEARTBEAT_URL - SNMP_COMMUNITY_PFSENSE - SNMP_COMMUNITY_APC - SNMP_COMMUNITY_MOKERLINK - SNMP_COMMUNITY_ILO -) +# The two inputs whose presence decides what this stack needs. Declared here +# rather than beside the blocks that render them, because the required-key list +# below is built from whether they exist. +SNMP_SRC="${STACK_DIR}/snmp-exporter/snmp.yaml" +AM_CONFIG="${STACK_DIR}/alertmanager/alertmanager.yaml" + +# --------------------------------------------------------------------------- +# What this stack requires +# +# Derived from the stack, not listed for one of them. Every render below is +# already guarded by `[[ -f ... ]]`, so the script has always coped with a +# stack that has no snmp-exporter — but this list was a flat array of all ten +# keys, which demanded four SNMP communities and four Alertmanager URLs from +# any stack at all. `make render STACK=lab` died on a lab that runs neither, +# naming secrets that stack has no use for. +# +# The Grafana half is read out of the stack's own compose.yaml: a ${VAR:?} +# guard IS the declaration that the service cannot start without it, so a +# second copy here could only ever drift from it. Same derivation +# seed-validation-env.sh uses against the same guards, for the same reason. +# The three excluded names are the ones THIS script writes into .env further +# down — they are guarded in compose.yaml but they are not secrets, and asking +# SOPS for them would fail every render. +# +# For `stacks/observability` this produces the identical ten names the flat +# array held, which is the property that makes the change safe: the two Grafana +# keys come from its compose guards, and both conditionals below are true. +# --------------------------------------------------------------------------- +REQUIRED=() +while read -r var; do + [[ -n "${var}" ]] || continue + case "${var}" in RENDER_UID | RENDER_GID | LOG_READ_GID) continue ;; esac + REQUIRED+=("${var}") +done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*:\?' "${STACK_DIR}/compose.yaml" \ + | sed 's/^\${//; s/:?$//' | sort -u) + +# The SNMP community names stay written out rather than derived from the +# placeholders in snmp.yaml, and that is deliberate. scripts/snmp-targets.sh +# --check asserts this array against the device inventory by grepping for each +# name on a line of its own — it is one of the five copies of the device list +# that check exists to hold together, and deriving it here would remove the +# copy rather than the drift, leaving --check asserting nothing. Keep the +# one-name-per-line shape. +if [[ -f "${SNMP_SRC}" ]]; then + REQUIRED+=( + SNMP_COMMUNITY_PFSENSE + SNMP_COMMUNITY_APC + SNMP_COMMUNITY_MOKERLINK + SNMP_COMMUNITY_ILO + ) +fi + +# One entry per AM_CHANNELS entry below; the pairing is asserted after they are +# rendered, so a channel added there without a key here fails at deploy time. +if [[ -f "${AM_CONFIG}" ]]; then + REQUIRED+=( + ALERTMANAGER_WEBHOOK_URL + ALERTMANAGER_URGENT_WEBHOOK_URL + ALERTMANAGER_SECURITY_WEBHOOK_URL + ALERTMANAGER_HEARTBEAT_URL + ) +fi + missing=() for var in "${REQUIRED[@]}"; do [[ -n "${!var:-}" ]] || missing+=("${var}") @@ -134,7 +185,6 @@ done # --------------------------------------------------------------------------- # Render snmp.yaml # --------------------------------------------------------------------------- -SNMP_SRC="${STACK_DIR}/snmp-exporter/snmp.yaml" SNMP_OUT_DIR="${STACK_DIR}/snmp-exporter/.rendered" if [[ -f "${SNMP_SRC}" ]]; then info "rendering snmp.yaml" @@ -202,7 +252,7 @@ AM_CHANNELS=( "ALERTMANAGER_HEARTBEAT_URL:heartbeat_url" ) AM_OUT_DIR="${STACK_DIR}/alertmanager/.rendered" -if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then +if [[ -f "${AM_CONFIG}" ]]; then info "rendering ${#AM_CHANNELS[@]} alertmanager receiver URL(s)" mkdir -p "${AM_OUT_DIR}" chmod 700 "${AM_OUT_DIR}" @@ -229,7 +279,7 @@ if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then # `secrets/[a-z_]+` also matched `secrets/observability.sops.yaml` in this # file's own header comment, and reported the header as a missing channel. done < <(grep -oE 'url_file:[[:space:]]*/etc/alertmanager/secrets/[a-z_]+' \ - "${STACK_DIR}/alertmanager/alertmanager.yaml" \ + "${AM_CONFIG}" \ | sed 's|.*/||' | sort -u) fi diff --git a/secrets/lab.example.yaml b/secrets/lab.example.yaml new file mode 100644 index 0000000..56e94d6 --- /dev/null +++ b/secrets/lab.example.yaml @@ -0,0 +1,43 @@ +--- +# Template for secrets/lab.sops.yaml. +# +# ONE KEY. That is not an oversight — it is what the lab stack actually needs, +# and the short list is worth reading as a description of the stack: no SNMP +# communities, because it polls no devices; no Alertmanager receiver URLs, +# because it has no Alertmanager (ADR-0020); no renderer token, because it +# ships no dashboards to screenshot. +# +# scripts/render-config.sh derives the required set per stack rather than +# demanding all ten the estate needs, so `make render STACK=lab` asks for this +# and nothing else. +# +# --------------------------------------------------------------------------- +# Create this ON THE LAB GUEST, not on the monitoring host +# --------------------------------------------------------------------------- +# +# make secrets-init STACK=lab # first time only — creates the age keypair +# make secrets-edit STACK=lab # opens the encrypted file in $EDITOR +# +# `.sops.yaml` gives secrets/lab.*.sops.yaml its own creation_rule and its own +# age recipient, above the catch-all that covers everything else under +# secrets/. That separation is the point: the single rule it sits above matches +# ALL of secrets/, so a key added there could decrypt the estate's SNMP +# communities and Grafana admin password too — a lab host holding the +# credentials of the estate it is meant to be isolated from. +# +# `make secrets-init STACK=lab` run on the monitoring host would do exactly +# that, which is why bootstrap.sh now refuses when the key it would write is +# already a recipient of another rule. Run it where the stack runs. + +# Grafana initial admin login. Grafana only reads this when it creates the +# admin user, so changing it later has no effect — change it in the UI, or wipe +# the grafana-data volume. +# +# NOT the same password as the estate's Grafana. Two Grafanas that share a +# password are one credential, and the whole reason there are two of them is +# that the lab is assumed to be the one that gets compromised. Generate one +# with `make gen-secret`. +GRAFANA_ADMIN_PASSWORD: change-me-to-something-long + +# GRAFANA_ADMIN_USER is optional and defaults to `admin` in compose.yaml. Add +# it here if you want a different one. diff --git a/stacks/lab/.env.example b/stacks/lab/.env.example new file mode 100644 index 0000000..f0caa5f --- /dev/null +++ b/stacks/lab/.env.example @@ -0,0 +1,47 @@ +# Non-sensitive tunables for the lab stack. +# +# Edit this file, not .env — scripts/render-config.sh regenerates .env from it +# on every `make up STACK=lab`. Secrets do NOT belong here; see secrets/README.md. + +# Address the published ports bind to. +# +# This guest is single-homed on VLAN 30 (ADR-0007: no trunk, no VLAN-aware +# bridge), so 0.0.0.0 here means ImaginationLAN and nothing else — there is no +# second interface for it to reach further on. That is a meaningfully smaller +# claim than the same line makes on the monitoring host, which sits on the +# management VLAN. +BIND_ADDR=0.0.0.0 + +# The one published port: Grafana, opened from Hicks over the 50 -> 30 rule +# that already exists (ADR-0007). Nothing else in this stack is published — +# compose.yaml says where the `ports:` block goes when #265 gives Prometheus +# and Loki their first off-host clients. +GRAFANA_PORT=3000 + +# Alloy's debug UI, pinned to 127.0.0.1 in compose.yaml. +ALLOY_PORT=12345 + +# Uncomment together with the matching `ports:` blocks in compose.yaml when the +# lab's guests start pushing. Publishing them before that opens an +# unauthenticated remote-write receiver and an unauthenticated log push +# endpoint on the segment that exists to hold attackers, with nothing using +# either (ADR-0012). +# PROMETHEUS_PORT=9090 +# LOKI_PORT=3100 + +# ALLOY_HOSTNAME is NOT set here. scripts/render-config.sh writes it from +# `hostname` at render time — it differs per monitored host, and setting it +# here would pin every host to the same name. + +# Metric retention. Loki's is set separately in loki/loki-config.yaml +# (retention_period) — keep the two in step. +# +# Deliberately shorter than the estate's 30 days, and deliberately not derived +# the way the estate's was: this stack has never run, so there is nothing to +# derive from. It is a bound against ADR-0007's stated constraint — a single +# mirrored pair of 7.2K disks shared with a Windows domain, Wazuh's indexer, +# Velociraptor and PBS. compose.yaml carries the queries to re-derive both once +# the lab has run for a fortnight, and PrometheusSizeRetentionActive in +# prometheus/rules/lab.rules.yaml is what says the ceiling started binding. +PROMETHEUS_RETENTION=15d +PROMETHEUS_RETENTION_SIZE=4GB diff --git a/stacks/lab/README.md b/stacks/lab/README.md new file mode 100644 index 0000000..fa96fbe --- /dev/null +++ b/stacks/lab/README.md @@ -0,0 +1,122 @@ +# Lab observability stack + +Runs on `alexander` (10.0.30.40), VLAN 30 — a **guest on `Saruman`**, not the +hypervisor. A compose stack is Docker, and Docker rewrites the iptables of a +box whose own firewall ADR-0014 relies on, which is why `Saruman` carries the +native `.deb` agent instead ([#88]) and why this runs one level down. +[ADR-0020](../../docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md). + +```bash +make up STACK=lab # from the repository root +``` + +| Service | Image | Port | Purpose | +| --- | --- | --- | --- | +| `prometheus` | `prom/prometheus` | *internal* | Metrics store, remote-write receiver, rule evaluation | +| `loki` | `grafana/loki` | *internal* | Log store | +| `grafana` | `grafana/grafana-oss` | 3000 (https) | Dashboards — the only published port, and the only service that terminates TLS or authenticates | +| `alloy` | `grafana/alloy` | 12345 (localhost) | Metric and log collection | + +Four services, where the estate has seven. What is absent is as deliberate as +what is here: + +- **No Alertmanager.** Nothing in the lab pages. ADR-0020 decided it, and it is + explicitly not an answer to [#257] — that asks whether the lab's *liveness* + may cross to the estate even though its telemetry may not. +- **No snmp-exporter.** `shiva`, the only SNMP device on this segment, is + polled by the estate over the exception ADR-0013 records. Two stacks polling + one device is two answers to "when did it last respond". +- **No blackbox-exporter, no renderer.** Nothing to probe from here yet, and no + dashboards to screenshot. + +## Why this exists at all + +ADR-0007: **lab telemetry stays in the lab.** Nothing in this stack +remote-writes to `10.0.99.20`; both of Alloy's sinks are services in this +compose file. The one exception on this segment predates it and belongs to the +hypervisor, not to any guest — `Saruman`'s own agent, over a single unlogged +pass ([#88]). + +## Layout + +```text +compose.yaml four services, one network, health-gated ordering +.env.example non-sensitive tunables — edit this, not .env +prometheus/ + prometheus.yaml four scrape jobs; no alerting block, no file_sd + rules/lab.rules.yaml 4 rules — this stack watching itself, nothing else + tests/lab.test.yaml promtool unit tests; all four rules, firing + quiet +loki/loki-config.yaml single-binary, filesystem, 15-day retention, no ruler +grafana/ + provisioning/ two datasources + dashboard provider + dashboards/README.md why there are no dashboards yet +``` + +No `alloy/` directory, deliberately. `compose.yaml` mounts `config.alloy` and +`docker.alloy` out of `../observability/alloy/` — ADR-0007 asks for the agent +config "reused unchanged", and a copy is reused-until-someone-edits-one. +`scripts/deploy-agent.sh` exists because `oracle` drifted four separate ways +from a hand-copied agent setup; its header states the rule as *"the fix is to +not copy."* `syslog.alloy` is not mounted: it opens a UDP listener for the +firewall's logs, which is the monitoring host's job and would be the wrong +thing entirely on this segment. + +## Things worth knowing before editing + +- **Nothing here is validated by CI yet.** Every checker in this repository is + pinned to `stacks/observability` — `validate.sh`, the four Python checkers, + `check_loki_rules.sh`, `seed-validation-env.sh` and `ci.yml`. Making them + multi-stack is [#263]. Until it lands, run the checks against this stack by + hand; the commands are at the bottom of this file. +- **Nothing converges this stack, either.** [#99] replaced deploying over SSH + with `scripts/converge.sh` on an hourly timer, and that script runs a bare + `make up` — which is `STACK=observability`, on the monitoring host. This + stack is deployed by hand, on this guest, and a commit that changes it + reaches the lab when someone goes and applies it. Worth knowing before + assuming a merged change is running. +- **The retention figures are a bound, not a measurement.** 15 days and 4 GiB, + against ADR-0007's "sized against spindles, not RAM". `compose.yaml` carries + the queries to re-derive them once this has run for a fortnight, and + `PrometheusSizeRetentionActive` is what says the ceiling started binding. +- **Grafana needs its own leaf, from the same CA as the estate's.** The + `grafana` DNS SAN is load-bearing: the `grafana` scrape job connects to the + compose service name and verifies against it. + + ```bash + make certs ARGS="--host grafana-lab.matrix.elysium --ip 10.0.30.40 --dns grafana" + ``` + +- **Secrets are one key, and it is created on this guest.** See + [`secrets/lab.example.yaml`](../../secrets/lab.example.yaml) — running + `make secrets-init STACK=lab` on the monitoring host would give one age key + both stacks, and `bootstrap.sh` now refuses rather than doing it quietly. +- **Prometheus and Loki are not published.** ADR-0012 publishes a port only + when something off-host uses it, and today only Alloy talks to them, over the + compose network. `compose.yaml` marks the exact lines to uncomment when [#265] + gives them their first real clients. +- **Image tags are pinned here but bumped separately.** `.github/dependabot.yml` + now watches this directory as well as the estate's, so the two do not drift. + Versions are deliberately absent from the table above — Dependabot only edits + `compose.yaml`, so a version written anywhere else goes stale the moment it + lands (#73). + +## Validate before deploying + +`make validate` does **not** cover this stack ([#263]). Until it does: + +```bash +make check-rules STACK=lab +``` + +That target already follows `STACK`, because it uses `$(STACK_DIR)`. The rest +needs the image directly: + +```bash +docker compose -f stacks/lab/compose.yaml config -q +``` + +[#99]: https://github.com/Gerrrt/HomeLab/issues/99 +[#88]: https://github.com/Gerrrt/HomeLab/issues/88 +[#257]: https://github.com/Gerrrt/HomeLab/issues/257 +[#263]: https://github.com/Gerrrt/HomeLab/issues/263 +[#265]: https://github.com/Gerrrt/HomeLab/issues/265 diff --git a/stacks/lab/compose.yaml b/stacks/lab/compose.yaml new file mode 100644 index 0000000..cafb132 --- /dev/null +++ b/stacks/lab/compose.yaml @@ -0,0 +1,286 @@ +--- +# Lab observability stack for the defended estate on `Saruman`. +# +# Deployed on: alexander (10.0.30.40, VLAN 30 / ImaginationLAN) — a guest on +# `Saruman`, NOT the hypervisor itself. A compose stack is Docker, +# and Docker rewrites the iptables of a box whose own firewall +# ADR-0014 relies on, which is why #88 put the native .deb agent +# on the hypervisor and why this runs one level down. +# See docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md. +# Deploy with: make up STACK=lab (from the repository root) +# +# This stack exists so that lab telemetry never reaches 10.0.99.20 (ADR-0007). +# Nothing here remote-writes anywhere; both of Alloy's sinks are services in +# this file. +# +# --------------------------------------------------------------------------- +# On the comments in this file +# --------------------------------------------------------------------------- +# The settings below are shared with `stacks/observability/compose.yaml`, and +# the reasoning for them — why `init: true`, why `pids_limit` and not +# `mem_limit`, why `cgroup: host` and not `privileged`, why loki has no +# healthcheck — is written out at length there and is NOT repeated here. Two +# copies of an argument drift the same way two copies of a config do, and the +# argument is the half nobody re-checks. What IS written out here is every +# place this stack DIFFERS from that one, because those are the only lines a +# reader cannot get from the sibling file. + +name: lab + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-service-defaults: &service-defaults + restart: unless-stopped + logging: *default-logging + networks: [lab] + init: true + pids_limit: 512 + +services: + # --------------------------------------------------------------------------- + # Metrics store. + # + # ADR-0007 named Loki, Grafana and Alloy and no Prometheus, and in the same + # sentence said config.alloy is reused with the two *_URL variables the only + # difference. That file has two sinks. Without this service the second one + # has nowhere to point but 10.0.99.20, which is the one thing that ADR's + # Decision forbids — so the omission was an omission. ADR-0020. + # --------------------------------------------------------------------------- + prometheus: + <<: *service-defaults + image: prom/prometheus:v3.14.0@sha256:5ce7540c3c00ef4ab0c9d2c995c6a5b9c421f44b4a115d97a2c7af3b1c21cbb0 + container_name: lab-prometheus + user: "65534:65534" + command: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/prometheus + # Deliberately NOT the estate's 30d/12GB, and deliberately not derived + # the way those were either — there is nothing to derive from yet. The + # estate's numbers came from measured per-block write rates over a + # retained window; this stack has never run, so any figure here is a + # bound rather than a measurement, and writing it as though it were + # measured would be the more dishonest choice. + # + # What IS known is the constraint. ADR-0007: "128 GB and 48 threads + # against a single mirrored pair of 7.2K disks. The fleet is sized + # against spindles, not RAM." This volume shares those two spindles with + # a Windows domain (#265), Wazuh's OpenSearch indexer (#266), + # Velociraptor (#267) and PBS (#268) — and Wazuh is the one expected to + # be what actually runs out. So this starts small and is meant to be + # re-derived, not defended. + # + # Re-derive it the way the estate's was, once the domain exists and this + # has run for a fortnight: + # rate(prometheus_tsdb_head_samples_appended_total[1d]) + # prometheus_tsdb_storage_blocks_bytes + # and check whether the ceiling has ever bound: + # prometheus_tsdb_time_retentions_total (0 means it never has) + # + # If it has bound, retention is quietly shorter than the line above + # claims. That is the same failure the estate documents; unlike the + # estate, there is no PrometheusSizeRetentionActive rule here to say so + # yet, because there are no dashboards or rules beyond lab.rules.yaml. + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d} + - --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE:-4GB} + # Enabled with nothing pushing to it yet, which is the one place this + # departs from ADR-0012's "publish only what has a consumer" — the flag + # is not a published port, and the receiver is reachable only from this + # compose network until the `ports:` block below is uncommented. It is on + # because Alloy in this same stack pushes through it: the agent uses + # remote_write, not a scrape, exactly as it does on the monitoring host. + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + volumes: + - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./prometheus/rules:/etc/prometheus/rules:ro + # Only the CA, never a key — the grafana job below scrapes over https and + # verifies the leaf against it. The SAME CA as the estate's: one lab CA, + # two leaves. A second CA would mean a second certificate for every + # browser on Hicks to trust, for no gain. + - ../../certificates/ca.pem:/etc/prometheus/tls/ca.pem:ro + - prometheus-data:/prometheus + # No `ports:` — deliberately, and this is the line to change when #265 + # lands. ADR-0012 publishes a port only when something OFF this host uses + # it, and today nothing does: the only client is Alloy, on this compose + # network. When the Windows endpoints and the Wazuh host get agents they + # become real off-host clients, and this becomes: + # ports: + # - "${BIND_ADDR:-0.0.0.0}:${PROMETHEUS_PORT:-9090}:9090" + # Publishing it now would open an unauthenticated remote-write receiver on + # the segment that exists to hold attackers, months before anything pushes + # to it — "a rule nobody can test", in the shape SECURITY.md keeps finding. + expose: + - "9090" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Log store. No healthcheck, for the reason written out against the `loki` + # service in stacks/observability/compose.yaml: the image is distroless and + # holds no shell, wget or curl to probe with. Readiness is observed by the + # `loki` scrape job in prometheus.yaml instead. + # --------------------------------------------------------------------------- + loki: + <<: *service-defaults + image: grafana/loki:3.7.7@sha256:d70e4659623f3e109af669cae76fe2a5dd5be54e2298fe8aed380d982fbc2500 + container_name: lab-loki + user: "10001:10001" + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki-data:/loki + # No rules mount, unlike the estate's. Loki's ruler needs an Alertmanager + # to deliver to and this stack has none (ADR-0020) — a ruler configured + # with nowhere to send is a rule that evaluates and is discarded, which + # reads as coverage and is not. Log-based alerting for the lab is part of + # the liveness question in #257, not something to half-wire here. + # + # Not published, for the same reason as prometheus above: Alloy is the only + # client and it is on this network. #265 is when that stops being true. + expose: + - "3100" + + # --------------------------------------------------------------------------- + # Visualisation. The one service here a human opens, and the one this stack + # publishes: ADR-0007 says the lab's Grafana is reached from Hicks, over the + # 50 -> 30 rule that already exists. No new firewall rule anywhere. + # --------------------------------------------------------------------------- + grafana: + <<: *service-defaults + image: grafana/grafana-oss:13.0.2@sha256:5dad0df181cb644a14e13617b913b261a54f7d4fd4510721dba420929f35bea2 + container_name: lab-grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/lab.sops.yaml} + GF_SECURITY_COOKIE_SECURE: "true" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /etc/grafana/tls/cert.pem + GF_SERVER_CERT_KEY: /etc/grafana/tls/key.pem + GF_SECURITY_DISABLE_GRAVATAR: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + PROMETHEUS_URL: http://prometheus:9090 + LOKI_URL: http://loki:3100 + # No GF_RENDERING_* here, and no `renderer` service at the foot of this + # file. The renderer exists on the monitoring host to screenshot + # dashboards into docs/images/; this stack ships no dashboards yet, so + # there is nothing to render. That absence is also why no + # GRAFANA_RENDERER_TOKEN appears in secrets/lab.example.yaml — Grafana + # only refuses to start without one *once a rendering server is + # configured*, and none is. + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + # A second leaf from the SAME CA as the estate's Grafana. Issue it with: + # make certs ARGS="--host grafana-lab.matrix.elysium --ip 10.0.30.40 --dns grafana" + # The `grafana` DNS SAN is not decoration: the prometheus job in this + # stack connects to the compose service name and verifies against it. + - ../../certificates/grafana-lab.matrix.elysium.pem:/etc/grafana/tls/cert.pem:ro + - ../../certificates/grafana-lab.matrix.elysium-key.pem:/etc/grafana/tls/key.pem:ro + group_add: + - "${RENDER_GID:?run make render STACK=lab}" + ports: + - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_started + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "https://localhost:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Collection agent for this guest. + # + # The config is MOUNTED FROM stacks/observability/alloy/, not copied into a + # stacks/lab/alloy/ of its own. ADR-0007 asks for config.alloy "reused + # unchanged"; a copy is reused-until-someone-edits-one, and this repository + # has already paid for that lesson — scripts/deploy-agent.sh exists because + # `oracle` drifted four separate ways from a hand-copied agent setup, and its + # header states the rule: "the fix is to not copy." + # + # Two files, not the directory, and that is the one place this departs from + # the sibling stack's `./alloy:/etc/alloy:ro`. Mounting the directory would + # bring syslog.alloy, which opens a UDP listener for the firewall's logs on + # the host's real interface — on VLAN 30, aimed at a stack that must not + # receive the estate's telemetry. Shipping config.alloy always, docker.alloy + # to hosts with a socket, and syslog.alloy never is exactly the subset + # deploy-agent.sh already ships to a remote Docker host. + # + # The cost, stated: a NEW *.alloy file added to that directory does not reach + # this stack without a line here. That is the same property deploy-agent.sh + # has, and the alternative — a symlinked directory — does not work at all, + # because a bind mount carries the symlink and not its target, and the target + # would have to resolve inside the container. + # --------------------------------------------------------------------------- + alloy: + <<: *service-defaults + image: grafana/alloy:v1.19.2@sha256:b8ec653c44235fbe910879145dac3597d66b0aaecf60bcbbe82580767771a839 + container_name: lab-alloy + pids_limit: 1024 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + group_add: + - "${LOG_READ_GID:?run make render STACK=lab}" + # The image's own `alloy` group; alloy-data is 0770 473:473. Re-derive + # after an image bump — the sibling compose file has the command. + - "473" + cgroup: host + environment: + # The two variables ADR-0007 calls "the only difference", pointed at this + # stack's own stores. They resolve to the same service names the defaults + # in config.alloy use, so they are strictly redundant — and set anyway, + # because "lab telemetry stays in the lab" should be visible in the file + # an operator reads rather than inferred from a coalesce() three + # directories away. + LOKI_URL: http://loki:3100/loki/api/v1/push + PROMETHEUS_REMOTE_WRITE_URL: http://prometheus:9090/api/v1/write + ALLOY_HOSTNAME: ${ALLOY_HOSTNAME:-alloy} + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy + volumes: + - ../observability/alloy/config.alloy:/etc/alloy/config.alloy:ro + - ../observability/alloy/docker.alloy:/etc/alloy/docker.alloy:ro + - alloy-data:/var/lib/alloy/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log:/var/log:ro + - /:/rootfs:ro + ports: + # Debug UI, loopback only. No syslog receiver on 1514/udp, because + # syslog.alloy is not mounted above. + - "127.0.0.1:${ALLOY_PORT:-12345}:12345" + depends_on: + loki: + condition: service_started + prometheus: + condition: service_healthy + +networks: + lab: + driver: bridge + +volumes: + prometheus-data: + loki-data: + grafana-data: + alloy-data: diff --git a/stacks/lab/grafana/dashboards/README.md b/stacks/lab/grafana/dashboards/README.md new file mode 100644 index 0000000..98fdeca --- /dev/null +++ b/stacks/lab/grafana/dashboards/README.md @@ -0,0 +1,47 @@ +# Lab dashboards + +There are none yet, and that is a decision rather than an omission. + +The estate has seven, and copying them here is the obvious move and the wrong +one. They are built on the estate's datasources, the estate's metrics and the +estate's device inventory: `homelab-network` draws SNMP counters from a switch +this stack does not poll, `homelab-ups` draws an APC on the other side of a +firewall, and `homelab-security` draws labels that +`stacks/observability/alloy/syslog.alloy` extracts from firewall logs this +stack deliberately never receives. A copy would render four rows of empty +panels and one that works, and an empty panel is indistinguishable from a +broken collector — which is the failure mode this repository has been bitten by +more than any other (#62, #63, #71). + +What the lab needs is not known yet, because the thing it exists to observe +does not exist yet. The Windows domain is [#265]; Wazuh is [#266]. Both come +with their own questions about what is worth drawing. + +Until then, `Explore` against the two provisioned datasources is the whole +interface, and it works from the first `make up STACK=lab` — the provider in +`../provisioning/dashboards/dashboards.yaml` reads this directory every 30 +seconds, so the first dashboard committed here appears without a redeploy. + +## When you do add one + +The conventions in +[`stacks/observability/grafana/dashboards/README.md`](../../../observability/grafana/dashboards/README.md) +hold here too, and they are worth reading before starting rather than after: +they are what `scripts/check_dashboards.py` enforces. + +Two things about this stack specifically: + +- **`check_dashboards.py` does not enforce them here.** That checker is pinned + to `stacks/observability`, along with every other validator in the + repository. Making them multi-stack is [#263]. Until it lands, a dashboard + committed to this directory is checked by nothing, so check it by hand. +- **`make dashboards-export STACK=lab` works, and nothing reminds you to run + it.** `scripts/export-dashboards.sh` takes a stack argument, and the provider + here sets `allowUiUpdates: true` so that it can. What the lab does not have + is the estate's `dashboards-drift` timer, which runs `--check` and turns a + forgotten export into a stale job. Here, a dashboard edited in the UI and + never exported simply stays uncommitted. + +[#263]: https://github.com/Gerrrt/HomeLab/issues/263 +[#265]: https://github.com/Gerrrt/HomeLab/issues/265 +[#266]: https://github.com/Gerrrt/HomeLab/issues/266 diff --git a/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml b/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..80846b7 --- /dev/null +++ b/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,35 @@ +--- +# The provider is here and the folder it reads is empty on purpose — see +# ../../dashboards/README.md. Provisioning an empty directory is not an error: +# Grafana logs nothing and starts normally, and the first dashboard committed +# appears within updateIntervalSeconds without a redeploy. +# +# `allowUiUpdates: true` matches the estate, and matching is the point. +# `scripts/export-dashboards.sh` takes a stack argument, so `make +# dashboards-export STACK=lab` already works against this stack — and it can +# only work with `true`, because Grafana answers `400 Cannot save provisioned +# dashboard` under `false` and the export reads back the file it provisioned +# from. Setting `false` here would have made the lab the one stack where the +# documented export loop silently does nothing (#100). +# +# The gap that leaves, stated rather than discovered later: `true` gives up the +# guarantee that the running dashboard and the committed one are the same, and +# the estate buys that back with the `dashboards-drift` timer running +# `export-dashboards.sh --check`. Nothing runs that against this stack — the +# timer lives on the monitoring host and points at `observability` — so an edit +# made in the lab's Grafana and never exported ages silently. That is the same +# shape as everything else nothing watches here (#257), and it costs nothing +# until this directory has a dashboard in it. +apiVersion: 1 + +providers: + - name: lab + orgId: 1 + folder: Lab + type: file + disableDeletion: false + allowUiUpdates: true + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/stacks/lab/grafana/provisioning/datasources/datasources.yaml b/stacks/lab/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..092ce1b --- /dev/null +++ b/stacks/lab/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,41 @@ +--- +# Two datasources, not the estate's three: there is no Alertmanager in this +# stack (ADR-0020), so there is none to provision. A datasource pointed at a +# service that does not exist is not inert — Grafana's Alerting pages render it +# as a configured-but-unreachable receiver, which is a worse answer than an +# absent one. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: ${PROMETHEUS_URL} + isDefault: true + editable: false + jsonData: + timeInterval: 15s + httpMethod: POST + manageAlerts: false + prometheusType: Prometheus + exemplarTraceIdDestinations: [] + + - name: Loki + type: loki + uid: loki + access: proxy + url: ${LOKI_URL} + editable: false + jsonData: + maxLines: 2000 + # Clicking a hostname in a log line jumps to that host's metrics rather + # than making you retype it into a new query. Kept identical to the + # estate's so the two Grafanas behave the same way under the hands of + # someone who uses both. + derivedFields: + - name: host + matcherRegex: 'host="([^"]+)"' + datasourceUid: prometheus + url: '$${__value.raw}' + urlDisplayLabel: "View host metrics" diff --git a/stacks/lab/loki/loki-config.yaml b/stacks/lab/loki/loki-config.yaml new file mode 100644 index 0000000..e6c7114 --- /dev/null +++ b/stacks/lab/loki/loki-config.yaml @@ -0,0 +1,72 @@ +--- +# Single-binary Loki with filesystem storage, same shape as the estate's and +# for the same reason — see docs/adr/0003-observability-stack-selection.md. +# +# Two differences from stacks/observability/loki/loki-config.yaml, both +# deliberate: +# +# retention_period 360h (15 days), not 720h. It tracks the Prometheus +# retention in compose.yaml, which is a bound rather than +# a measurement — this store shares two 7.2K spindles with +# a Windows domain, Wazuh's indexer, Velociraptor and PBS. +# Keep the two in step: a log store outliving the metrics +# that explain it is half an investigation. +# +# no `ruler:` block The estate's ruler delivers to its Alertmanager. This +# stack has none (ADR-0020), and a ruler with nowhere to +# deliver evaluates rules and discards the result, which +# reads as coverage and is not. compose.yaml therefore +# mounts no rules directory either. + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + compaction_interval: 10m + +limits_config: + volume_enabled: true + reject_old_samples: true + reject_old_samples_max_age: 168h + # 15 days, matching PROMETHEUS_RETENTION in .env.example. + retention_period: 360h + max_query_series: 5000 + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/stacks/lab/prometheus/prometheus.yaml b/stacks/lab/prometheus/prometheus.yaml new file mode 100644 index 0000000..f9e334c --- /dev/null +++ b/stacks/lab/prometheus/prometheus.yaml @@ -0,0 +1,68 @@ +--- +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 30s + external_labels: + site: matrix.elysium + # The estate's says `winterfell`. This one names the segment it serves, so + # that a series copied into a paste, a screenshot or a bug report says which + # of the two stacks it came from without anyone having to remember there are + # two. They are deliberately never joined, so the label is the only thing + # that would ever tell them apart. + monitor: imaginationlan + +rule_files: + - /etc/prometheus/rules/*.rules.yaml + +# No `alerting:` block, because there is no Alertmanager in this stack — +# ADR-0020 decided that, and the reasoning is there. Rules below still evaluate; +# a firing alert is visible in this Prometheus's own /alerts and through +# Grafana. Nothing pages, and nothing is meant to. +# +# What that leaves uncovered is #257: the estate's RemoteWriteJobStale keys on +# jobs arriving on VLAN 99, so it can never see this stack, and a lab that has +# died looks exactly like a lab nobody is using. Adding an alertmanagers: block +# pointed at 10.0.99.20 would "fix" that by opening the remote-write path +# ADR-0007 exists to refuse. Whatever #257 settles goes outside this file. + +scrape_configs: + # --------------------------------------------------------------------------- + # The stack monitoring itself. Service names resolve on the compose network, + # so nothing here carries this guest's address. + # --------------------------------------------------------------------------- + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # This is also loki's readiness check. The image is distroless and can carry + # no compose healthcheck, so `up{job="loki"}` is the only thing that observes + # it — the same arrangement, and the same reasoning, as the estate's. + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + # Verified properly rather than with insecure_skip_verify: the lab CA is + # mounted into this container, and the leaf carries `grafana` as a SAN + # alongside its FQDN precisely so this scrape can check the name it dials. + - job_name: grafana + scheme: https + tls_config: + ca_file: /etc/prometheus/tls/ca.pem + server_name: grafana + static_configs: + - targets: ["grafana:3000"] + + - job_name: alloy + static_configs: + - targets: ["alloy:12345"] + +# No `snmp` or `blackbox` jobs: this stack runs neither exporter. The devices +# on this segment that answer SNMP — `shiva`, the iLO of this guest's own host +# — are polled by the estate's stack over the exception ADR-0013 records, and +# polling them from here as well would be two stacks scraping one device and +# two answers to "when did it last respond". +# +# No remote-write job list either. Agents on the lab's guests do not need one: +# they push, and a pushed series appears without a target to edit. That is +# #265's work, and it needs the ports: block in compose.yaml opened first. diff --git a/stacks/lab/prometheus/rules/lab.rules.yaml b/stacks/lab/prometheus/rules/lab.rules.yaml new file mode 100644 index 0000000..bf69b3b --- /dev/null +++ b/stacks/lab/prometheus/rules/lab.rules.yaml @@ -0,0 +1,94 @@ +--- +# The lab stack watching itself. +# +# Four rules, and the number is the point. The estate has 48; ADR-0007 names +# giving up "Loki, the dashboards and 40 alert rules" as the price of keeping +# lab telemetry out of the estate, and this file does not pretend to replace +# them. What it covers is this stack's own working — the four things that would +# make every other signal here silently wrong. +# +# NOTHING HERE PAGES. There is no Alertmanager in this stack (ADR-0020), so a +# firing alert is visible in Prometheus's /alerts and in Grafana and nowhere +# else. That is a deliberate limit, not an oversight, and it is why these are +# scoped to "the collector is broken" rather than to anything about the estate +# being defended: an alert nobody is notified of is only useful to someone who +# is already looking, and someone already looking is looking at the lab. +# +# The names deliberately match the estate's rules for the same conditions. +# `external_labels: monitor: imaginationlan` in prometheus.yaml is what +# distinguishes them, and it travels with every series; two different names for +# one condition would not survive a screenshot pasted into an issue. +# +# What this does NOT cover, and cannot: this Prometheus going down. `up == 0` +# is evaluated by the thing that would have stopped. The estate answers that +# for itself with a watcher on `oracle` (ADR-0015) and a heartbeat receiver; +# the lab has neither available to it, which is #257. +groups: + - name: lab + interval: 60s + rules: + # Covers every target in prometheus.yaml — itself, loki, grafana, alloy. + # `up` is the only one of these four that sees a component which has + # stopped publishing altogether; the other three read a metric the + # component emits about its own working, which requires it to be working + # enough to emit. + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + component: lab + severity: critical + category: availability + annotations: + summary: "{{ $labels.job }} target {{ $labels.instance }} is down in the lab stack" + description: >- + The lab's Prometheus has failed to scrape {{ $labels.instance }} + ({{ $labels.job }}) for 5 minutes. Nothing outside the lab can see + this — see #257. + + - alert: PrometheusConfigReloadFailed + expr: prometheus_config_last_reload_successful == 0 + for: 5m + labels: + component: lab + severity: critical + category: correctness + annotations: + summary: "The lab's Prometheus rejected its own config reload" + description: >- + It is still serving the config it last parsed successfully, so the + change you deployed is not the one running. + + # The rule this stack most needs, because its retention figures are a + # bound rather than a measurement — compose.yaml says so at length. If + # 4 GiB turns out to be too small, retention quietly becomes shorter than + # the 15 days configured, the store keeps working and the dashboards keep + # drawing. This counter is the only thing that distinguishes "the ceiling + # engaged" from "old data aged out normally": do not substitute + # prometheus_tsdb_time_retentions_total, which increments by design. + - alert: PrometheusSizeRetentionActive + expr: increase(prometheus_tsdb_size_retentions_total[1h]) > 0 + for: 5m + labels: + component: lab + severity: warning + category: correctness + annotations: + summary: "The lab's size ceiling is evicting blocks — retention is shorter than 15 days" + description: >- + Re-derive the retention figures against the spindles rather than + raising the ceiling blind; compose.yaml carries the queries. + + - alert: LokiIngestionStalled + expr: | + sum(rate(loki_distributor_lines_received_total[30m])) == 0 + for: 30m + labels: + component: lab + severity: warning + category: availability + annotations: + summary: "The lab's Loki has received no log lines for 30 minutes" + description: >- + Either the lab's Alloy is down or the push endpoint is unreachable. + Logs are being lost, not queued indefinitely. diff --git a/stacks/lab/prometheus/tests/lab.test.yaml b/stacks/lab/prometheus/tests/lab.test.yaml new file mode 100644 index 0000000..3d2eb85 --- /dev/null +++ b/stacks/lab/prometheus/tests/lab.test.yaml @@ -0,0 +1,138 @@ +--- +# Unit tests for lab.rules.yaml. +# +# The pairing is the point, and it is the argument #63 produced: a case that +# only ever expects silence would pass against a rule that can never fire. +# `ContainerHighMemory` divided by a limit nothing set, guarded on it being +# non-zero, and was therefore unfireable for any input — while `promtool check +# rules` passed it every time, because that parses PromQL and never asks +# whether an expression can be true. So each rule below gets a firing case AND +# a quiet case: the first proves the expression produces an alert for a real +# input, the second proves it does not produce one for the input production +# actually supplies. +# +# All four rules in lab.rules.yaml are covered here. That is a deliberate +# difference from the estate's tests, where four of the nine stack rules are +# syntax-checked only: this file is small enough that "test the ones that were +# hard" has no advantage over testing all of them, and a new stack has no +# operational history to lean on when deciding which ones were hard. +rule_files: + - ../rules/lab.rules.yaml + +evaluation_interval: 1m + +tests: + # --- InstanceDown -------------------------------------------------------- + - interval: 1m + input_series: + - series: 'up{instance="loki:3100",job="loki"}' + values: "1+0x5 0+0x20" + alert_rule_test: + - eval_time: 20m + alertname: InstanceDown + exp_alerts: + - exp_labels: + alertname: InstanceDown + instance: loki:3100 + job: loki + component: lab + severity: critical + category: availability + exp_annotations: + summary: "loki target loki:3100 is down in the lab stack" + description: >- + The lab's Prometheus has failed to scrape loki:3100 (loki) for + 5 minutes. Nothing outside the lab can see this — see #257. + # Quiet while the target is up, including inside the `for:` window. + - eval_time: 4m + alertname: InstanceDown + exp_alerts: [] + + # --- PrometheusConfigReloadFailed ---------------------------------------- + - interval: 1m + input_series: + - series: 'prometheus_config_last_reload_successful{instance="localhost:9090",job="prometheus"}' + values: "0+0x20" + alert_rule_test: + - eval_time: 10m + alertname: PrometheusConfigReloadFailed + exp_alerts: + - exp_labels: + alertname: PrometheusConfigReloadFailed + instance: localhost:9090 + job: prometheus + component: lab + severity: critical + category: correctness + exp_annotations: + summary: "The lab's Prometheus rejected its own config reload" + description: >- + It is still serving the config it last parsed successfully, so + the change you deployed is not the one running. + + - interval: 1m + input_series: + - series: 'prometheus_config_last_reload_successful{instance="localhost:9090",job="prometheus"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 10m + alertname: PrometheusConfigReloadFailed + exp_alerts: [] + + # --- PrometheusSizeRetentionActive --------------------------------------- + # The counter is flat until the size ceiling starts evicting, then steps. + - interval: 1m + input_series: + - series: 'prometheus_tsdb_size_retentions_total{instance="localhost:9090",job="prometheus"}' + values: "0+0x40 1+0x30" + alert_rule_test: + - eval_time: 60m + alertname: PrometheusSizeRetentionActive + exp_alerts: + - exp_labels: + alertname: PrometheusSizeRetentionActive + instance: localhost:9090 + job: prometheus + component: lab + severity: warning + category: correctness + exp_annotations: + summary: "The lab's size ceiling is evicting blocks — retention is shorter than 15 days" + description: >- + Re-derive the retention figures against the spindles rather + than raising the ceiling blind; compose.yaml carries the + queries. + # A store that has never evicted is the normal case and must stay quiet. + - eval_time: 30m + alertname: PrometheusSizeRetentionActive + exp_alerts: [] + + # --- LokiIngestionStalled ------------------------------------------------ + # A flat counter is a zero rate: lines are arriving nowhere. + - interval: 1m + input_series: + - series: 'loki_distributor_lines_received_total{instance="loki:3100",job="loki"}' + values: "1000+0x90" + alert_rule_test: + - eval_time: 70m + alertname: LokiIngestionStalled + exp_alerts: + - exp_labels: + alertname: LokiIngestionStalled + component: lab + severity: warning + category: availability + exp_annotations: + summary: "The lab's Loki has received no log lines for 30 minutes" + description: >- + Either the lab's Alloy is down or the push endpoint is + unreachable. Logs are being lost, not queued indefinitely. + + - interval: 1m + input_series: + - series: 'loki_distributor_lines_received_total{instance="loki:3100",job="loki"}' + values: "1000+120x90" + alert_rule_test: + - eval_time: 70m + alertname: LokiIngestionStalled + exp_alerts: []