From ba38606446376d24c3e2427fa92f0018674686f3 Mon Sep 17 00:00:00 2001 From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:10:30 +0000 Subject: [PATCH 1/2] refactor(scripts): make the deploy and secrets tooling stack-aware (#264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `STACK ?=` reached the lifecycle targets and stopped. Underneath, four scripts assumed the estate's stack was the only one, and each would have failed — or worse, quietly done the wrong thing — the first time a second one existed. render-config.sh demanded all ten of the estate's keys from any stack. Every render was already guarded by `[[ -f ... ]]`, so the script coped with a stack that has no snmp-exporter; the required-key list did not, and `make render STACK=lab` died naming four SNMP communities and four Alertmanager URLs the lab has no use for. The list is now derived: the Grafana half from the stack's own compose.yaml, because a ${VAR:?} guard IS the declaration that a service cannot start without it, and a second copy could only drift from it — the same derivation seed-validation-env.sh already runs against the same guards. The SNMP and Alertmanager halves stay conditional on their input files existing. For stacks/observability this produces the identical ten names, verified against main's array rather than reasoned about. reload-config.sh reloaded a SERVICES array that is the union across stacks, so it died on the first service a stack does not have — through the "is not running, so there is nothing to reload" path, which is the right message for a stopped service and the wrong one for an absent one. It now filters on what the compose file DECLARES, from `compose config --services`, so those two cases stay separate: a declared service that is down is still a failed deploy. bootstrap.sh filled in "the first placeholder found" in .sops.yaml. With a rule per stack that writes the lab guest's key into the estate's rule, or the reverse. It now looks for this stack's placeholder, and refuses outright when the key it would write is already a recipient of another rule — which is exactly what `make secrets-init STACK=lab` typed on the monitoring host would do, and it would have looked like a successful bootstrap. backup-firewall.sh read the age recipient as the first age1 key in .sops.yaml, which was the right key while there was only one rule. It now reads the key from the rule that actually covers backups/firewall/, because the moment the lab rule's placeholder becomes real, `head -1` would encrypt every firewall export to the lab guest — silently, and sops would do it happily. .sops.yaml gets that lab rule, above the catch-all, because the catch-all matches all of secrets/: a recipient added there can decrypt the estate's SNMP communities and Grafana admin password too. ADR-0020. Co-Authored-By: Claude Opus 5 --- .sops.yaml | 25 ++++++++++++ scripts/backup-firewall.sh | 20 +++++++++- scripts/bootstrap.sh | 38 ++++++++++++++++-- scripts/reload-config.sh | 31 ++++++++++++++- scripts/render-config.sh | 80 +++++++++++++++++++++++++++++++------- 5 files changed, 174 insertions(+), 20 deletions(-) diff --git a/.sops.yaml b/.sops.yaml index 9dee59a..70f5198 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -24,6 +24,31 @@ # public half into this file. creation_rules: + # The lab stack, FIRST — SOPS takes the first rule whose path_regex matches, + # so this has to sit above the catch-all below or it would never apply. + # + # It exists because the rule below matches all of secrets/, which means any + # recipient added to it can decrypt every file there. The lab guest needs to + # decrypt secrets/lab.sops.yaml on `make render STACK=lab`; adding its key to + # the general rule to achieve that would also hand it the estate's SNMP + # communities and Grafana admin password — a lab host holding the credentials + # of the estate it is supposed to be isolated from, which inverts the trust + # direction ADR-0007 exists to protect. So it gets a rule, and a key, of its + # own. See docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md. + # + # This is NOT the "a second rule would be a second copy of the key" case the + # note above warns about: that argument is against two rules sharing one + # recipient. These two hold different recipients on purpose, which is the + # whole point. + # + # Placeholder until the lab guest exists. `make secrets-init STACK=lab`, run + # ON that guest, fills it in — and refuses if the key it would write is + # already a recipient below, because that would collapse the two rules back + # into one. + - path_regex: secrets/lab\..*\.sops\.ya?ml$ + age: >- + REPLACE_WITH_LAB_AGE_PUBLIC_KEY + - path_regex: (secrets/.*|backups/firewall/.*)\.sops\.ya?ml$ age: >- age1yrdu996u5mhdh0qf93l7s8zz8stneqnqxpncrcarrmgxvsy264rqmkcs6x diff --git a/scripts/backup-firewall.sh b/scripts/backup-firewall.sh index 09f51c7..8f44b0f 100755 --- a/scripts/backup-firewall.sh +++ b/scripts/backup-firewall.sh @@ -137,8 +137,26 @@ need() { command -v "$1" >/dev/null 2>&1 || { red "missing dependency: $1"; exit # The age recipient is read from .sops.yaml rather than duplicated here. One # source of truth for the key; rotating it in .sops.yaml rotates it here too. +# +# Read from the rule that actually covers backups/firewall/, not the first key +# in the file. This was `grep ... | head -1` while .sops.yaml held exactly one +# creation_rule, which made "first key" and "the right key" the same string. +# ADR-0020 added a second rule above it for the lab stack, and the moment that +# rule's placeholder is replaced with a real key, `head -1` would encrypt every +# firewall backup to the LAB guest — silently, since sops would happily do it +# and the file would still look like a backup. That is the same trust inversion +# the second rule was created to prevent, arriving through the back door. +# +# Anchored on the path_regex naming backups/firewall, so it follows the rule +# rather than the ordering. Exits at the first key after that line: `age:` uses +# a folded scalar, so the key is on the line following the one that matches. recipient() { - grep -oE 'age1[0-9a-z]{50,}' "$SOPS_POLICY" | head -1 + awk ' + /path_regex:.*backups\/firewall/ { inrule = 1 } + inrule && match($0, /age1[0-9a-z]{50,}/) { + print substr($0, RSTART, RLENGTH); exit + } + ' "$SOPS_POLICY" } # Newest first, sorted by NAME and not by mtime. The stamp is UTC ISO-8601 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index a91c812..cad2d96 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -51,9 +51,40 @@ info "public key: ${PUBLIC_KEY}" # --------------------------------------------------------------------------- # 2. register it in .sops.yaml # --------------------------------------------------------------------------- -if grep -q "REPLACE_WITH_YOUR_AGE_PUBLIC_KEY" "${SOPS_CONFIG}"; then - info "writing public key into .sops.yaml" - sed -i.bak "s|REPLACE_WITH_YOUR_AGE_PUBLIC_KEY|${PUBLIC_KEY}|" "${SOPS_CONFIG}" +# Which placeholder belongs to THIS stack. +# +# There is no longer one placeholder to fill. .sops.yaml carries a creation_rule +# per stack that needs its own recipient — ADR-0020 gives `lab` one, because the +# single rule that used to match all of secrets/ meant any recipient added to it +# could decrypt every other stack's credentials too. Filling in "the first +# placeholder found" would write the lab guest's key into the estate's rule, or +# the estate's into the lab's, which is the failure that decision exists to +# prevent. +# +# The generic name is kept as the fallback so a fresh clone bootstrapping +# `observability` behaves exactly as it always has. +PLACEHOLDER="REPLACE_WITH_${STACK^^}_AGE_PUBLIC_KEY" +grep -q "${PLACEHOLDER}" "${SOPS_CONFIG}" 2>/dev/null \ + || PLACEHOLDER="REPLACE_WITH_YOUR_AGE_PUBLIC_KEY" + +if grep -q "${PLACEHOLDER}" "${SOPS_CONFIG}"; then + # Refused rather than warned about. Reaching here means this host's key is + # already a recipient of some other rule in this file, and is now being asked + # to become ${STACK}'s as well — one key that decrypts both stacks, which is + # the exact collapse the separate rules exist to stop. It is also the easy + # mistake: `make secrets-init STACK=lab` typed on the monitoring host rather + # than on the lab guest does precisely this, and the result would look like a + # successful bootstrap. + if grep -q "${PUBLIC_KEY}" "${SOPS_CONFIG}"; then + die "this host's key is already a recipient in ${SOPS_CONFIG##*/}, and +making it ${STACK}'s recipient as well would give one key both stacks. + +Run this on the host that will run ${STACK}, so that stack gets a key of its +own. If one key for both is genuinely what you want, edit .sops.yaml by hand — +it should be a decision, not a side effect of where you happened to type this." + fi + info "writing public key into .sops.yaml (${PLACEHOLDER})" + sed -i.bak "s|${PLACEHOLDER}|${PUBLIC_KEY}|" "${SOPS_CONFIG}" rm -f "${SOPS_CONFIG}.bak" elif grep -q "${PUBLIC_KEY}" "${SOPS_CONFIG}"; then info ".sops.yaml already lists this key" @@ -61,6 +92,7 @@ else warn ".sops.yaml lists a different age recipient." warn "Add this key as an additional recipient by hand, then run:" warn " sops updatekeys ${SECRETS_FILE}" + warn "Add it to the rule matching secrets/${STACK}. — NOT to another stack's." fi # --------------------------------------------------------------------------- diff --git a/scripts/reload-config.sh b/scripts/reload-config.sh index cf986b0..e6391cf 100755 --- a/scripts/reload-config.sh +++ b/scripts/reload-config.sh @@ -328,8 +328,37 @@ restart_alloy() { ok "alloy (restarted)" } +# SERVICES above is the union across every stack, not the contents of this one. +# `stacks/lab` runs Prometheus, Loki, Grafana and Alloy and no Alertmanager, +# snmp-exporter or blackbox-exporter (ADR-0020), so reloading the array as +# written died on the first service that stack does not have — and it died +# through reload_one's "is not running, so there is nothing to reload", which is +# the correct message for a stopped service and a wrong one for an absent one. +# +# The two cases must stay separate, which is why this filters on what the +# compose file DEFINES rather than on what is running. A service this stack +# declares and is not running is still a failed deploy and still dies below; +# only one it never declared is skipped. +# +# `compose config --services` rather than grepping for two-space-indented keys: +# the answer is compose's own, so a service list this script disagrees with is +# not a shape it can be tricked by. It needs the ${VAR:?} guards satisfied, +# which is true wherever this runs — `make up` renders .env immediately before +# calling this, and `make reload` acts on a stack that is already up. If it +# cannot answer at all, fall back to the full array: that is exactly today's +# behaviour, so the failure mode is the one that has always been there rather +# than a new one. +declared="" +if ! declared="$("${COMPOSE[@]}" config --services 2>/dev/null)"; then + declared="" +fi + for entry in "${SERVICES[@]}"; do - reload_one "${entry%%:*}" "${entry##*:}" + svc="${entry%%:*}" + if [[ -n "${declared}" ]] && ! grep -qx "${svc}" <<<"${declared}"; then + continue + fi + reload_one "${svc}" "${entry##*:}" done restart_alloy diff --git a/scripts/render-config.sh b/scripts/render-config.sh index 6d00ad4..ae92d25 100755 --- a/scripts/render-config.sh +++ b/scripts/render-config.sh @@ -113,18 +113,69 @@ unset absent clobbered cert info "decrypting $(basename "${SECRETS_FILE}")" load_secrets "${STACK}" -REQUIRED=( - GRAFANA_ADMIN_PASSWORD - GRAFANA_RENDERER_TOKEN - ALERTMANAGER_WEBHOOK_URL - ALERTMANAGER_URGENT_WEBHOOK_URL - ALERTMANAGER_SECURITY_WEBHOOK_URL - ALERTMANAGER_HEARTBEAT_URL - SNMP_COMMUNITY_PFSENSE - SNMP_COMMUNITY_APC - SNMP_COMMUNITY_MOKERLINK - SNMP_COMMUNITY_ILO -) +# The two inputs whose presence decides what this stack needs. Declared here +# rather than beside the blocks that render them, because the required-key list +# below is built from whether they exist. +SNMP_SRC="${STACK_DIR}/snmp-exporter/snmp.yaml" +AM_CONFIG="${STACK_DIR}/alertmanager/alertmanager.yaml" + +# --------------------------------------------------------------------------- +# What this stack requires +# +# Derived from the stack, not listed for one of them. Every render below is +# already guarded by `[[ -f ... ]]`, so the script has always coped with a +# stack that has no snmp-exporter — but this list was a flat array of all ten +# keys, which demanded four SNMP communities and four Alertmanager URLs from +# any stack at all. `make render STACK=lab` died on a lab that runs neither, +# naming secrets that stack has no use for. +# +# The Grafana half is read out of the stack's own compose.yaml: a ${VAR:?} +# guard IS the declaration that the service cannot start without it, so a +# second copy here could only ever drift from it. Same derivation +# seed-validation-env.sh uses against the same guards, for the same reason. +# The three excluded names are the ones THIS script writes into .env further +# down — they are guarded in compose.yaml but they are not secrets, and asking +# SOPS for them would fail every render. +# +# For `stacks/observability` this produces the identical ten names the flat +# array held, which is the property that makes the change safe: the two Grafana +# keys come from its compose guards, and both conditionals below are true. +# --------------------------------------------------------------------------- +REQUIRED=() +while read -r var; do + [[ -n "${var}" ]] || continue + case "${var}" in RENDER_UID | RENDER_GID | LOG_READ_GID) continue ;; esac + REQUIRED+=("${var}") +done < <(grep -oE '\$\{[A-Za-z_][A-Za-z0-9_]*:\?' "${STACK_DIR}/compose.yaml" \ + | sed 's/^\${//; s/:?$//' | sort -u) + +# The SNMP community names stay written out rather than derived from the +# placeholders in snmp.yaml, and that is deliberate. scripts/snmp-targets.sh +# --check asserts this array against the device inventory by grepping for each +# name on a line of its own — it is one of the five copies of the device list +# that check exists to hold together, and deriving it here would remove the +# copy rather than the drift, leaving --check asserting nothing. Keep the +# one-name-per-line shape. +if [[ -f "${SNMP_SRC}" ]]; then + REQUIRED+=( + SNMP_COMMUNITY_PFSENSE + SNMP_COMMUNITY_APC + SNMP_COMMUNITY_MOKERLINK + SNMP_COMMUNITY_ILO + ) +fi + +# One entry per AM_CHANNELS entry below; the pairing is asserted after they are +# rendered, so a channel added there without a key here fails at deploy time. +if [[ -f "${AM_CONFIG}" ]]; then + REQUIRED+=( + ALERTMANAGER_WEBHOOK_URL + ALERTMANAGER_URGENT_WEBHOOK_URL + ALERTMANAGER_SECURITY_WEBHOOK_URL + ALERTMANAGER_HEARTBEAT_URL + ) +fi + missing=() for var in "${REQUIRED[@]}"; do [[ -n "${!var:-}" ]] || missing+=("${var}") @@ -134,7 +185,6 @@ done # --------------------------------------------------------------------------- # Render snmp.yaml # --------------------------------------------------------------------------- -SNMP_SRC="${STACK_DIR}/snmp-exporter/snmp.yaml" SNMP_OUT_DIR="${STACK_DIR}/snmp-exporter/.rendered" if [[ -f "${SNMP_SRC}" ]]; then info "rendering snmp.yaml" @@ -202,7 +252,7 @@ AM_CHANNELS=( "ALERTMANAGER_HEARTBEAT_URL:heartbeat_url" ) AM_OUT_DIR="${STACK_DIR}/alertmanager/.rendered" -if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then +if [[ -f "${AM_CONFIG}" ]]; then info "rendering ${#AM_CHANNELS[@]} alertmanager receiver URL(s)" mkdir -p "${AM_OUT_DIR}" chmod 700 "${AM_OUT_DIR}" @@ -229,7 +279,7 @@ if [[ -f "${STACK_DIR}/alertmanager/alertmanager.yaml" ]]; then # `secrets/[a-z_]+` also matched `secrets/observability.sops.yaml` in this # file's own header comment, and reported the header as a missing channel. done < <(grep -oE 'url_file:[[:space:]]*/etc/alertmanager/secrets/[a-z_]+' \ - "${STACK_DIR}/alertmanager/alertmanager.yaml" \ + "${AM_CONFIG}" \ | sed 's|.*/||' | sort -u) fi From 7101a810d73dddf53e25fd09432ae4cab3b295a9 Mon Sep 17 00:00:00 2001 From: Garrett Allen <98648590+Gerrrt@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:12:49 +0000 Subject: [PATCH 2/2] =?UTF-8?q?feat(lab):=20build=20stacks/lab=20=E2=80=94?= =?UTF-8?q?=20Prometheus,=20Loki,=20Grafana,=20Alloy=20(#264)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR-0020 settled the shape; this builds it. Four services, on a guest (`alexander`, 10.0.30.40) rather than on `Saruman` itself, because a compose stack is Docker and Docker rewrites the iptables of the box whose own firewall ADR-0014 relies on. Prometheus is here because ADR-0007 named three services while also saying config.alloy is reused with only the two *_URL variables changed, and that file has two sinks. Without it the second points at 10.0.99.20, which the Decision forbids. The agent config is MOUNTED from stacks/observability/alloy/, not copied — two files, not the directory, so syslog.alloy stays on the monitoring host where it belongs. deploy-agent.sh exists because `oracle` drifted four ways from a hand-copied agent, and its header states the rule: the fix is to not copy. Absent on purpose: no Alertmanager (nothing in the lab pages, and that is not an answer to #257), no snmp-exporter (the estate already polls `shiva`; two stacks polling one device is two answers to when it last responded), no blackbox-exporter, no renderer, no dashboards. The dashboards README says why a copy of the estate's seven would be four rows of empty panels, and an empty panel is indistinguishable from a broken collector. Prometheus and Loki are exposed, not published: ADR-0012 publishes a port only when something off-host uses it, and today only Alloy talks to them. compose.yaml marks the lines to uncomment when #265 gives them real clients. Retention is 15d/4GB and is a bound rather than a measurement — the stack has never run, so there is nothing to derive from. compose.yaml carries the queries to re-derive it, and PrometheusSizeRetentionActive is what says the ceiling started binding. Four rules, all four unit-tested firing and quiet, because a case that only expects silence passes against a rule that cannot fire (#63). check_docs.py grew the case it was never designed for: a stack committed before its host is racked. The marker inverts the check rather than switching it off — a normal row's host must appear in network.md, a row marked "not built yet" must be absent from it — so racking the host and adding its row fails, saying the marker is stale. It also drops such a row from the Alloy agent count, which makes removing the marker fail hardware.md's "three Alloy agents" on the same commit, which is when that sentence should have to change. All three new branches were verified to fire. Not covered, and written down rather than discovered: no validator runs against this stack (#263), and converge.sh runs a bare `make up`, so nothing deploys it either — it is applied by hand, on that guest. Co-Authored-By: Claude Opus 5 --- .github/dependabot.yml | 22 ++ README.md | 3 + docs/architecture.md | 3 +- docs/network.md | 16 +- docs/observability.md | 23 ++ docs/roadmap.md | 19 +- scripts/check_docs.py | 73 ++++- secrets/lab.example.yaml | 43 +++ stacks/lab/.env.example | 47 +++ stacks/lab/README.md | 122 ++++++++ stacks/lab/compose.yaml | 286 ++++++++++++++++++ stacks/lab/grafana/dashboards/README.md | 47 +++ .../provisioning/dashboards/dashboards.yaml | 35 +++ .../provisioning/datasources/datasources.yaml | 41 +++ stacks/lab/loki/loki-config.yaml | 72 +++++ stacks/lab/prometheus/prometheus.yaml | 68 +++++ stacks/lab/prometheus/rules/lab.rules.yaml | 94 ++++++ stacks/lab/prometheus/tests/lab.test.yaml | 138 +++++++++ 18 files changed, 1142 insertions(+), 10 deletions(-) create mode 100644 secrets/lab.example.yaml create mode 100644 stacks/lab/.env.example create mode 100644 stacks/lab/README.md create mode 100644 stacks/lab/compose.yaml create mode 100644 stacks/lab/grafana/dashboards/README.md create mode 100644 stacks/lab/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 stacks/lab/grafana/provisioning/datasources/datasources.yaml create mode 100644 stacks/lab/loki/loki-config.yaml create mode 100644 stacks/lab/prometheus/prometheus.yaml create mode 100644 stacks/lab/prometheus/rules/lab.rules.yaml create mode 100644 stacks/lab/prometheus/tests/lab.test.yaml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 174843d..409bc57 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,6 +19,28 @@ updates: prometheus-stack: patterns: ["prom/*"] + # The lab stack. A second entry rather than a second directory on the one + # above, because Dependabot takes a single directory per entry — and without + # it the lab's pins would rot exactly the way the comment above says pinned + # tags do, while the estate's stayed current. The two run the same four + # images, so they are expected to move together; they get separate PRs + # because they are separate deploys to separate hosts, and the lab's is a + # host the operator has to be sitting in front of. + - package-ecosystem: docker-compose + directory: /stacks/lab + schedule: + interval: weekly + day: sunday + open-pull-requests-limit: 5 + commit-message: + prefix: "chore(deps)" + labels: ["dependencies", "lab"] + groups: + grafana-stack: + patterns: ["grafana/*"] + prometheus-stack: + patterns: ["prom/*"] + - package-ecosystem: github-actions directory: / schedule: diff --git a/README.md b/README.md index f62584e..a1e90d2 100644 --- a/README.md +++ b/README.md @@ -169,6 +169,9 @@ Full topology and data flow in [`docs/architecture.md`](docs/architecture.md). │ ├── alloy/ # the agent config directory, shipped to every host │ ├── snmp-exporter/ # generator.yaml is the source of truth │ └── grafana/ # provisioning + 7 dashboards +├── stacks/lab/ # the lab's own stack — four services, not yet deployed +│ # runs on a guest on Saruman, never remote-writes +│ # to VLAN 99. See its README and ADR-0020 ├── secrets/ # SOPS-encrypted; see secrets/README.md ├── scripts/ # bootstrap, render, validate, pin-digests, purge ├── SECURITY.md # disclosure policy and known exposure diff --git a/docs/architecture.md b/docs/architecture.md index 89f1bc7..31bdbaf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -181,7 +181,8 @@ hole from the monitoring VLAN into the monitored one. | Host | VLAN | Stack | Contents | | --- | --- | --- | --- | | `prometheus` (10.0.99.20) | 🔴 99 | [`stacks/observability`](../stacks/observability) | Prometheus, Alertmanager, Loki, Grafana, snmp-exporter, blackbox-exporter, Alloy | -| `Saruman` (10.0.30.110) | 🟢 30 | *(none yet)* | Proxmox VE 9, no guests — see [roadmap](roadmap.md); Alloy agent (native package) | +| `Saruman` (10.0.30.110) | 🟢 30 | *(none — and none intended)* | Proxmox VE 9, no guests yet — see [roadmap](roadmap.md); Alloy agent (native package). It runs no compose stack by decision, not by omission: Docker would rewrite the iptables its own firewall relies on ([ADR-0014](adr/0014-put-ifrit-on-imaginationlan-and-give-the-targets-no-route.md)), which is why the agent here is the native package and why `stacks/lab` runs in a guest | +| `alexander` (10.0.30.40) | 🟢 30 | [`stacks/lab`](../stacks/lab) | **Not built yet** — the guest is [#262](https://github.com/Gerrrt/HomeLab/issues/262), the stack is committed and deployable. Prometheus, Loki, Grafana, Alloy: the lab's own observability, which never remote-writes to VLAN 99 ([ADR-0007](adr/0007-defensive-estate-and-offensive-range.md), [ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md)) | | `oracle` (10.0.99.30) | 🔴 99 | *(none — hand-run containers)* | The Lemmiwinks wiki and its Postgres, since 2025-11-12 ([ADR-0011](adr/0011-keep-the-wiki-internal.md)); Alloy agent (Docker, `scripts/deploy-agent.sh`); the off-host copy of the firewall export (`make backup-firewall`). The estate's host for small off-host jobs — [ADR-0015](adr/0015-give-oracle-the-off-host-jobs.md) | One directory per stack, not one per service. A stack is the unit that gets diff --git a/docs/network.md b/docs/network.md index cf18be9..ae9eb97 100644 --- a/docs/network.md +++ b/docs/network.md @@ -278,7 +278,21 @@ Where things get broken on purpose. dedicated port, `Saruman` is the Proxmox install. They are separate addresses and separate names, and conflating them is a mistake this document previously made. -- `Saruman` currently runs no guests. +- `Saruman` currently runs no guests. The first will be `alexander`, at + `10.0.30.40` — a static below `.100` with a reservation, single-homed on this + segment like its host — which runs + [`stacks/lab`](../stacks/lab): the lab's own Prometheus, Loki, Grafana and + Alloy. The stack is built and committed; the guest is + [#262](https://github.com/Gerrrt/HomeLab/issues/262), which is why it is + described here and not in the table above. **It is a guest and not the + hypervisor for a reason**: a compose stack is Docker, and Docker would + rewrite the iptables of the box whose own firewall ADR-0014 relies on — the + same fact that put the native `.deb` agent on `Saruman` rather than a + container + ([ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md)). + It gets **no** pass into Winterfell: the rule below is the hypervisor's, and + ADR-0007's "guests get no such rule" covers this one too. Nothing in that + stack remote-writes off the segment. - `Saruman` runs an Alloy agent and is the one host on this segment with a path into Winterfell: a single pass, `10.0.30.110 → 10.0.99.20` on 9090 and 3100 TCP, unlogged and above the ADR-0014 tripwire. The hypervisor's own telemetry diff --git a/docs/observability.md b/docs/observability.md index 4275d80..93c910d 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -2,6 +2,29 @@ What is collected, where it goes, and how to change it. +**This document describes the estate's stack, on `prometheus` (10.0.99.20).** +There is a second one. [`stacks/lab`](../stacks/lab) is the lab's own +Prometheus, Loki, Grafana and Alloy, and it is deliberately not part of any of +what follows: no series it holds reaches this Prometheus, no log line reaches +this Loki, and none of the alert rules or dashboards below can see it. That is +ADR-0007's decision — lab telemetry stays in the lab, so that deliberately +hostile data never lands in the store the estate is actually run from — and +[ADR-0020](adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md) +settles its shape. It is built but not yet deployed; the guest that runs it is +[#262](https://github.com/Gerrrt/HomeLab/issues/262). + +The one path that does cross belongs to the hypervisor and not to any guest: +`Saruman`'s own agent remote-writes here over a single unlogged pass +([#88](https://github.com/Gerrrt/HomeLab/issues/88)). A DL360 with an ageing +mirrored pair is estate hardware, and its health belongs with the rest of the +estate's. + +The consequence worth carrying into everything below: **nothing here can tell a +quiet lab from a dead one.** `RemoteWriteJobStale` keys on jobs that arrive on +this Prometheus, so by construction it can never cover a stack that never +arrives. That gap is [#257](https://github.com/Gerrrt/HomeLab/issues/257), and +it is not closed by anything in this document. + ## What is collected | Source | Via | Interval | Examples | diff --git a/docs/roadmap.md b/docs/roadmap.md index 4ace329..93c03ad 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -298,12 +298,19 @@ what left this one unfireable for months. `*_URL` variables changed, and that file has two sinks, so a lab without a Prometheus points the second one at `10.0.99.20` and inverts the isolation the ADR exists for. - In order — [#262](https://github.com/Gerrrt/HomeLab/issues/262) the guest, - then [#263](https://github.com/Gerrrt/HomeLab/issues/263) the validators, - because `STACK ?=` reaches the lifecycle targets and stops there and every - checker in this repository is pinned to `stacks/observability`, so a second - stack today would be one CI has never seen; then - [#264](https://github.com/Gerrrt/HomeLab/issues/264) the stack itself. + [#264](https://github.com/Gerrrt/HomeLab/issues/264) is built: + `stacks/lab/` holds the compose file, both configs, four alert rules and + their unit tests, and the secrets template. What is left of it is a deploy, + which needs [#262](https://github.com/Gerrrt/HomeLab/issues/262) — the guest + on `Saruman` — to exist first. Building it made the tooling stack-aware + (`render-config.sh` derives its required keys per stack rather than demanding + the estate's ten, `reload-config.sh` skips services a stack does not declare, + `bootstrap.sh` refuses to give one age key both stacks) and gave `.sops.yaml` + the lab rule ADR-0020 asked for. It did **not** touch + [#263](https://github.com/Gerrrt/HomeLab/issues/263): `STACK ?=` reaches the + lifecycle targets and stops there, every checker is still pinned to + `stacks/observability`, and so the new stack is one CI has never seen — + validated only by hand and by the checks that already follow `STACK`. [#265](https://github.com/Gerrrt/HomeLab/issues/265) the domain is what everything else is pointed at, and blocks both [#266](https://github.com/Gerrrt/HomeLab/issues/266) Wazuh — the heaviest diff --git a/scripts/check_docs.py b/scripts/check_docs.py index 62aafe4..ae18080 100755 --- a/scripts/check_docs.py +++ b/scripts/check_docs.py @@ -178,6 +178,16 @@ def strip_md(cell: str) -> str: return cell.strip() +# A host-and-stack row for something that does not exist yet. See the block +# above check_host_stack_table() for what it does there; count_alloy_agents() +# below reads it too, because a row describing an undeployed host describes an +# undeployed agent. +# +# Matched against the RAW cell, never strip_md()'s output: that helper removes +# every `*`, which takes the emphasis with it and leaves the marker unfindable. +NOT_BUILT = re.compile(r"\*\*not built yet\*\*", re.I) + + # --------------------------------------------------------------------------- # Facts, computed from the configs # --------------------------------------------------------------------------- @@ -253,6 +263,13 @@ def count_alloy_agents() -> int: is the machine-readable side. A row whose Contents cell names Alloy is an agent; "two Alloy agents" in hardware.md was unguarded and stale for as long as it took to deploy a third (#88). + + A row marked NOT_BUILT is not counted, because an agent on a host that does + not exist is not an agent. `stacks/lab` declares one for `alexander`, and it + collects nothing until that guest is racked (#262). The exclusion is not a + convenience: dropping the marker on the commit that builds the host pushes + this count to four and fails hardware.md's "three Alloy agents" in the same + run, which is exactly when that sentence should be forced to change. """ tables = tables_under( ARCH_MD.read_text(encoding="utf-8"), @@ -262,7 +279,9 @@ def count_alloy_agents() -> int: return 0 return sum( 1 for row in tables[0][1:] - if len(row) > 3 and "alloy" in strip_md(row[3]).lower() + if len(row) > 3 + and "alloy" in strip_md(row[3]).lower() + and not NOT_BUILT.search(row[3]) ) @@ -407,6 +426,27 @@ def check_snmp_targets() -> list[str]: # --------------------------------------------------------------------------- # 3. Host and stack mapping # --------------------------------------------------------------------------- +# A stack directory can legitimately exist before the host that runs it does. +# `stacks/lab` was committed complete — compose file, configs, rules, unit +# tests — while the guest that will run it, `alexander`, was still an issue +# (#262, #264). ADR-0004 puts the host-to-stack mapping in this document, so the +# row has to exist; but docs/network.md is the inventory of what is actually on +# the wire, and writing an unbuilt guest into it would be the precise kind of +# false claim this file exists to catch. It would also mean inventing a MAC, a +# device and an OS for a machine whose distribution is explicitly undecided. +# +# So the row is marked, and the marker INVERTS the check rather than switching +# it off. A normal row's host must APPEAR in network.md; a row marked "not built +# yet" must be ABSENT from it. That is what makes the marker self-clearing — +# rack the host, add its network.md row, and this fails saying the marker is +# stale, instead of quietly tolerating a row that claims both things at once. +# The address is still required and still checked for collisions, so a plan is +# held to the same standard as a deployment. +# +# The marker itself is defined next to strip_md(), because count_alloy_agents() +# reads it too. + + def check_host_stack_table() -> list[str]: text = ARCH_MD.read_text(encoding="utf-8") tables = tables_under(text, re.compile(r"^##\s+Host and stack mapping")) @@ -424,6 +464,7 @@ def check_host_stack_table() -> list[str]: ip_match = re.search(r"(\d+\.\d+\.\d+\.\d+)", host_cell) vlan_match = re.search(r"(\d+)", strip_md(row[1])) named_stacks.update(re.findall(r"stacks/([a-z0-9-]+)", row[2])) + planned = bool(len(row) > 3 and NOT_BUILT.search(row[3])) if not (ip_match and vlan_match): problems.append( @@ -438,7 +479,35 @@ def check_host_stack_table() -> list[str]: r for r in rows_for_vlan if strip_md(r[0]).lower() == host.lower() and ip in strip_md(r[1]) ] - if not hit: + + if planned: + # The VLAN must be one network.md actually describes, or a typo'd + # segment would make every assertion below vacuously true. + if not rows_for_vlan: + problems.append( + f"docs/architecture.md plans {host} on VLAN {vlan}, which " + f"docs/network.md has no table for" + ) + if hit: + problems.append( + f"docs/architecture.md still marks {host} 'not built yet', " + f"and docs/network.md now lists it at {ip} on VLAN {vlan} — " + f"it has been built, so drop the marker" + ) + # An unbuilt host planned onto an address something else already + # holds is a real conflict, and the cheapest possible moment to + # find it is before anyone racks it. + clash = [ + r for r in rows_for_vlan + if ip in strip_md(r[1]) and strip_md(r[0]).lower() != host.lower() + ] + if clash: + problems.append( + f"docs/architecture.md plans {host} at {ip}, which " + f"docs/network.md already gives to " + f"{strip_md(clash[0][0])} on VLAN {vlan}" + ) + elif not hit: problems.append( f"docs/architecture.md places {host} at {ip} on VLAN {vlan}; " f"docs/network.md does not list it there" diff --git a/secrets/lab.example.yaml b/secrets/lab.example.yaml new file mode 100644 index 0000000..56e94d6 --- /dev/null +++ b/secrets/lab.example.yaml @@ -0,0 +1,43 @@ +--- +# Template for secrets/lab.sops.yaml. +# +# ONE KEY. That is not an oversight — it is what the lab stack actually needs, +# and the short list is worth reading as a description of the stack: no SNMP +# communities, because it polls no devices; no Alertmanager receiver URLs, +# because it has no Alertmanager (ADR-0020); no renderer token, because it +# ships no dashboards to screenshot. +# +# scripts/render-config.sh derives the required set per stack rather than +# demanding all ten the estate needs, so `make render STACK=lab` asks for this +# and nothing else. +# +# --------------------------------------------------------------------------- +# Create this ON THE LAB GUEST, not on the monitoring host +# --------------------------------------------------------------------------- +# +# make secrets-init STACK=lab # first time only — creates the age keypair +# make secrets-edit STACK=lab # opens the encrypted file in $EDITOR +# +# `.sops.yaml` gives secrets/lab.*.sops.yaml its own creation_rule and its own +# age recipient, above the catch-all that covers everything else under +# secrets/. That separation is the point: the single rule it sits above matches +# ALL of secrets/, so a key added there could decrypt the estate's SNMP +# communities and Grafana admin password too — a lab host holding the +# credentials of the estate it is meant to be isolated from. +# +# `make secrets-init STACK=lab` run on the monitoring host would do exactly +# that, which is why bootstrap.sh now refuses when the key it would write is +# already a recipient of another rule. Run it where the stack runs. + +# Grafana initial admin login. Grafana only reads this when it creates the +# admin user, so changing it later has no effect — change it in the UI, or wipe +# the grafana-data volume. +# +# NOT the same password as the estate's Grafana. Two Grafanas that share a +# password are one credential, and the whole reason there are two of them is +# that the lab is assumed to be the one that gets compromised. Generate one +# with `make gen-secret`. +GRAFANA_ADMIN_PASSWORD: change-me-to-something-long + +# GRAFANA_ADMIN_USER is optional and defaults to `admin` in compose.yaml. Add +# it here if you want a different one. diff --git a/stacks/lab/.env.example b/stacks/lab/.env.example new file mode 100644 index 0000000..f0caa5f --- /dev/null +++ b/stacks/lab/.env.example @@ -0,0 +1,47 @@ +# Non-sensitive tunables for the lab stack. +# +# Edit this file, not .env — scripts/render-config.sh regenerates .env from it +# on every `make up STACK=lab`. Secrets do NOT belong here; see secrets/README.md. + +# Address the published ports bind to. +# +# This guest is single-homed on VLAN 30 (ADR-0007: no trunk, no VLAN-aware +# bridge), so 0.0.0.0 here means ImaginationLAN and nothing else — there is no +# second interface for it to reach further on. That is a meaningfully smaller +# claim than the same line makes on the monitoring host, which sits on the +# management VLAN. +BIND_ADDR=0.0.0.0 + +# The one published port: Grafana, opened from Hicks over the 50 -> 30 rule +# that already exists (ADR-0007). Nothing else in this stack is published — +# compose.yaml says where the `ports:` block goes when #265 gives Prometheus +# and Loki their first off-host clients. +GRAFANA_PORT=3000 + +# Alloy's debug UI, pinned to 127.0.0.1 in compose.yaml. +ALLOY_PORT=12345 + +# Uncomment together with the matching `ports:` blocks in compose.yaml when the +# lab's guests start pushing. Publishing them before that opens an +# unauthenticated remote-write receiver and an unauthenticated log push +# endpoint on the segment that exists to hold attackers, with nothing using +# either (ADR-0012). +# PROMETHEUS_PORT=9090 +# LOKI_PORT=3100 + +# ALLOY_HOSTNAME is NOT set here. scripts/render-config.sh writes it from +# `hostname` at render time — it differs per monitored host, and setting it +# here would pin every host to the same name. + +# Metric retention. Loki's is set separately in loki/loki-config.yaml +# (retention_period) — keep the two in step. +# +# Deliberately shorter than the estate's 30 days, and deliberately not derived +# the way the estate's was: this stack has never run, so there is nothing to +# derive from. It is a bound against ADR-0007's stated constraint — a single +# mirrored pair of 7.2K disks shared with a Windows domain, Wazuh's indexer, +# Velociraptor and PBS. compose.yaml carries the queries to re-derive both once +# the lab has run for a fortnight, and PrometheusSizeRetentionActive in +# prometheus/rules/lab.rules.yaml is what says the ceiling started binding. +PROMETHEUS_RETENTION=15d +PROMETHEUS_RETENTION_SIZE=4GB diff --git a/stacks/lab/README.md b/stacks/lab/README.md new file mode 100644 index 0000000..fa96fbe --- /dev/null +++ b/stacks/lab/README.md @@ -0,0 +1,122 @@ +# Lab observability stack + +Runs on `alexander` (10.0.30.40), VLAN 30 — a **guest on `Saruman`**, not the +hypervisor. A compose stack is Docker, and Docker rewrites the iptables of a +box whose own firewall ADR-0014 relies on, which is why `Saruman` carries the +native `.deb` agent instead ([#88]) and why this runs one level down. +[ADR-0020](../../docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md). + +```bash +make up STACK=lab # from the repository root +``` + +| Service | Image | Port | Purpose | +| --- | --- | --- | --- | +| `prometheus` | `prom/prometheus` | *internal* | Metrics store, remote-write receiver, rule evaluation | +| `loki` | `grafana/loki` | *internal* | Log store | +| `grafana` | `grafana/grafana-oss` | 3000 (https) | Dashboards — the only published port, and the only service that terminates TLS or authenticates | +| `alloy` | `grafana/alloy` | 12345 (localhost) | Metric and log collection | + +Four services, where the estate has seven. What is absent is as deliberate as +what is here: + +- **No Alertmanager.** Nothing in the lab pages. ADR-0020 decided it, and it is + explicitly not an answer to [#257] — that asks whether the lab's *liveness* + may cross to the estate even though its telemetry may not. +- **No snmp-exporter.** `shiva`, the only SNMP device on this segment, is + polled by the estate over the exception ADR-0013 records. Two stacks polling + one device is two answers to "when did it last respond". +- **No blackbox-exporter, no renderer.** Nothing to probe from here yet, and no + dashboards to screenshot. + +## Why this exists at all + +ADR-0007: **lab telemetry stays in the lab.** Nothing in this stack +remote-writes to `10.0.99.20`; both of Alloy's sinks are services in this +compose file. The one exception on this segment predates it and belongs to the +hypervisor, not to any guest — `Saruman`'s own agent, over a single unlogged +pass ([#88]). + +## Layout + +```text +compose.yaml four services, one network, health-gated ordering +.env.example non-sensitive tunables — edit this, not .env +prometheus/ + prometheus.yaml four scrape jobs; no alerting block, no file_sd + rules/lab.rules.yaml 4 rules — this stack watching itself, nothing else + tests/lab.test.yaml promtool unit tests; all four rules, firing + quiet +loki/loki-config.yaml single-binary, filesystem, 15-day retention, no ruler +grafana/ + provisioning/ two datasources + dashboard provider + dashboards/README.md why there are no dashboards yet +``` + +No `alloy/` directory, deliberately. `compose.yaml` mounts `config.alloy` and +`docker.alloy` out of `../observability/alloy/` — ADR-0007 asks for the agent +config "reused unchanged", and a copy is reused-until-someone-edits-one. +`scripts/deploy-agent.sh` exists because `oracle` drifted four separate ways +from a hand-copied agent setup; its header states the rule as *"the fix is to +not copy."* `syslog.alloy` is not mounted: it opens a UDP listener for the +firewall's logs, which is the monitoring host's job and would be the wrong +thing entirely on this segment. + +## Things worth knowing before editing + +- **Nothing here is validated by CI yet.** Every checker in this repository is + pinned to `stacks/observability` — `validate.sh`, the four Python checkers, + `check_loki_rules.sh`, `seed-validation-env.sh` and `ci.yml`. Making them + multi-stack is [#263]. Until it lands, run the checks against this stack by + hand; the commands are at the bottom of this file. +- **Nothing converges this stack, either.** [#99] replaced deploying over SSH + with `scripts/converge.sh` on an hourly timer, and that script runs a bare + `make up` — which is `STACK=observability`, on the monitoring host. This + stack is deployed by hand, on this guest, and a commit that changes it + reaches the lab when someone goes and applies it. Worth knowing before + assuming a merged change is running. +- **The retention figures are a bound, not a measurement.** 15 days and 4 GiB, + against ADR-0007's "sized against spindles, not RAM". `compose.yaml` carries + the queries to re-derive them once this has run for a fortnight, and + `PrometheusSizeRetentionActive` is what says the ceiling started binding. +- **Grafana needs its own leaf, from the same CA as the estate's.** The + `grafana` DNS SAN is load-bearing: the `grafana` scrape job connects to the + compose service name and verifies against it. + + ```bash + make certs ARGS="--host grafana-lab.matrix.elysium --ip 10.0.30.40 --dns grafana" + ``` + +- **Secrets are one key, and it is created on this guest.** See + [`secrets/lab.example.yaml`](../../secrets/lab.example.yaml) — running + `make secrets-init STACK=lab` on the monitoring host would give one age key + both stacks, and `bootstrap.sh` now refuses rather than doing it quietly. +- **Prometheus and Loki are not published.** ADR-0012 publishes a port only + when something off-host uses it, and today only Alloy talks to them, over the + compose network. `compose.yaml` marks the exact lines to uncomment when [#265] + gives them their first real clients. +- **Image tags are pinned here but bumped separately.** `.github/dependabot.yml` + now watches this directory as well as the estate's, so the two do not drift. + Versions are deliberately absent from the table above — Dependabot only edits + `compose.yaml`, so a version written anywhere else goes stale the moment it + lands (#73). + +## Validate before deploying + +`make validate` does **not** cover this stack ([#263]). Until it does: + +```bash +make check-rules STACK=lab +``` + +That target already follows `STACK`, because it uses `$(STACK_DIR)`. The rest +needs the image directly: + +```bash +docker compose -f stacks/lab/compose.yaml config -q +``` + +[#99]: https://github.com/Gerrrt/HomeLab/issues/99 +[#88]: https://github.com/Gerrrt/HomeLab/issues/88 +[#257]: https://github.com/Gerrrt/HomeLab/issues/257 +[#263]: https://github.com/Gerrrt/HomeLab/issues/263 +[#265]: https://github.com/Gerrrt/HomeLab/issues/265 diff --git a/stacks/lab/compose.yaml b/stacks/lab/compose.yaml new file mode 100644 index 0000000..cafb132 --- /dev/null +++ b/stacks/lab/compose.yaml @@ -0,0 +1,286 @@ +--- +# Lab observability stack for the defended estate on `Saruman`. +# +# Deployed on: alexander (10.0.30.40, VLAN 30 / ImaginationLAN) — a guest on +# `Saruman`, NOT the hypervisor itself. A compose stack is Docker, +# and Docker rewrites the iptables of a box whose own firewall +# ADR-0014 relies on, which is why #88 put the native .deb agent +# on the hypervisor and why this runs one level down. +# See docs/adr/0020-run-the-lab-stack-in-a-guest-with-its-own-prometheus.md. +# Deploy with: make up STACK=lab (from the repository root) +# +# This stack exists so that lab telemetry never reaches 10.0.99.20 (ADR-0007). +# Nothing here remote-writes anywhere; both of Alloy's sinks are services in +# this file. +# +# --------------------------------------------------------------------------- +# On the comments in this file +# --------------------------------------------------------------------------- +# The settings below are shared with `stacks/observability/compose.yaml`, and +# the reasoning for them — why `init: true`, why `pids_limit` and not +# `mem_limit`, why `cgroup: host` and not `privileged`, why loki has no +# healthcheck — is written out at length there and is NOT repeated here. Two +# copies of an argument drift the same way two copies of a config do, and the +# argument is the half nobody re-checks. What IS written out here is every +# place this stack DIFFERS from that one, because those are the only lines a +# reader cannot get from the sibling file. + +name: lab + +x-logging: &default-logging + driver: json-file + options: + max-size: "10m" + max-file: "3" + +x-service-defaults: &service-defaults + restart: unless-stopped + logging: *default-logging + networks: [lab] + init: true + pids_limit: 512 + +services: + # --------------------------------------------------------------------------- + # Metrics store. + # + # ADR-0007 named Loki, Grafana and Alloy and no Prometheus, and in the same + # sentence said config.alloy is reused with the two *_URL variables the only + # difference. That file has two sinks. Without this service the second one + # has nowhere to point but 10.0.99.20, which is the one thing that ADR's + # Decision forbids — so the omission was an omission. ADR-0020. + # --------------------------------------------------------------------------- + prometheus: + <<: *service-defaults + image: prom/prometheus:v3.14.0@sha256:5ce7540c3c00ef4ab0c9d2c995c6a5b9c421f44b4a115d97a2c7af3b1c21cbb0 + container_name: lab-prometheus + user: "65534:65534" + command: + - --config.file=/etc/prometheus/prometheus.yaml + - --storage.tsdb.path=/prometheus + # Deliberately NOT the estate's 30d/12GB, and deliberately not derived + # the way those were either — there is nothing to derive from yet. The + # estate's numbers came from measured per-block write rates over a + # retained window; this stack has never run, so any figure here is a + # bound rather than a measurement, and writing it as though it were + # measured would be the more dishonest choice. + # + # What IS known is the constraint. ADR-0007: "128 GB and 48 threads + # against a single mirrored pair of 7.2K disks. The fleet is sized + # against spindles, not RAM." This volume shares those two spindles with + # a Windows domain (#265), Wazuh's OpenSearch indexer (#266), + # Velociraptor (#267) and PBS (#268) — and Wazuh is the one expected to + # be what actually runs out. So this starts small and is meant to be + # re-derived, not defended. + # + # Re-derive it the way the estate's was, once the domain exists and this + # has run for a fortnight: + # rate(prometheus_tsdb_head_samples_appended_total[1d]) + # prometheus_tsdb_storage_blocks_bytes + # and check whether the ceiling has ever bound: + # prometheus_tsdb_time_retentions_total (0 means it never has) + # + # If it has bound, retention is quietly shorter than the line above + # claims. That is the same failure the estate documents; unlike the + # estate, there is no PrometheusSizeRetentionActive rule here to say so + # yet, because there are no dashboards or rules beyond lab.rules.yaml. + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-15d} + - --storage.tsdb.retention.size=${PROMETHEUS_RETENTION_SIZE:-4GB} + # Enabled with nothing pushing to it yet, which is the one place this + # departs from ADR-0012's "publish only what has a consumer" — the flag + # is not a published port, and the receiver is reachable only from this + # compose network until the `ports:` block below is uncommented. It is on + # because Alloy in this same stack pushes through it: the agent uses + # remote_write, not a scrape, exactly as it does on the monitoring host. + - --web.enable-remote-write-receiver + - --web.enable-lifecycle + volumes: + - ./prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro + - ./prometheus/rules:/etc/prometheus/rules:ro + # Only the CA, never a key — the grafana job below scrapes over https and + # verifies the leaf against it. The SAME CA as the estate's: one lab CA, + # two leaves. A second CA would mean a second certificate for every + # browser on Hicks to trust, for no gain. + - ../../certificates/ca.pem:/etc/prometheus/tls/ca.pem:ro + - prometheus-data:/prometheus + # No `ports:` — deliberately, and this is the line to change when #265 + # lands. ADR-0012 publishes a port only when something OFF this host uses + # it, and today nothing does: the only client is Alloy, on this compose + # network. When the Windows endpoints and the Wazuh host get agents they + # become real off-host clients, and this becomes: + # ports: + # - "${BIND_ADDR:-0.0.0.0}:${PROMETHEUS_PORT:-9090}:9090" + # Publishing it now would open an unauthenticated remote-write receiver on + # the segment that exists to hold attackers, months before anything pushes + # to it — "a rule nobody can test", in the shape SECURITY.md keeps finding. + expose: + - "9090" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://localhost:9090/-/healthy"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Log store. No healthcheck, for the reason written out against the `loki` + # service in stacks/observability/compose.yaml: the image is distroless and + # holds no shell, wget or curl to probe with. Readiness is observed by the + # `loki` scrape job in prometheus.yaml instead. + # --------------------------------------------------------------------------- + loki: + <<: *service-defaults + image: grafana/loki:3.7.7@sha256:d70e4659623f3e109af669cae76fe2a5dd5be54e2298fe8aed380d982fbc2500 + container_name: lab-loki + user: "10001:10001" + command: -config.file=/etc/loki/loki-config.yaml + volumes: + - ./loki/loki-config.yaml:/etc/loki/loki-config.yaml:ro + - loki-data:/loki + # No rules mount, unlike the estate's. Loki's ruler needs an Alertmanager + # to deliver to and this stack has none (ADR-0020) — a ruler configured + # with nowhere to send is a rule that evaluates and is discarded, which + # reads as coverage and is not. Log-based alerting for the lab is part of + # the liveness question in #257, not something to half-wire here. + # + # Not published, for the same reason as prometheus above: Alloy is the only + # client and it is on this network. #265 is when that stops being true. + expose: + - "3100" + + # --------------------------------------------------------------------------- + # Visualisation. The one service here a human opens, and the one this stack + # publishes: ADR-0007 says the lab's Grafana is reached from Hicks, over the + # 50 -> 30 rule that already exists. No new firewall rule anywhere. + # --------------------------------------------------------------------------- + grafana: + <<: *service-defaults + image: grafana/grafana-oss:13.0.2@sha256:5dad0df181cb644a14e13617b913b261a54f7d4fd4510721dba420929f35bea2 + container_name: lab-grafana + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:-admin} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?set in secrets/lab.sops.yaml} + GF_SECURITY_COOKIE_SECURE: "true" + GF_SERVER_PROTOCOL: https + GF_SERVER_CERT_FILE: /etc/grafana/tls/cert.pem + GF_SERVER_CERT_KEY: /etc/grafana/tls/key.pem + GF_SECURITY_DISABLE_GRAVATAR: "true" + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_PATHS_PROVISIONING: /etc/grafana/provisioning + PROMETHEUS_URL: http://prometheus:9090 + LOKI_URL: http://loki:3100 + # No GF_RENDERING_* here, and no `renderer` service at the foot of this + # file. The renderer exists on the monitoring host to screenshot + # dashboards into docs/images/; this stack ships no dashboards yet, so + # there is nothing to render. That absence is also why no + # GRAFANA_RENDERER_TOKEN appears in secrets/lab.example.yaml — Grafana + # only refuses to start without one *once a rendering server is + # configured*, and none is. + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards:ro + - grafana-data:/var/lib/grafana + # A second leaf from the SAME CA as the estate's Grafana. Issue it with: + # make certs ARGS="--host grafana-lab.matrix.elysium --ip 10.0.30.40 --dns grafana" + # The `grafana` DNS SAN is not decoration: the prometheus job in this + # stack connects to the compose service name and verifies against it. + - ../../certificates/grafana-lab.matrix.elysium.pem:/etc/grafana/tls/cert.pem:ro + - ../../certificates/grafana-lab.matrix.elysium-key.pem:/etc/grafana/tls/key.pem:ro + group_add: + - "${RENDER_GID:?run make render STACK=lab}" + ports: + - "${BIND_ADDR:-0.0.0.0}:${GRAFANA_PORT:-3000}:3000" + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_started + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "--no-check-certificate", "https://localhost:3000/api/health"] + interval: 30s + timeout: 5s + retries: 3 + start_period: 30s + + # --------------------------------------------------------------------------- + # Collection agent for this guest. + # + # The config is MOUNTED FROM stacks/observability/alloy/, not copied into a + # stacks/lab/alloy/ of its own. ADR-0007 asks for config.alloy "reused + # unchanged"; a copy is reused-until-someone-edits-one, and this repository + # has already paid for that lesson — scripts/deploy-agent.sh exists because + # `oracle` drifted four separate ways from a hand-copied agent setup, and its + # header states the rule: "the fix is to not copy." + # + # Two files, not the directory, and that is the one place this departs from + # the sibling stack's `./alloy:/etc/alloy:ro`. Mounting the directory would + # bring syslog.alloy, which opens a UDP listener for the firewall's logs on + # the host's real interface — on VLAN 30, aimed at a stack that must not + # receive the estate's telemetry. Shipping config.alloy always, docker.alloy + # to hosts with a socket, and syslog.alloy never is exactly the subset + # deploy-agent.sh already ships to a remote Docker host. + # + # The cost, stated: a NEW *.alloy file added to that directory does not reach + # this stack without a line here. That is the same property deploy-agent.sh + # has, and the alternative — a symlinked directory — does not work at all, + # because a bind mount carries the symlink and not its target, and the target + # would have to resolve inside the container. + # --------------------------------------------------------------------------- + alloy: + <<: *service-defaults + image: grafana/alloy:v1.19.2@sha256:b8ec653c44235fbe910879145dac3597d66b0aaecf60bcbbe82580767771a839 + container_name: lab-alloy + pids_limit: 1024 + cap_drop: [ALL] + security_opt: + - no-new-privileges:true + group_add: + - "${LOG_READ_GID:?run make render STACK=lab}" + # The image's own `alloy` group; alloy-data is 0770 473:473. Re-derive + # after an image bump — the sibling compose file has the command. + - "473" + cgroup: host + environment: + # The two variables ADR-0007 calls "the only difference", pointed at this + # stack's own stores. They resolve to the same service names the defaults + # in config.alloy use, so they are strictly redundant — and set anyway, + # because "lab telemetry stays in the lab" should be visible in the file + # an operator reads rather than inferred from a coalesce() three + # directories away. + LOKI_URL: http://loki:3100/loki/api/v1/push + PROMETHEUS_REMOTE_WRITE_URL: http://prometheus:9090/api/v1/write + ALLOY_HOSTNAME: ${ALLOY_HOSTNAME:-alloy} + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy + volumes: + - ../observability/alloy/config.alloy:/etc/alloy/config.alloy:ro + - ../observability/alloy/docker.alloy:/etc/alloy/docker.alloy:ro + - alloy-data:/var/lib/alloy/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log:/var/log:ro + - /:/rootfs:ro + ports: + # Debug UI, loopback only. No syslog receiver on 1514/udp, because + # syslog.alloy is not mounted above. + - "127.0.0.1:${ALLOY_PORT:-12345}:12345" + depends_on: + loki: + condition: service_started + prometheus: + condition: service_healthy + +networks: + lab: + driver: bridge + +volumes: + prometheus-data: + loki-data: + grafana-data: + alloy-data: diff --git a/stacks/lab/grafana/dashboards/README.md b/stacks/lab/grafana/dashboards/README.md new file mode 100644 index 0000000..98fdeca --- /dev/null +++ b/stacks/lab/grafana/dashboards/README.md @@ -0,0 +1,47 @@ +# Lab dashboards + +There are none yet, and that is a decision rather than an omission. + +The estate has seven, and copying them here is the obvious move and the wrong +one. They are built on the estate's datasources, the estate's metrics and the +estate's device inventory: `homelab-network` draws SNMP counters from a switch +this stack does not poll, `homelab-ups` draws an APC on the other side of a +firewall, and `homelab-security` draws labels that +`stacks/observability/alloy/syslog.alloy` extracts from firewall logs this +stack deliberately never receives. A copy would render four rows of empty +panels and one that works, and an empty panel is indistinguishable from a +broken collector — which is the failure mode this repository has been bitten by +more than any other (#62, #63, #71). + +What the lab needs is not known yet, because the thing it exists to observe +does not exist yet. The Windows domain is [#265]; Wazuh is [#266]. Both come +with their own questions about what is worth drawing. + +Until then, `Explore` against the two provisioned datasources is the whole +interface, and it works from the first `make up STACK=lab` — the provider in +`../provisioning/dashboards/dashboards.yaml` reads this directory every 30 +seconds, so the first dashboard committed here appears without a redeploy. + +## When you do add one + +The conventions in +[`stacks/observability/grafana/dashboards/README.md`](../../../observability/grafana/dashboards/README.md) +hold here too, and they are worth reading before starting rather than after: +they are what `scripts/check_dashboards.py` enforces. + +Two things about this stack specifically: + +- **`check_dashboards.py` does not enforce them here.** That checker is pinned + to `stacks/observability`, along with every other validator in the + repository. Making them multi-stack is [#263]. Until it lands, a dashboard + committed to this directory is checked by nothing, so check it by hand. +- **`make dashboards-export STACK=lab` works, and nothing reminds you to run + it.** `scripts/export-dashboards.sh` takes a stack argument, and the provider + here sets `allowUiUpdates: true` so that it can. What the lab does not have + is the estate's `dashboards-drift` timer, which runs `--check` and turns a + forgotten export into a stale job. Here, a dashboard edited in the UI and + never exported simply stays uncommitted. + +[#263]: https://github.com/Gerrrt/HomeLab/issues/263 +[#265]: https://github.com/Gerrrt/HomeLab/issues/265 +[#266]: https://github.com/Gerrrt/HomeLab/issues/266 diff --git a/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml b/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..80846b7 --- /dev/null +++ b/stacks/lab/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,35 @@ +--- +# The provider is here and the folder it reads is empty on purpose — see +# ../../dashboards/README.md. Provisioning an empty directory is not an error: +# Grafana logs nothing and starts normally, and the first dashboard committed +# appears within updateIntervalSeconds without a redeploy. +# +# `allowUiUpdates: true` matches the estate, and matching is the point. +# `scripts/export-dashboards.sh` takes a stack argument, so `make +# dashboards-export STACK=lab` already works against this stack — and it can +# only work with `true`, because Grafana answers `400 Cannot save provisioned +# dashboard` under `false` and the export reads back the file it provisioned +# from. Setting `false` here would have made the lab the one stack where the +# documented export loop silently does nothing (#100). +# +# The gap that leaves, stated rather than discovered later: `true` gives up the +# guarantee that the running dashboard and the committed one are the same, and +# the estate buys that back with the `dashboards-drift` timer running +# `export-dashboards.sh --check`. Nothing runs that against this stack — the +# timer lives on the monitoring host and points at `observability` — so an edit +# made in the lab's Grafana and never exported ages silently. That is the same +# shape as everything else nothing watches here (#257), and it costs nothing +# until this directory has a dashboard in it. +apiVersion: 1 + +providers: + - name: lab + orgId: 1 + folder: Lab + type: file + disableDeletion: false + allowUiUpdates: true + updateIntervalSeconds: 30 + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: false diff --git a/stacks/lab/grafana/provisioning/datasources/datasources.yaml b/stacks/lab/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..092ce1b --- /dev/null +++ b/stacks/lab/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,41 @@ +--- +# Two datasources, not the estate's three: there is no Alertmanager in this +# stack (ADR-0020), so there is none to provision. A datasource pointed at a +# service that does not exist is not inert — Grafana's Alerting pages render it +# as a configured-but-unreachable receiver, which is a worse answer than an +# absent one. +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: ${PROMETHEUS_URL} + isDefault: true + editable: false + jsonData: + timeInterval: 15s + httpMethod: POST + manageAlerts: false + prometheusType: Prometheus + exemplarTraceIdDestinations: [] + + - name: Loki + type: loki + uid: loki + access: proxy + url: ${LOKI_URL} + editable: false + jsonData: + maxLines: 2000 + # Clicking a hostname in a log line jumps to that host's metrics rather + # than making you retype it into a new query. Kept identical to the + # estate's so the two Grafanas behave the same way under the hands of + # someone who uses both. + derivedFields: + - name: host + matcherRegex: 'host="([^"]+)"' + datasourceUid: prometheus + url: '$${__value.raw}' + urlDisplayLabel: "View host metrics" diff --git a/stacks/lab/loki/loki-config.yaml b/stacks/lab/loki/loki-config.yaml new file mode 100644 index 0000000..e6c7114 --- /dev/null +++ b/stacks/lab/loki/loki-config.yaml @@ -0,0 +1,72 @@ +--- +# Single-binary Loki with filesystem storage, same shape as the estate's and +# for the same reason — see docs/adr/0003-observability-stack-selection.md. +# +# Two differences from stacks/observability/loki/loki-config.yaml, both +# deliberate: +# +# retention_period 360h (15 days), not 720h. It tracks the Prometheus +# retention in compose.yaml, which is a bound rather than +# a measurement — this store shares two 7.2K spindles with +# a Windows domain, Wazuh's indexer, Velociraptor and PBS. +# Keep the two in step: a log store outliving the metrics +# that explain it is half an investigation. +# +# no `ruler:` block The estate's ruler delivers to its Alertmanager. This +# stack has none (ADR-0020), and a ruler with nowhere to +# deliver evaluates rules and discards the result, which +# reads as coverage and is not. compose.yaml therefore +# mounts no rules directory either. + +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + log_level: info + +common: + path_prefix: /loki + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + filesystem: + directory: /loki/chunks + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + compaction_interval: 10m + +limits_config: + volume_enabled: true + reject_old_samples: true + reject_old_samples_max_age: 168h + # 15 days, matching PROMETHEUS_RETENTION in .env.example. + retention_period: 360h + max_query_series: 5000 + allow_structured_metadata: true + +analytics: + reporting_enabled: false diff --git a/stacks/lab/prometheus/prometheus.yaml b/stacks/lab/prometheus/prometheus.yaml new file mode 100644 index 0000000..f9e334c --- /dev/null +++ b/stacks/lab/prometheus/prometheus.yaml @@ -0,0 +1,68 @@ +--- +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 30s + external_labels: + site: matrix.elysium + # The estate's says `winterfell`. This one names the segment it serves, so + # that a series copied into a paste, a screenshot or a bug report says which + # of the two stacks it came from without anyone having to remember there are + # two. They are deliberately never joined, so the label is the only thing + # that would ever tell them apart. + monitor: imaginationlan + +rule_files: + - /etc/prometheus/rules/*.rules.yaml + +# No `alerting:` block, because there is no Alertmanager in this stack — +# ADR-0020 decided that, and the reasoning is there. Rules below still evaluate; +# a firing alert is visible in this Prometheus's own /alerts and through +# Grafana. Nothing pages, and nothing is meant to. +# +# What that leaves uncovered is #257: the estate's RemoteWriteJobStale keys on +# jobs arriving on VLAN 99, so it can never see this stack, and a lab that has +# died looks exactly like a lab nobody is using. Adding an alertmanagers: block +# pointed at 10.0.99.20 would "fix" that by opening the remote-write path +# ADR-0007 exists to refuse. Whatever #257 settles goes outside this file. + +scrape_configs: + # --------------------------------------------------------------------------- + # The stack monitoring itself. Service names resolve on the compose network, + # so nothing here carries this guest's address. + # --------------------------------------------------------------------------- + - job_name: prometheus + static_configs: + - targets: ["localhost:9090"] + + # This is also loki's readiness check. The image is distroless and can carry + # no compose healthcheck, so `up{job="loki"}` is the only thing that observes + # it — the same arrangement, and the same reasoning, as the estate's. + - job_name: loki + static_configs: + - targets: ["loki:3100"] + + # Verified properly rather than with insecure_skip_verify: the lab CA is + # mounted into this container, and the leaf carries `grafana` as a SAN + # alongside its FQDN precisely so this scrape can check the name it dials. + - job_name: grafana + scheme: https + tls_config: + ca_file: /etc/prometheus/tls/ca.pem + server_name: grafana + static_configs: + - targets: ["grafana:3000"] + + - job_name: alloy + static_configs: + - targets: ["alloy:12345"] + +# No `snmp` or `blackbox` jobs: this stack runs neither exporter. The devices +# on this segment that answer SNMP — `shiva`, the iLO of this guest's own host +# — are polled by the estate's stack over the exception ADR-0013 records, and +# polling them from here as well would be two stacks scraping one device and +# two answers to "when did it last respond". +# +# No remote-write job list either. Agents on the lab's guests do not need one: +# they push, and a pushed series appears without a target to edit. That is +# #265's work, and it needs the ports: block in compose.yaml opened first. diff --git a/stacks/lab/prometheus/rules/lab.rules.yaml b/stacks/lab/prometheus/rules/lab.rules.yaml new file mode 100644 index 0000000..bf69b3b --- /dev/null +++ b/stacks/lab/prometheus/rules/lab.rules.yaml @@ -0,0 +1,94 @@ +--- +# The lab stack watching itself. +# +# Four rules, and the number is the point. The estate has 48; ADR-0007 names +# giving up "Loki, the dashboards and 40 alert rules" as the price of keeping +# lab telemetry out of the estate, and this file does not pretend to replace +# them. What it covers is this stack's own working — the four things that would +# make every other signal here silently wrong. +# +# NOTHING HERE PAGES. There is no Alertmanager in this stack (ADR-0020), so a +# firing alert is visible in Prometheus's /alerts and in Grafana and nowhere +# else. That is a deliberate limit, not an oversight, and it is why these are +# scoped to "the collector is broken" rather than to anything about the estate +# being defended: an alert nobody is notified of is only useful to someone who +# is already looking, and someone already looking is looking at the lab. +# +# The names deliberately match the estate's rules for the same conditions. +# `external_labels: monitor: imaginationlan` in prometheus.yaml is what +# distinguishes them, and it travels with every series; two different names for +# one condition would not survive a screenshot pasted into an issue. +# +# What this does NOT cover, and cannot: this Prometheus going down. `up == 0` +# is evaluated by the thing that would have stopped. The estate answers that +# for itself with a watcher on `oracle` (ADR-0015) and a heartbeat receiver; +# the lab has neither available to it, which is #257. +groups: + - name: lab + interval: 60s + rules: + # Covers every target in prometheus.yaml — itself, loki, grafana, alloy. + # `up` is the only one of these four that sees a component which has + # stopped publishing altogether; the other three read a metric the + # component emits about its own working, which requires it to be working + # enough to emit. + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + component: lab + severity: critical + category: availability + annotations: + summary: "{{ $labels.job }} target {{ $labels.instance }} is down in the lab stack" + description: >- + The lab's Prometheus has failed to scrape {{ $labels.instance }} + ({{ $labels.job }}) for 5 minutes. Nothing outside the lab can see + this — see #257. + + - alert: PrometheusConfigReloadFailed + expr: prometheus_config_last_reload_successful == 0 + for: 5m + labels: + component: lab + severity: critical + category: correctness + annotations: + summary: "The lab's Prometheus rejected its own config reload" + description: >- + It is still serving the config it last parsed successfully, so the + change you deployed is not the one running. + + # The rule this stack most needs, because its retention figures are a + # bound rather than a measurement — compose.yaml says so at length. If + # 4 GiB turns out to be too small, retention quietly becomes shorter than + # the 15 days configured, the store keeps working and the dashboards keep + # drawing. This counter is the only thing that distinguishes "the ceiling + # engaged" from "old data aged out normally": do not substitute + # prometheus_tsdb_time_retentions_total, which increments by design. + - alert: PrometheusSizeRetentionActive + expr: increase(prometheus_tsdb_size_retentions_total[1h]) > 0 + for: 5m + labels: + component: lab + severity: warning + category: correctness + annotations: + summary: "The lab's size ceiling is evicting blocks — retention is shorter than 15 days" + description: >- + Re-derive the retention figures against the spindles rather than + raising the ceiling blind; compose.yaml carries the queries. + + - alert: LokiIngestionStalled + expr: | + sum(rate(loki_distributor_lines_received_total[30m])) == 0 + for: 30m + labels: + component: lab + severity: warning + category: availability + annotations: + summary: "The lab's Loki has received no log lines for 30 minutes" + description: >- + Either the lab's Alloy is down or the push endpoint is unreachable. + Logs are being lost, not queued indefinitely. diff --git a/stacks/lab/prometheus/tests/lab.test.yaml b/stacks/lab/prometheus/tests/lab.test.yaml new file mode 100644 index 0000000..3d2eb85 --- /dev/null +++ b/stacks/lab/prometheus/tests/lab.test.yaml @@ -0,0 +1,138 @@ +--- +# Unit tests for lab.rules.yaml. +# +# The pairing is the point, and it is the argument #63 produced: a case that +# only ever expects silence would pass against a rule that can never fire. +# `ContainerHighMemory` divided by a limit nothing set, guarded on it being +# non-zero, and was therefore unfireable for any input — while `promtool check +# rules` passed it every time, because that parses PromQL and never asks +# whether an expression can be true. So each rule below gets a firing case AND +# a quiet case: the first proves the expression produces an alert for a real +# input, the second proves it does not produce one for the input production +# actually supplies. +# +# All four rules in lab.rules.yaml are covered here. That is a deliberate +# difference from the estate's tests, where four of the nine stack rules are +# syntax-checked only: this file is small enough that "test the ones that were +# hard" has no advantage over testing all of them, and a new stack has no +# operational history to lean on when deciding which ones were hard. +rule_files: + - ../rules/lab.rules.yaml + +evaluation_interval: 1m + +tests: + # --- InstanceDown -------------------------------------------------------- + - interval: 1m + input_series: + - series: 'up{instance="loki:3100",job="loki"}' + values: "1+0x5 0+0x20" + alert_rule_test: + - eval_time: 20m + alertname: InstanceDown + exp_alerts: + - exp_labels: + alertname: InstanceDown + instance: loki:3100 + job: loki + component: lab + severity: critical + category: availability + exp_annotations: + summary: "loki target loki:3100 is down in the lab stack" + description: >- + The lab's Prometheus has failed to scrape loki:3100 (loki) for + 5 minutes. Nothing outside the lab can see this — see #257. + # Quiet while the target is up, including inside the `for:` window. + - eval_time: 4m + alertname: InstanceDown + exp_alerts: [] + + # --- PrometheusConfigReloadFailed ---------------------------------------- + - interval: 1m + input_series: + - series: 'prometheus_config_last_reload_successful{instance="localhost:9090",job="prometheus"}' + values: "0+0x20" + alert_rule_test: + - eval_time: 10m + alertname: PrometheusConfigReloadFailed + exp_alerts: + - exp_labels: + alertname: PrometheusConfigReloadFailed + instance: localhost:9090 + job: prometheus + component: lab + severity: critical + category: correctness + exp_annotations: + summary: "The lab's Prometheus rejected its own config reload" + description: >- + It is still serving the config it last parsed successfully, so + the change you deployed is not the one running. + + - interval: 1m + input_series: + - series: 'prometheus_config_last_reload_successful{instance="localhost:9090",job="prometheus"}' + values: "1+0x20" + alert_rule_test: + - eval_time: 10m + alertname: PrometheusConfigReloadFailed + exp_alerts: [] + + # --- PrometheusSizeRetentionActive --------------------------------------- + # The counter is flat until the size ceiling starts evicting, then steps. + - interval: 1m + input_series: + - series: 'prometheus_tsdb_size_retentions_total{instance="localhost:9090",job="prometheus"}' + values: "0+0x40 1+0x30" + alert_rule_test: + - eval_time: 60m + alertname: PrometheusSizeRetentionActive + exp_alerts: + - exp_labels: + alertname: PrometheusSizeRetentionActive + instance: localhost:9090 + job: prometheus + component: lab + severity: warning + category: correctness + exp_annotations: + summary: "The lab's size ceiling is evicting blocks — retention is shorter than 15 days" + description: >- + Re-derive the retention figures against the spindles rather + than raising the ceiling blind; compose.yaml carries the + queries. + # A store that has never evicted is the normal case and must stay quiet. + - eval_time: 30m + alertname: PrometheusSizeRetentionActive + exp_alerts: [] + + # --- LokiIngestionStalled ------------------------------------------------ + # A flat counter is a zero rate: lines are arriving nowhere. + - interval: 1m + input_series: + - series: 'loki_distributor_lines_received_total{instance="loki:3100",job="loki"}' + values: "1000+0x90" + alert_rule_test: + - eval_time: 70m + alertname: LokiIngestionStalled + exp_alerts: + - exp_labels: + alertname: LokiIngestionStalled + component: lab + severity: warning + category: availability + exp_annotations: + summary: "The lab's Loki has received no log lines for 30 minutes" + description: >- + Either the lab's Alloy is down or the push endpoint is + unreachable. Logs are being lost, not queued indefinitely. + + - interval: 1m + input_series: + - series: 'loki_distributor_lines_received_total{instance="loki:3100",job="loki"}' + values: "1000+120x90" + alert_rule_test: + - eval_time: 70m + alertname: LokiIngestionStalled + exp_alerts: []