diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go new file mode 100644 index 000000000..36d2c3dac --- /dev/null +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -0,0 +1,352 @@ +package gate + +import ( + "fmt" + "slices" + "time" +) + +// keepLastReason is the instance Reason that check 9 watches for (§10.2). +const keepLastReason = "KeepLast" + +// Obligations this phase leaves for later ones — carried forward the same +// way P6's own deviations list did, so a later review has something concrete +// to check against: +// +// - fromFutureTolerance (§5: 60s) has no constant and no hard-error check +// anywhere yet. Check 2 below implements only "from < StartedAt"; the +// second clause — from more than fromFutureTolerance ahead is a hard +// error — is once-per-run input validation, not a per-rule coverage +// check, and belongs to Check's construction in a later phase (P9). +// - decide (P8) must read a rule's skipped status from the definitions +// (LoggedRule.IsPaused / Definition.IsPaused), never from the polls, and +// must do so BEFORE calling proveCoverage for that rule: a rule paused +// before the window opened is never scheduled or polled (§4.3), so it +// reaches this function with zero polls and today reads as one large +// heartbeat_gap, not skipped (pinned by +// TestProveCoverage_SkippedRuleWithZeroPollsPinnedAsHeartbeatGap). + +// UnobservableReason names why proveCoverage could not prove a rule's window. +// It is machine-readable — this reaches the action's JSON outputs, so it is a +// published vocabulary like Outcome (§19.0); prose belongs in Notes. +type UnobservableReason string + +const ( + ReasonNoSentinel UnobservableReason = "no_sentinel" + ReasonSentinelEarly UnobservableReason = "sentinel_early" + ReasonFromBeforeRecord UnobservableReason = "from_before_record" + ReasonHeartbeatGap UnobservableReason = "heartbeat_gap" + ReasonHealthError UnobservableReason = "health_error" + ReasonStaleEvaluation UnobservableReason = "stale_evaluation" + ReasonPausedInWindow UnobservableReason = "paused_in_window" + ReasonRuleAbsent UnobservableReason = "rule_absent" + // ReasonDrainTimeout is set by check.go's drain wait (a later phase), + // never by proveCoverage: the wait is I/O and must not be added to this + // pure function — that would put HTTP inside the pure layer and destroy + // the seam §2's architecture depends on. + ReasonDrainTimeout UnobservableReason = "drain_timeout" +) + +// CoverageResult is proveCoverage's whole answer for one rule. No interval +// list: proved-or-not plus the largest gap and where is everything a human +// reads on exit 2, and everything §20.2's table needs. +type CoverageResult struct { + Proved bool + LargestGap time.Duration + LargestGapAt time.Time + Unobservable bool + Reason UnobservableReason + Notes []string + // BlindFor is the worst staleness (GrafanaNow - LastEvaluation) that + // tripped check 6; zero when check 6 never fired. + BlindFor time.Duration +} + +// proveCoverage applies the nine coverage checks (§6, §10, §14) to one rule's +// polls and is PURE: no HTTP, no files, no clock reads — everything it needs +// arrives as an argument, which is what lets §22's tests build []Poll literals +// instead of a fixture server (§2). +// +// polls need not be pre-filtered to this rule: proveCoverage selects by +// def.UID itself, exactly as Reduce selects by UID rather than by title +// (§14.5) — a caller handing it a whole log's polls must not have to +// pre-filter to get a correct answer. +// +// Every check always runs, even once an earlier one has already set +// Unobservable: LargestGap and the notes are diagnostics an operator reads on +// exit 2 regardless of which check actually failed (§20.2). Reason names the +// FIRST check, in the order below, that failed; a later failure still adds +// its own Note. +func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, def Definition, + from, to time.Time, grace time.Duration) CoverageResult { + + windowEnd := to.Add(grace) + + var rulePolls []Poll + for _, p := range polls { + if p.RuleUID == def.UID { + rulePolls = append(rulePolls, p) + } + } + // Stable, not sort.Slice: two polls sharing a GrafanaNow (a coarse Date + // header, or a corrupted/replayed log) must not reorder nondeterministically + // in a function that promises to be pure. + slices.SortStableFunc(rulePolls, func(a, b Poll) int { return a.GrafanaNow.Compare(b.GrafanaNow) }) + + var res CoverageResult + fail := func(reason UnobservableReason, note string) { + res.Unobservable = true + if res.Reason == "" { + res.Reason = reason + } + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: %s", def.Title, note)) + } + + // Check 1 — sentinel (§4.5). Present and At >= to+grace -> coverage + // provable; absent, or short of it, is never a pass. A recorder that died + // early must look exactly like a coverage gap, because it is one. + switch { + case sentinel == nil: + fail(ReasonNoSentinel, "no stopped sentinel: the recorder never reported finishing") + case sentinel.Before(windowEnd): + fail(ReasonSentinelEarly, fmt.Sprintf("stopped sentinel at %s is before the required %s (to+grace)", + sentinel.Format(time.RFC3339), windowEnd.Format(time.RFC3339))) + } + + // Check 2 — from bounds (§7), first sentence only: from < StartedAt makes + // coverage unprovable, no matter how healthy the polls that DO exist look. + // Both are runner-domain clock reads (the recorder's own Clock.Now()), so + // no cross-domain translation applies here. The second sentence — from + // more than fromFutureTolerance ahead is a hard error — is Check's input + // validation, once per run rather than per rule, and belongs to a later + // phase: this function has no error return, only a per-rule verdict. + if from.Before(h.StartedAt) { + fail(ReasonFromBeforeRecord, fmt.Sprintf( + "requested from %s is before recording started at %s", from.Format(time.RFC3339), h.StartedAt.Format(time.RFC3339))) + } + + // Filtered once, here, and threaded through every remaining check — + // ruleHeartbeatGap included — rather than re-filtered per check: two + // independent filters over the same polls would only invite one of them + // drifting from the other's membership test. + inWindow := inWindowPolls(rulePolls, from, windowEnd) + + // Check 3 — heartbeat continuity (§6). Data at both ends with a hole in + // between is not enough (§22.4): this scans every gap inside the window, + // not just its edges. + res.LargestGap, res.LargestGapAt = ruleHeartbeatGap(inWindow, from, windowEnd) + if res.LargestGap > t.maxGap { + fail(ReasonHeartbeatGap, fmt.Sprintf( + "gap of %s starting at %s exceeds maxGap %s", res.LargestGap, res.LargestGapAt.Format(time.RFC3339), t.maxGap)) + } + + // Check 4 — health=="error" (§10.1). A short blip is a note only (§22.1: + // one failed evaluation must not exit 2 over an otherwise clean window); + // only a run longer than healthGrace consumes coverage. + if runLen, sawAny := longestHealthRun(inWindow, "error"); sawAny { + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: health=error observed (longest run %s)", def.Title, runLen)) + if runLen > t.healthGrace { + fail(ReasonHealthError, fmt.Sprintf("health=error for %s exceeds healthGrace %s", runLen, t.healthGrace)) + } + } + + // Check 5 — health=="nodata" (§10.1/§10.2). Never fatal here: 96% of the + // fleet runs no_data_state:OK, so treating this as fatal by default would + // block nearly every healthy deploy in an idle environment. Escalating it + // under Policy.NodataIsUnobservable is decide's job (a later phase), + // applied directly against the raw polls — this pure function has no + // Policy to consult and must not invent one. + if _, sawAny := longestHealthRun(inWindow, "nodata"); sawAny { + res.Notes = append(res.Notes, fmt.Sprintf("rule %q: health=nodata observed (not fatal; see --nodata-is-unobservable)", def.Title)) + } + + // Check 6 — liveness (H3). Absolute only, per poll: GrafanaNow and + // LastEvaluation are both Grafana-domain reads off the SAME response, so + // this is a same-domain comparison and uses raw values — never a delta + // against a previous poll, which reports stale on ~half the polls of a + // perfectly healthy rule (polling runs at intervalSeconds/2). + // + // Skipped only for a poll whose own flags SAY there is nothing to check: + // IsPaused (a zero LastEvaluation is legal only while paused, §2.3; check + // 7 is its detector) or !Found (no rule, no evaluation; check 8 is its + // detector). Deliberately NOT skipped merely because LastEvaluation is + // zero: ReadLog does no field validation, so a corrupted or hand-edited + // log line can claim found:true, is_paused:false and still carry a zero + // LastEvaluation, and that combination must read as maximally stale + // rather than being silently waved through. + var staleCount int + var worstStale time.Duration + var worstStaleAt time.Time + for _, p := range inWindow { + if p.IsPaused || !p.Found { + continue + } + if stale := p.GrafanaNow.Sub(p.LastEvaluation); stale > t.evalStaleAfter { + staleCount++ + if stale > worstStale { + worstStale, worstStaleAt = stale, p.GrafanaNow + } + } + } + if staleCount > 0 { + res.BlindFor = worstStale + fail(ReasonStaleEvaluation, fmt.Sprintf( + "lastEvaluation stale on %d poll(s); worst %s (> evalStaleAfter %s) as of %s", + staleCount, worstStale, t.evalStaleAfter, worstStaleAt.Format(time.RFC3339))) + } + + // Check 7 — isPaused in-window (§12.2, §14.8). The PRIMARY pause + // detector: liveness (check 6) is only the backup for what IsPaused + // cannot show (a deleted rule, a stopped scheduler, a blocked + // evaluation). This is what catches pause-then-unpause, which the drain + // wait alone passes (§14.7). + var pausedCount int + var pausedAt time.Time + for _, p := range inWindow { + if p.IsPaused { + pausedCount++ + if pausedAt.IsZero() { + pausedAt = p.GrafanaNow + } + } + } + if pausedCount > 0 { + fail(ReasonPausedInWindow, fmt.Sprintf("observed paused on %d poll(s), first at %s", pausedCount, pausedAt.Format(time.RFC3339))) + } + + // Check 8 — rule absent (§14.5). Found==false is authoritative (P2 + // already retried every transport failure before a Poll record ever + // exists): the rule resolved at resolve time but the state endpoint + // stopped serving it. Never drop a watched rule from the verdict set + // silently. + var absentCount int + var absentAt time.Time + for _, p := range inWindow { + if !p.Found { + absentCount++ + if absentAt.IsZero() { + absentAt = p.GrafanaNow + } + } + } + if absentCount > 0 { + fail(ReasonRuleAbsent, fmt.Sprintf("state endpoint returned no rule on %d poll(s), first at %s", absentCount, absentAt.Format(time.RFC3339))) + } + + // Check 9 — KeepLast (§10.2). A note, never fatal. It surfaces only as an + // instance Reason after P1.2a's parsing, and Reasons keys can be + // comma-joined composites, so membership (reasonsContain) is required — + // indexing "KeepLast" directly would miss "KeepLast, MissingSeries". + for _, p := range inWindow { + if reasonsContain(p.Reasons, keepLastReason) { + res.Notes = append(res.Notes, fmt.Sprintf( + "rule %q: KeepLast observed at %s: a held-over state may hide a real blind spot", def.Title, p.GrafanaNow.Format(time.RFC3339))) + break + } + } + + res.Proved = !res.Unobservable + return res +} + +// inWindowPolls filters polls to those inside [from, windowEnd] using the +// CROSS-DOMAIN membership test (§16): each poll's Grafana-domain GrafanaNow +// is translated to the runner domain by its OWN skew, and its own skew bound +// is the membership tolerance, so a poll that is genuinely inside the window +// is never excluded by ordinary clock imprecision. +// +// Everything downstream of this filter (health runs, liveness, pause, +// absence) reads the poll's raw fields: GrafanaNow paired with +// LastEvaluation on the SAME response, or one poll's GrafanaNow against the +// next's, are same-domain comparisons and need no translation (§16, "Clock +// domains" — only window membership and check 3's two boundary segments do). +func inWindowPolls(polls []Poll, from, windowEnd time.Time) []Poll { + var out []Poll + for _, p := range polls { + bound := p.SkewBound() + runner := p.GrafanaNow.Add(-p.Skew()) + if runner.Before(from.Add(-bound)) || runner.After(windowEnd.Add(bound)) { + continue + } + out = append(out, p) + } + return out +} + +// ruleHeartbeatGap finds the largest unobserved span inside [from, windowEnd] +// (§6), including the two boundary segments — which is why "data at both +// ends with a hole in the middle" still fails (§22.4): the segment between +// the polls just inside each edge is exactly what this measures. in must +// already be filtered to this window (inWindowPolls) and sorted by +// GrafanaNow — proveCoverage computes that filter once and threads it through +// every check, this one included, rather than each check re-filtering. +// +// The two boundary segments compare a Grafana-domain poll time against the +// runner-domain from/windowEnd, so each is translated by its own poll's skew +// AND widened by that same poll's skew bound (§16: "with that poll's bound as +// the tolerance") — on the side that makes the segment larger, never smaller, +// so an uncertain boundary reads as at least as big a gap as it might really +// be. Understating it by up to the bound would be fail-open. The spacing +// BETWEEN consecutive polls compares two Grafana-domain reads to each other — +// same domain — and uses the raw GrafanaNow difference, no bound needed. +func ruleHeartbeatGap(in []Poll, from, windowEnd time.Time) (largestGap time.Duration, largestGapAt time.Time) { + if len(in) == 0 { + return windowEnd.Sub(from), from + } + + runnerOf := func(p Poll) time.Time { return p.GrafanaNow.Add(-p.Skew()) } + + first := in[0] + if gap := runnerOf(first).Sub(from) + first.SkewBound(); gap > largestGap { + largestGap, largestGapAt = gap, from + } + for i := 1; i < len(in); i++ { + if gap := in[i].GrafanaNow.Sub(in[i-1].GrafanaNow); gap > largestGap { + largestGap, largestGapAt = gap, runnerOf(in[i-1]) + } + } + last := in[len(in)-1] + if gap := windowEnd.Sub(runnerOf(last)) + last.SkewBound(); gap > largestGap { + largestGap, largestGapAt = gap, runnerOf(last) + } + return largestGap, largestGapAt +} + +// longestHealthRun returns the longest contiguous wall-clock span (§10.1) +// during which polls — already sorted by GrafanaNow, same-domain spacing +// (§16) — read the given rule-level Health, and whether any poll matched it +// at all. +// +// It detects the span as it accumulates rather than waiting for the run to +// end, so an open-ended run that is still failing at the last poll in the +// window is measured correctly without needing data past the window: waiting +// for the run to "end" would have to assume the best case about what happens +// next, which is exactly what this gate must not do (§1). +func longestHealthRun(polls []Poll, health string) (longest time.Duration, sawAny bool) { + var runStart time.Time + for _, p := range polls { + if p.Health != health { + runStart = time.Time{} + continue + } + sawAny = true + if runStart.IsZero() { + runStart = p.GrafanaNow + } + if span := p.GrafanaNow.Sub(runStart); span > longest { + longest = span + } + } + return longest, sawAny +} + +// reasonsContain reports whether any key of reasons names want, honoring +// Grafana's comma-joined composite reason strings via reasonNames (log.go). +func reasonsContain(reasons map[string]int, want string) bool { + for reason := range reasons { + if reasonNames(reason, want) { + return true + } + } + return false +} diff --git a/grafana-alertcheck/internal/gate/coverage_test.go b/grafana-alertcheck/internal/gate/coverage_test.go new file mode 100644 index 000000000..7ef4b5427 --- /dev/null +++ b/grafana-alertcheck/internal/gate/coverage_test.go @@ -0,0 +1,655 @@ +package gate + +import ( + "strings" + "testing" + "time" +) + +func TestProveCoverage_CleanWindowIsProved(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", State: "inactive", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved || res.Unobservable || res.Reason != "" { + t.Fatalf("res = %+v, want a clean proved window", res) + } +} + +func TestProveCoverage_FiltersPollsByUID(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + // A different rule's polls, deliberately broken, must never + // contaminate r1's verdict: proveCoverage selects by UID itself. + polls = append(polls, Poll{RuleUID: "other", GrafanaNow: ts, Found: false}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved { + t.Fatalf("res = %+v, want proved: a different rule's broken polls must not affect this rule's verdict", res) + } +} + +// --- Check 1: sentinel (§4.5) --- + +func TestProveCoverage_NoSentinelIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, rt, def, from, to, 0) + if res.Proved || res.Reason != ReasonNoSentinel { + t.Fatalf("res = %+v, want unobservable/no_sentinel: an absent sentinel must never be a pass", res) + } +} + +func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := 2 * time.Minute + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to.Add(grace).Add(-time.Second) // one second short of to+grace + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, grace) + if res.Reason != ReasonSentinelEarly { + t.Fatalf("Reason = %q, want sentinel_early", res.Reason) + } +} + +func TestProveCoverage_SentinelExactlyAtGraceIsFine(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := 2 * time.Minute + windowEnd := to.Add(grace) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(windowEnd); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + sentinel := windowEnd + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, grace) + if !res.Proved { + t.Fatalf("Proved = false, want true: sentinel exactly at to+grace must satisfy check 1: %+v", res) + } +} + +// --- Check 2: from bounds (§7) --- + +func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { + started := time.Date(2026, 1, 1, 1, 0, 0, 0, time.UTC) + from := started.Add(-time.Minute) // the requested window opens before recording started + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + sentinel := to + res := proveCoverage(Header{StartedAt: started}, nil, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonFromBeforeRecord { + t.Fatalf("Reason = %q, want from_before_record", res.Reason) + } +} + +// --- Check 3: heartbeat continuity (§6) --- + +// TestProveCoverage_HeartbeatGapBetweenBoundariesIsUnobservable is §22.4's +// core regression: data at both ends with a hole between is not enough. +func TestProveCoverage_HeartbeatGapBetweenBoundariesIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // maxGap = 60s + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + {RuleUID: "r1", GrafanaNow: from.Add(time.Second), Found: true, Health: "ok", LastEvaluation: from}, + {RuleUID: "r1", GrafanaNow: to.Add(-time.Second), Found: true, Health: "ok", LastEvaluation: to}, + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonHeartbeatGap { + t.Fatalf("Reason = %q, want heartbeat_gap: healthy edges with a hole in the middle must still fail (§22.4)", res.Reason) + } + // The gap is the SPACING between the two polls (598s), not either + // boundary segment (1s each) — pin the actual values, not just the verdict. + if res.LargestGap != 598*time.Second { + t.Fatalf("LargestGap = %s, want 598s (the spacing between the two polls, not a boundary segment)", res.LargestGap) + } + wantAt := from.Add(time.Second) + if !res.LargestGapAt.Equal(wantAt) { + t.Fatalf("LargestGapAt = %s, want %s (where the gap starts, at the first poll)", res.LargestGapAt, wantAt) + } +} + +// --- Check 4/5: health (§10.1/§10.2) --- + +func TestProveCoverage_HealthErrorShortBlipPassesWithNote(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // healthGrace = 60s + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + blip := from.Add(2 * time.Minute) + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + health := "ok" + if ts.Equal(blip) { + health = "error" // one isolated failed evaluation + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: health, LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved { + t.Fatalf("Proved = false, want true: one failed evaluation must not fail an otherwise clean window (§22.1): %+v", res) + } + if !anyContains(res.Notes, "health=error") { + t.Fatalf("Notes = %v, want a health=error note even though it did not fail the window", res.Notes) + } +} + +func TestProveCoverage_HealthErrorSustainedIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // healthGrace = 60s + def := Definition{UID: "r1", Title: "R1"} + + runStart, runEnd := from.Add(2*time.Minute), from.Add(5*time.Minute) // a 3-minute run, well past healthGrace + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + health := "ok" + if !ts.Before(runStart) && !ts.After(runEnd) { + health = "error" + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: health, LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonHealthError { + t.Fatalf("Reason = %q, want health_error for a run that outlasts healthGrace", res.Reason) + } +} + +func TestProveCoverage_HealthNodataNeverFatalHere(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved { + t.Fatalf("Proved = false, want true: health=nodata for the WHOLE window must still not be fatal by itself "+ + "(escalating it is Policy.NodataIsUnobservable's job, applied by decide in a later phase): %+v", res) + } + if !anyContains(res.Notes, "health=nodata") { + t.Fatalf("Notes = %v, want a health=nodata note", res.Notes) + } +} + +// --- Check 6: liveness / H3 --- + +// TestProveCoverage_LivenessAbsoluteNeverFalseStale is §22.7's disproportionate +// test: a healthy rule polled at intervalSeconds/2, across the full window, +// must show zero staleness violations. lastEvaluation only advances once per +// full evaluation interval here — the realistic shape a delta check +// misreads as stale on roughly half of all polls (H3). +func TestProveCoverage_LivenessAbsoluteNeverFalseStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + pollEvery := 30 * time.Second + intervalSeconds := 60 + windowEnd := from.Add(10 * time.Minute) + rt := newRuleTimings(pollEvery, intervalSeconds) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + lastEval := from + for ts := from; !ts.After(windowEnd); ts = ts.Add(pollEvery) { + if ts.Sub(lastEval) >= time.Duration(intervalSeconds)*time.Second { + lastEval = ts + } + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: lastEval}) + } + sentinel := windowEnd + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, windowEnd, 0) + if res.Reason == ReasonStaleEvaluation || res.BlindFor != 0 { + t.Fatalf("proveCoverage flagged staleness on a healthy rule polled at intervalSeconds/2 — H3 must be absolute, "+ + "never a delta against a previous poll: %+v", res) + } + if !res.Proved { + t.Fatalf("Proved = false, want true: %+v (notes: %v)", res, res.Notes) + } +} + +func TestProveCoverage_StaleEvaluationIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // evalStaleAfter = 120s + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 6 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + staleAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(staleAt) { + polls[i].LastEvaluation = staleAt.Add(-3 * time.Minute) + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonStaleEvaluation { + t.Fatalf("Reason = %q, want stale_evaluation", res.Reason) + } + if res.BlindFor != 3*time.Minute { + t.Fatalf("BlindFor = %s, want 3m", res.BlindFor) + } +} + +func TestProveCoverage_ZeroLastEvaluationNeverFalseStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // A paused rule legitimately reports the zero time (§2.3); check 6 must + // not read that as an enormous staleness violation. Check 7 is its + // detector. + polls := []Poll{ + {RuleUID: "r1", GrafanaNow: from.Add(time.Minute), Found: true, IsPaused: true}, + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason == ReasonStaleEvaluation { + t.Fatalf("a zero lastEvaluation on a paused poll must not trigger check 6: %+v", res) + } +} + +// --- Check 7: isPaused in-window (§12.2, §14.8) --- + +func TestProveCoverage_PausedInWindowIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 7 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + pausedAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(pausedAt) { + polls[i].IsPaused = true + polls[i].LastEvaluation = time.Time{} // legal only while paused, §2.3 + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonPausedInWindow { + t.Fatalf("Reason = %q, want paused_in_window", res.Reason) + } +} + +// TestProveCoverage_PausedAfterWindowIsFine pins check 7's respect for the +// window boundary: a poll that reports paused but lands beyond windowEnd (a +// rule paused only after THIS release window closed) is filtered out by +// inWindowPolls and must not fail the window. Without that filter, a pause in +// the next release's window would wrongly fail this one. +func TestProveCoverage_PausedAfterWindowIsFine(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: to.Add(2 * time.Minute), + Found: true, Health: "ok", IsPaused: true, + }) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason == ReasonPausedInWindow { + t.Fatalf("a paused poll after windowEnd tripped check 7: %+v", res.Notes) + } + if !res.Proved { + t.Fatalf("Proved = false, want a clean window: %+v", res.Notes) + } +} + +// --- Check 8: rule absent (§14.5) --- + +func TestProveCoverage_RuleAbsentIsUnobservable(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + // Dense, otherwise-healthy polling so heartbeat continuity (check 3) + // stays intact — only check 8 should be able to fire. + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + absentAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(absentAt) { + polls[i].Found = false + polls[i].Health = "" + polls[i].LastEvaluation = time.Time{} + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonRuleAbsent { + t.Fatalf("Reason = %q, want rule_absent", res.Reason) + } +} + +// denseHealthyPolls builds a clean poll sequence at a fixed cadence, with +// zero staleness and nothing abnormal — the baseline the single-check tests +// mutate exactly one poll of, so heartbeat continuity (check 3) never +// confounds the check under test. +func denseHealthyPolls(uid string, from, to time.Time, every time.Duration) []Poll { + var out []Poll + for ts := from; !ts.After(to); ts = ts.Add(every) { + out = append(out, Poll{RuleUID: uid, GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + return out +} + +// --- Check 9: KeepLast (§10.2) --- + +func TestProveCoverage_KeepLastIsNoteOnly(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts, + // A comma-joined composite — reasonsContain must match by + // membership, never by an exact key, per P5's markers. + Reasons: map[string]int{"KeepLast, MissingSeries": 1}, + }) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved { + t.Fatalf("Proved = false, want true: KeepLast is a note, never fatal: %+v", res) + } + if !anyContains(res.Notes, "KeepLast") { + t.Fatalf("Notes = %v, want a KeepLast note (comma-joined membership, not a literal-key match)", res.Notes) + } +} + +// --- Clock domains (§16) --- + +// TestProveCoverage_SkewTranslationAtWindowBoundary pins §16's "Clock +// domains" rule: a constant clock skew on every poll must not itself read as +// a coverage gap or a from-before-record violation, because every +// cross-domain comparison translates by that poll's own skew first. +func TestProveCoverage_SkewTranslationAtWindowBoundary(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + const skew = 45 * time.Second // Grafana's clock reads 45s ahead of the runner's + const bound = 5 * time.Second + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + grafanaTime := ts.Add(skew) + polls = append(polls, Poll{ + RuleUID: "r1", GrafanaNow: grafanaTime, SkewMS: skew.Milliseconds(), SkewBoundMS: bound.Milliseconds(), + Found: true, Health: "ok", LastEvaluation: grafanaTime, + }) + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if !res.Proved { + t.Fatalf("res = %+v, want proved: a constant clock skew must not itself read as a coverage gap (§16)", res) + } +} + +// --- Override round-trip (P5's "two authorities") --- + +// TestProveCoverage_OverrideRoundTrip is P7's other disproportionate done-gate +// test: it exercises DeriveTimingsFromLog and proveCoverage together, exactly +// as check will, to prove maxGap tracks the RECORDED cadence, never a +// re-derivation from the rule's own evaluation interval. +func TestProveCoverage_OverrideRoundTrip(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + + t.Run("slower override on a tighter rule classifies clean", func(t *testing.T) { + windowEnd := from.Add(10 * time.Minute) + h := Header{ + StartedAt: from.Add(-time.Hour), + Rules: []LoggedRule{{UID: "r1", Title: "R1", IntervalSeconds: 60, PollEverySeconds: 120}}, + } + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60}} + rt, _, err := DeriveTimingsFromLog(h, defs) + if err != nil { + t.Fatalf("DeriveTimingsFromLog: %v", err) + } + + var polls []Poll + for ts := from; !ts.After(windowEnd); ts = ts.Add(120 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + } + sentinel := windowEnd + + res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) + if !res.Proved { + t.Fatalf("Proved = false, want true (maxGap must come from the recorded 120s cadence, not the 30s default): %+v", res) + } + }) + + t.Run("faster override still catches a real recorder gap", func(t *testing.T) { + windowEnd := from.Add(20 * time.Minute) + h := Header{ + StartedAt: from.Add(-time.Hour), + Rules: []LoggedRule{{UID: "r1", Title: "R1", IntervalSeconds: 300, PollEverySeconds: 5}}, + } + defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 300}} + rt, _, err := DeriveTimingsFromLog(h, defs) + if err != nil { + t.Fatalf("DeriveTimingsFromLog: %v", err) + } + + var polls []Poll + ts := from + for range 20 { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + ts = ts.Add(5 * time.Second) + } + // The one real gap: 250s, nowhere near this recording's actual 5s + // cadence. Resume 5s polling afterward all the way to windowEnd, so + // this hole is the ONLY gap in the window — otherwise an uncovered + // tail would exceed even the WRONG (definition-derived) 300s maxGap + // on its own, and the test could not tell the two derivations apart. + ts = ts.Add(250 * time.Second) + for !ts.After(windowEnd) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", LastEvaluation: ts}) + ts = ts.Add(5 * time.Second) + } + sentinel := windowEnd + + res := proveCoverage(h, polls, &sentinel, rt["r1"], defs[0], from, windowEnd, 0) + if res.Reason != ReasonHeartbeatGap { + t.Fatalf("Reason = %q, want heartbeat_gap: if maxGap had been re-derived from the 300s definition instead of "+ + "the recorded 5s cadence, this 250s gap would pass silently — the fail-open direction P5 warns about", res.Reason) + } + }) +} + +func anyContains(notes []string, substr string) bool { + for _, n := range notes { + if strings.Contains(n, substr) { + return true + } + } + return false +} + +// --- Check 6, tightened: a corrupted log must not silently disable liveness --- + +// TestProveCoverage_ZeroLastEvaluationWithoutPauseIsStale guards check 6's +// skip condition. ReadLog does no field validation, so a log line can claim +// found:true, is_paused:false and still carry a zero LastEvaluation (a +// corrupted write, a hand-edited fixture, a future log format bug). That +// combination must read as maximally stale, not be waved through the way a +// legitimately paused poll's zero time is (§2.3) — the skip must key off +// IsPaused/Found, never off LastEvaluation being zero. +func TestProveCoverage_ZeroLastEvaluationWithoutPauseIsStale(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + corruptAt := from.Add(5 * time.Minute) + for i := range polls { + if polls[i].GrafanaNow.Equal(corruptAt) { + polls[i].LastEvaluation = time.Time{} // found:true, is_paused:false, yet zero + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonStaleEvaluation { + t.Fatalf("Reason = %q, want stale_evaluation: a zero lastEvaluation on a found, non-paused poll must fail "+ + "closed, not be silently skipped as if it were a legitimately paused observation", res.Reason) + } +} + +// --- Check 3, tightened: the boundary segments must widen by the skew bound --- + +// TestProveCoverage_BoundaryGapWidensBySkewBound pins §16's "with that +// poll's bound as the tolerance" for the two boundary segments specifically: +// a boundary gap that lands EXACTLY at maxGap must still fail once the +// poll's own skew bound is added, because the translation is only a best +// estimate and understating the gap by up to the bound would be fail-open. +func TestProveCoverage_BoundaryGapWidensBySkewBound(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) // maxGap = 60s + def := Definition{UID: "r1", Title: "R1"} + + const bound = 5 * time.Second + first := Poll{ + RuleUID: "r1", GrafanaNow: from.Add(rt.maxGap), Found: true, Health: "ok", + LastEvaluation: from.Add(rt.maxGap), SkewBoundMS: bound.Milliseconds(), + } + rest := denseHealthyPolls("r1", from.Add(rt.maxGap+30*time.Second), to, 30*time.Second) + polls := append([]Poll{first}, rest...) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonHeartbeatGap { + t.Fatalf("Reason = %q, want heartbeat_gap: the leading boundary segment sits at EXACTLY maxGap (60s) before "+ + "widening; the poll's own %s skew bound must push it past the threshold (§16), not just the skew translation", res.Reason, bound) + } +} + +// --- Multi-failure contract --- + +// TestProveCoverage_MultipleFailuresReasonIsFirstButAllNoted exercises two +// checks failing in the same rule: check 7 (paused in-window) precedes check +// 8 (rule absent) in the §5 order, so Reason must name the pause even though +// the rule also goes absent later — and the later failure must still add its +// own Note rather than being swallowed once Reason is set. +func TestProveCoverage_MultipleFailuresReasonIsFirstButAllNoted(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1"} + + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + pausedAt := from.Add(3 * time.Minute) + absentAt := from.Add(6 * time.Minute) + for i := range polls { + switch { + case polls[i].GrafanaNow.Equal(pausedAt): + polls[i].IsPaused = true + polls[i].LastEvaluation = time.Time{} + case polls[i].GrafanaNow.Equal(absentAt): + polls[i].Found = false + polls[i].Health = "" + polls[i].LastEvaluation = time.Time{} + } + } + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonPausedInWindow { + t.Fatalf("Reason = %q, want paused_in_window (the FIRST check to fail, in §5's order)", res.Reason) + } + if !anyContains(res.Notes, "paused") { + t.Fatalf("Notes = %v, want a note about the pause", res.Notes) + } + if !anyContains(res.Notes, "no rule") { + t.Fatalf("Notes = %v, want a note about the absence too — a later failure must still be recorded, "+ + "not swallowed once Reason is already set", res.Notes) + } +} + +// --- Skipped rules (P6/P8 obligation) --- + +// TestProveCoverage_SkippedRuleWithZeroPollsPinnedAsHeartbeatGap pins a known +// gap in this function's contract, not a bug in it: a rule paused BEFORE the +// window opened is never scheduled or polled (watch.go, §4.3), so it reaches +// proveCoverage with zero polls at all. proveCoverage has no notion of +// "skipped" — that classification belongs to the definitions +// (LoggedRule.IsPaused / Definition.IsPaused), never to the polls — so today +// it reports the whole window as one big heartbeat_gap instead. decide (P8) +// MUST read skipped status from the definitions and either skip calling this +// function for that rule entirely, or override this result — this test pins +// today's behavior so that review has something concrete to check against. +func TestProveCoverage_SkippedRuleWithZeroPollsPinnedAsHeartbeatGap(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + def := Definition{UID: "r1", Title: "R1", IsPaused: true} + + sentinel := to + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, rt, def, from, to, 0) + if res.Reason != ReasonHeartbeatGap { + t.Fatalf("Reason = %q, want heartbeat_gap (pinned, not the desired end state): proveCoverage has no "+ + "'skipped' concept, so decide (P8) must handle a skipped rule's classification itself, before or "+ + "instead of calling this function", res.Reason) + } +}