diff --git a/grafana-alertcheck/internal/gate/check_test.go b/grafana-alertcheck/internal/gate/check_test.go index 2971c30e4..8cd94468f 100644 --- a/grafana-alertcheck/internal/gate/check_test.go +++ b/grafana-alertcheck/internal/gate/check_test.go @@ -3,6 +3,7 @@ package gate import ( "bufio" "context" + "encoding/json" "errors" "fmt" "os" @@ -300,6 +301,83 @@ func TestCheckSingleStepCleanWindowPasses(t *testing.T) { } } +// §22.2: the collapse-note-plus-satisfied-MinObserved path (resolve_test.go's +// TestResolve_CollapseByUIDGivesNoteNotError and +// TestResolve_MinObservedCountIsPostCollapse) is proven only at Resolve() +// directly; this drives the same shape through check() end to end — the two +// input names must collapse to one verdict, the run must pass, and the +// collapse note must reach the run's own notes, not just Resolve()'s return +// value. +func TestCheckSingleStepDuplicateAlertNamesCollapseWithNote(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.Alerts = []string{"uid:" + checkUID, checkTitle} // the same rule, named two different ways + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + if err != nil { + t.Fatalf("check() = %v, want nil\nnotes:\n%s", err, notesOf(cfg)) + } + if len(res.Verdicts) != 1 { + t.Fatalf("Verdicts = %+v, want exactly one — the duplicate must collapse to a single rule", res.Verdicts) + } + if len(res.Violations) != 0 { + t.Fatalf("Violations = %+v, want none: MinObserved must be satisfied by the post-collapse count of 1", res.Violations) + } + if notes := notesOf(cfg); !strings.Contains(notes, "counted once") { + t.Errorf("want the collapse note in the run's own notes; got:\n%s", notes) + } +} + +// §22.1's highest-priority regression: a rule with health=error for the +// whole window is unobservable, exit 2 — using the real "[JD] No Job +// Proposals" capture (testdata/README.md), not a synthetic Poll table, so a +// change in how the real payload shapes health/lastError cannot slip past a +// hand-built fixture that happens to still look right. +func TestCheckSingleStepContinuousHealthErrorIsUnobservable(t *testing.T) { + body := readFixture(t, "state_health_error.json") + rules, err := ParseState(body) + if err != nil { + t.Fatalf("ParseState: %v", err) + } + base := rules[0] + def := Definition{ + UID: base.UID, Title: base.Title, Folder: base.Folder, Group: base.Group, + IntervalSeconds: int(base.Interval / time.Second), NoDataState: "OK", ExecErrState: "OK", + Kind: KindGrafanaManaged, + } + + clock := newVirtualClock(testNow) + cfg := Config{ + URL: "https://grafana.example.com", Alerts: []string{"uid:" + def.UID}, + From: testNow, To: testNow.Add(5 * time.Minute), Clock: clock, Notes: &strings.Builder{}, + }.withDefaults() + + src := newCheckSource(func(_ string, _ int) (Observation, error) { + // Every field but LastEvaluation stays exactly as the real capture + // shaped it (health=error, the real lastError text, the real Error + // instance); LastEvaluation tracks the poll so staleness (a + // different coverage check, §14) never becomes the actual cause. + r := base + r.LastEvaluation = clock.Now() + return Observation{Rules: []StateRule{r}, GrafanaNow: clock.Now(), Latency: 200 * time.Millisecond}, nil + }) + src.defs = []Definition{def} + + res, err := check(context.Background(), cfg, src) + if err == nil { + t.Fatalf("check() = nil, want an error: continuous health=error must be unobservable (§22.1, H6/H7)\nnotes:\n%s", notesOf(cfg)) + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want one unobservable verdict", res.Verdicts) + } + if cov := res.Coverage[def.UID]; cov.Reason != ReasonHealthError { + t.Fatalf("Coverage[%s].Reason = %q, want %q", def.UID, cov.Reason, ReasonHealthError) + } +} + // H5: a certain violation does not release the runner early, and it does not // stop the gate reporting exit-1 shape — violations with a nil error. func TestCheckSingleStepFiringInstanceReportsWithoutExitingEarly(t *testing.T) { @@ -329,6 +407,65 @@ func TestCheckSingleStepFiringInstanceReportsWithoutExitingEarly(t *testing.T) { } } +// §22.8: "newly_bad at from+30s gives exit 1, but ONLY after +// to+transition_grace." The test above pins H5 for a rule already bad +// before the window opened (persistently_bad); this pins the anti-fail-fast +// case the plan names explicitly — a fresh onset just inside the window +// must not release the runner the instant it is first observed. +func TestCheckSingleStepNewOnsetDoesNotExitEarly(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + onset := testNow.Add(30 * time.Second) + src := newCheckSource(func(_ string, _ int) (Observation, error) { + now := clock.Now() + if now.Before(onset) { + return healthyObservation(now), nil + } + firing := Instance{ + Labels: map[string]string{"alertname": checkTitle, "instance": "a"}, + State: StateFiring, + ActiveAt: onset, + } + return healthyObservation(now, firing), nil + }) + + res, err := check(context.Background(), cfg, src) + if err != nil { + t.Fatalf("check() = %v, want nil (a violation is exit 1, not an error)", err) + } + if len(res.Violations) != 1 || res.Violations[0].Outcome != OutcomeNewlyBad { + t.Fatalf("Violations = %+v, want exactly one newly_bad", res.Violations) + } + if windowEnd := cfg.To.Add(checkGrace); clock.Now().Before(windowEnd) { + t.Errorf("exited early at %s; H5 requires collecting to %s even for a fresh onset at from+30s", clock.Now(), windowEnd) + } +} + +// §22.9: an ABSENT `from` in single-step mode (as opposed to recorder mode, +// which hard-errors — TestCheckValidateRejectsBadConfigurations's "log mode +// without from") falls back to the start of this check step, with the same +// declared-blind-interval warning as an explicit early `from`. +func TestCheckSingleStepAbsentFromFallsBackToStepStart(t *testing.T) { + clock := newVirtualClock(testNow) + cfg := baseConfig(t, clock) + cfg.From = time.Time{} + src := newCheckSource(func(_ string, _ int) (Observation, error) { + return healthyObservation(clock.Now()), nil + }) + + res, err := check(context.Background(), cfg, src) + if err != nil { + t.Fatalf("check() = %v, want nil: an absent `from` in single-step mode is a fallback, not an error\nnotes:\n%s", err, notesOf(cfg)) + } + notes := notesOf(cfg) + if !strings.Contains(notes, "no `from` given") { + t.Errorf("want the §4.2 fallback note; notes were:\n%s", notes) + } + if !res.From.Equal(testNow) { + t.Errorf("Result.From = %s, want the step-start fallback %s", res.From, testNow) + } +} + // §4.2/§22.4: in single-step mode an explicit `from` earlier than the first // observation is a DECLARED blind interval — a warning and a pass, naming the // exact interval it cannot see. Recorder mode keeps P7 check 2 strict. @@ -645,6 +782,51 @@ func TestCheckFailClosedOnCoverageGap(t *testing.T) { } } +// §22.4: "an episode fully between the deploy and the start of the check" — +// recorder mode must find this at the LEADING edge of the window too, right +// after `from` (the deploy's completion), not only in the middle +// (TestCheckFailClosedOnCoverageGap above). No poll exists for +// [from, from+3m): whatever happened there is invisible to every per-poll +// check, so only the coverage gap itself can catch it — the reason this +// two-phase recorder model exists at all (§4.2). +func TestCheckRecorderModeFindsAGapImmediatelyAfterTheDeploy(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + path := filepath.Join(dir, "log.jsonl") + clock := newFakeClock(windowEnd.Add(30 * time.Second)) + w, err := NewWriter(path, clock) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + if err := w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, + }); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + gapEnd := testNow.Add(3 * time.Minute) // nothing recorded from `from` (testNow) to here + for at := gapEnd; !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { + if err := w.WritePoll(Poll{ + RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at, + }); err != nil { + t.Fatalf("WritePoll: %v", err) + } + } + if err := w.Stop(); err != nil { + t.Fatalf("Stop: %v", err) + } + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + if err == nil { + t.Fatalf("check() = nil, want exit 2: a hole right after the deploy hides whatever happened there just as much as one in the middle") + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want one unobservable verdict, never clean", res.Verdicts) + } +} + // §19.3 case 5: the drain limit passed. The recording itself is clean, so this // isolates the drain wait — the rule simply never evaluates through the end of // the window, and a rule that cannot answer that question is unobservable. @@ -1027,6 +1209,88 @@ func TestCheckDoesNotSignalABystanderHoldingAReusedPid(t *testing.T) { } } +// §22.5: a dead pidfile (the recorder process has already exited, holding no +// flock) with NO sentinel in the log — the shape a killed `watch` leaves +// behind — must not hang the stop wait: the flock is free immediately, so +// check reads the log at once, finds no sentinel, and fails closed. +func TestCheckDeadPidWithNoSentinelIsUnobservable(t *testing.T) { + dir := t.TempDir() + windowEnd := testNow.Add(5*time.Minute + checkGrace) + logPath := filepath.Join(dir, "log.jsonl") + w, err := NewWriter(logPath, newFakeClock(testNow)) + if err != nil { + t.Fatalf("NewWriter: %v", err) + } + if err := w.WriteHeader(Header{ + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{ + UID: checkUID, Title: checkTitle, Folder: "F", Group: "G", + IntervalSeconds: 60, NoDataState: "OK", ExecErrState: "OK", + PollEverySeconds: checkPollEvery.Seconds(), + }}, + }); err != nil { + t.Fatalf("WriteHeader: %v", err) + } + // Healthy heartbeats all the way past windowEnd — evaluatedThrough is + // satisfied, so the drain wait needs no live re-poll — but no sentinel is + // ever written: the recorder died before it could call Stop. + for at := testNow.Add(-time.Minute); !at.After(windowEnd.Add(30 * time.Second)); at = at.Add(checkPollEvery) { + if err := w.WritePoll(Poll{RuleUID: checkUID, GrafanaNow: at, Found: true, State: "inactive", Health: "ok", LastEvaluation: at}); err != nil { + t.Fatalf("WritePoll: %v", err) + } + } + if err := w.Close(); err != nil { // no sentinel — a clean exit would call Stop + t.Fatalf("Close: %v", err) + } + writePid(t, logPath+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), logPath) + res, err := check(context.Background(), cfg, newCheckSource(nil)) + if err == nil { + t.Fatalf("check() = nil, want an error: no sentinel means the recorder never proved it ran to the end") + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want one unobservable verdict", res.Verdicts) + } +} + +// §22.5: "an incomplete last line gives exit 2" is otherwise proven only +// indirectly — log_test.go's TestReadLogRejectsBadLogs pins ReadLog's own +// error, and TestExitCode pins that any non-nil error maps to exit 2 — but +// nothing feeds a genuinely truncated log through check() itself. This closes +// that seam: a raw file with a valid header and poll, then a torn JSON tail, +// exactly what a recorder killed mid-write leaves behind. +func TestCheckRecorderModeTruncatedLogFailsClosed(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "log.jsonl") + + h := Header{ + SchemaVersion: LogSchemaVersion, + URL: "https://grafana.example.com", GrafanaVersion: "13.1.0", StartedAt: testNow.Add(-time.Minute), + Rules: []LoggedRule{{UID: checkUID, Title: checkTitle, IntervalSeconds: 60, PollEverySeconds: checkPollEvery.Seconds()}}, + } + hb, err := json.Marshal(headerRecord{Type: RecordHeader, Header: h}) + if err != nil { + t.Fatalf("marshal header: %v", err) + } + pb, err := json.Marshal(pollRecord{Type: RecordPoll, Poll: Poll{ + RuleUID: checkUID, GrafanaNow: testNow, Found: true, State: "inactive", Health: "ok", LastEvaluation: testNow, + }}) + if err != nil { + t.Fatalf("marshal poll: %v", err) + } + content := string(hb) + "\n" + string(pb) + "\n" + `{"type":"poll","rule_ui` // torn mid-write + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write log: %v", err) + } + writePid(t, path+".pid", fmt.Sprintf("%d\n", deadPid(t))) + + cfg := recorderConfig(t, newVirtualClock(testNow), path) + if _, err := check(context.Background(), cfg, newCheckSource(nil)); err == nil || !strings.Contains(err.Error(), "unparseable") { + t.Fatalf("check() = %v, want a refusal naming the unparseable tail", err) + } +} + // P5's "two authorities", from check's side: maxGap comes from the cadence the // header records, never from a re-derivation off intervalSeconds. The // fail-open direction is the one asserted — a log recorded at 5s on a 60s rule diff --git a/grafana-alertcheck/internal/gate/classify_test.go b/grafana-alertcheck/internal/gate/classify_test.go index 25d766b6c..6522f6e38 100644 --- a/grafana-alertcheck/internal/gate/classify_test.go +++ b/grafana-alertcheck/internal/gate/classify_test.go @@ -134,6 +134,33 @@ func TestClassifyRule_PreexistingThatRecoversIsRecoveredAndNotAViolation(t *test } } +// §22.2's "late condition": bad for 58 of a 60-minute window, clear at +// minute 58, still passes with a large BadFor — never a fail against some +// derived deadline (e.g. "must clear before 90% of the window"). +func TestClassifyRule_LateRecoveryPassesRegardlessOfHowLateItIs(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(60 * time.Minute) + clearAt := from.Add(58 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeRecovered { + t.Fatalf("outcome = %v, want recovered even 58 minutes into a 60-minute window", outcome) + } + if want := clearAt.Sub(from); badFor != want { + t.Fatalf("badFor = %v, want the full %v bad duration, not a value clamped against a deadline", badFor, want) + } + if len(viols) != 0 { + t.Fatalf("viols = %+v, want none: there is no deadline a preexisting recovery must beat", viols) + } +} + func TestClassifyRule_PreexistingStillBadAtWindowEndIsPersistentlyBad(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(10 * time.Minute) @@ -178,6 +205,45 @@ func TestClassifyRule_ClearThenBadAgainIsFlapping(t *testing.T) { } } +// §22.2: "a clear and then a second bad state gives flapping, at each +// possible time of the second bad state." A table over where the second +// onset lands — immediately after the clear, mid-window, and right at the +// last instant before windowEnd — closes the boundary this single fixed +// timing above cannot. +func TestClassifyRule_FlappingAtEveryTimingOfTheSecondOnset(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + clearAt := from.Add(2 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + tests := []struct { + name string + secondOnset time.Time + }{ + {"immediately after the clear", clearAt.Add(time.Second)}, + {"mid-window", from.Add(5 * time.Minute)}, + {"the last instant before windowEnd", to.Add(-time.Second)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from), + clearedPoll("r1", clearAt, key), + abnormalPoll("r1", tc.secondOnset, StateFiring, lbl("a"), tc.secondOnset), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeFlapping { + t.Fatalf("outcome = %v, want flapping for a second onset at %s", outcome, tc.secondOnset) + } + if len(viols) != 1 || viols[0].Outcome != OutcomeFlapping { + t.Fatalf("viols = %+v, want one flapping violation", viols) + } + }) + } +} + // --- H2: vanished is a discontinuity, never a clear --- func TestClassifyRule_VanishedWhileBadStaysPersistentlyBad(t *testing.T) { @@ -409,6 +475,148 @@ func TestDecide_UnobservableWinsEvenAlongsideARealViolation(t *testing.T) { } } +// §22.10: "a clean verdict with a coverage gap ... must never give exit 0", +// and "a recovered verdict and a skipped verdict also need proved coverage +// of the full window." One genuinely unobservable rule ("broken", zero +// polls) alongside a rule with each of the three favorable outcomes — none +// of them may waive the run. +func TestDecide_UnobservableRuleWinsOverEveryFavorableOutcome(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + tests := []struct { + name string + goodPolls []Poll + pausedAtStart bool + wantOutcome Outcome + }{ + { + name: "clean", + goodPolls: func() []Poll { + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("good", ts)) + } + return polls + }(), + wantOutcome: OutcomeClean, + }, + { + // Dense 30s-spaced polls throughout, so "good"'s own coverage + // proves clean on its own — a sparse abnormal/cleared/quiet + // triple (enough for classifyRule alone) would leave a + // heartbeat gap that muddies which rule made the run fail. + name: "recovered", + goodPolls: func() []Poll { + var polls []Poll + clearAt := from.Add(3 * time.Minute) + key := instanceKey(lbl("a")) + cleared := false + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + switch { + case ts.Equal(clearAt): + polls = append(polls, clearedPoll("good", ts, key)) + cleared = true + case !cleared: + polls = append(polls, abnormalPoll("good", ts, StateFiring, lbl("a"), from.Add(-time.Hour))) + default: + polls = append(polls, quietPoll("good", ts)) + } + } + return polls + }(), + wantOutcome: OutcomeRecovered, + }, + { + name: "skipped", + pausedAtStart: true, + wantOutcome: OutcomeSkipped, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + defs := []Definition{{UID: "good", Title: "Good"}, {UID: "broken", Title: "Broken"}} + rt := map[string]ruleTimings{ + "good": newRuleTimings(30*time.Second, 60), + "broken": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to} + + h := Header{StartedAt: from.Add(-time.Hour)} + if tc.pausedAtStart { + h.Rules = []LoggedRule{{UID: "good", IsPaused: true}} + } + + // "broken" gets no polls at all: no sentinel-worthy heartbeats, + // so it is unobservable regardless of "good". + sentinel := to + res, err := decide(h, tc.goodPolls, &sentinel, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want non-nil: 'broken' is unobservable regardless of 'good' being %s", tc.name) + } + var gotGood, gotBroken Outcome + for _, v := range res.Verdicts { + switch v.RuleUID { + case "good": + gotGood = v.Outcome + case "broken": + gotBroken = v.Outcome + } + } + if gotGood != tc.wantOutcome { + t.Errorf("good.Outcome = %v, want %v", gotGood, tc.wantOutcome) + } + if gotBroken != OutcomeUnobservable { + t.Errorf("broken.Outcome = %v, want unobservable", gotBroken) + } + }) + } +} + +// §22.10: the table above puts the coverage gap on a DIFFERENT rule from the +// one with the favorable outcome. This pins the tighter claim: a rule that +// itself recovers, but ALSO itself has a coverage gap, is still overridden to +// unobservable — the favorable classification of a rule is never a reason to +// skip that same rule's own coverage check. +func TestDecide_RecoveredOutcomeOverriddenByItsOwnCoverageGap(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} // maxGap = 60s + gt := globalTimings{} + pol := Policy{From: from, To: to} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", from.Add(30*time.Second), key), + } + for ts := from.Add(time.Minute); !ts.After(to); ts = ts.Add(30 * time.Second) { + // A gap from from+1.5m to from+4m — well past the 60s maxGap — + // sitting entirely AFTER the clear, so classifyRule alone would + // still call this rule `recovered`. + if ts.After(from.Add(90*time.Second)) && ts.Before(from.Add(4*time.Minute)) { + continue + } + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want non-nil: r1's own coverage gap must fail the run even though it recovered") + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want unobservable, never recovered", res.Verdicts) + } + if cov := res.Coverage["r1"]; cov.Proved { + t.Fatalf("Coverage = %+v, want not proved", cov) + } +} + func TestDecide_CleanWindowIsAPass(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(10 * time.Minute) @@ -436,6 +644,55 @@ func TestDecide_CleanWindowIsAPass(t *testing.T) { } } +// §22.7's second of the plan's "if only three tests could exist" cases: a +// pause and then an unpause inside the window, with an episode that would +// fire and resolve entirely inside the blind interval. A drain wait alone — +// "did the rule eventually evaluate through windowEnd?" — would see +// lastEvaluation catch up after the unpause and answer yes, a pass. decide() +// never runs a drain wait (that is check.go's I/O concern, §14.6); this pins +// that proveCoverage's own per-poll checks already refuse the window without +// one, so a live drain wait is not what is saving this case. +func TestDecide_PauseThenUnpauseWithHiddenEpisodeGivesUnobservableNotClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(20 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + pauseStart := from.Add(5 * time.Minute) + pauseEnd := from.Add(10 * time.Minute) + + var polls []Poll + for ts := from; !ts.After(pauseStart.Add(-30 * time.Second)); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + for ts := pauseStart; !ts.After(pauseEnd); ts = ts.Add(30 * time.Second) { + // No fire/resolve is ever observed here: the rule was not + // evaluating, so any real episode inside this stretch is invisible + // to every poll (§14.7). + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "ok", IsPaused: true, LastEvaluation: pauseStart}) + } + for ts := pauseEnd.Add(30 * time.Second); !ts.After(to); ts = ts.Add(30 * time.Second) { + // Evaluations resume and catch straight up — a drain wait's final + // "did it reach windowEnd" question would answer yes. + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want the pause-then-unpause blind interval to fail closed") + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want unobservable, never clean", res.Verdicts) + } + if cov := res.Coverage["r1"]; cov.Proved { + t.Fatalf("Coverage = %+v, want not proved", cov) + } +} + // --- MinObserved shortfall (§12) --- func TestDecide_SkippedOnlyShortfallProducesAViolationWithoutAnError(t *testing.T) { @@ -808,6 +1065,34 @@ func TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean(t *testing.T) { } if len(viols) != 1 { t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) + + } +} + +// §22.2: "a clear after `to` gives persistently_bad." classifyRule filters +// its input to [from, windowEnd] itself (inWindowPolls), so a Cleared event +// GENUINELY past windowEnd — well beyond any skew bound, unlike the clamp +// case above — never reaches the timeline at all: the instance is still bad +// at windowEnd as far as this window is concerned. +func TestClassifyRule_ClearAfterWindowEndIsPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", to.Add(time.Hour), key), // far past `to`, not a boundary case + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad: a clear outside the window must not read as a recovery", outcome) + } + if badFor != to.Sub(from) { + t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) + } + if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { + t.Fatalf("viols = %+v, want one persistently_bad violation", viols) } } diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go index 59a5dfb0f..4eac5eac1 100644 --- a/grafana-alertcheck/internal/gate/coverage.go +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -231,10 +231,21 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d fail(ReasonRuleAbsent, fmt.Sprintf("state endpoint returned no rule on %d poll(s), first at %s", absentCount, absentAt.Format(time.RFC3339))) } - // Check 9 — KeepLast (§10.2). A note, never fatal. It surfaces only as an - // instance Reason after P1.2a's parsing, and Reasons keys can be - // comma-joined composites, so membership (reasonsContain) is required — - // indexing "KeepLast" directly would miss "KeepLast, MissingSeries". + // Check 9 — KeepLast (§10.2). Two distinct notes, both non-fatal: + // + // DECLARED: the rule's own no_data_state/exec_err_state is configured as + // KeepLast — a standing blind spot (§10.2's "unclear condition") whether + // or not it is ever exercised during this particular window. This reads + // def, not polls, so it fires exactly once regardless of poll content. + if def.NoDataState == keepLastReason || def.ExecErrState == keepLastReason { + res.Notes = append(res.Notes, fmt.Sprintf( + "rule %q: configured with no_data_state/exec_err_state=KeepLast — a stale state can continue past a real fault (§10.2)", def.Title)) + } + // OBSERVED: an instance actually reported the KeepLast reason during the + // window. It surfaces only as an instance Reason after P1.2a's parsing, + // and Reasons keys can be comma-joined composites, so membership + // (reasonsContain) is required — indexing "KeepLast" directly would miss + // "KeepLast, MissingSeries". for _, p := range inWindow { if reasonsContain(p.Reasons, keepLastReason) { res.Notes = append(res.Notes, fmt.Sprintf( diff --git a/grafana-alertcheck/internal/gate/coverage_test.go b/grafana-alertcheck/internal/gate/coverage_test.go index 7ef4b5427..9dbba6594 100644 --- a/grafana-alertcheck/internal/gate/coverage_test.go +++ b/grafana-alertcheck/internal/gate/coverage_test.go @@ -71,6 +71,22 @@ func TestProveCoverage_SentinelBeforeGraceIsUnobservable(t *testing.T) { if res.Reason != ReasonSentinelEarly { t.Fatalf("Reason = %q, want sentinel_early", res.Reason) } + if !res.Unobservable || res.Proved { + t.Fatalf("res = %+v, want Unobservable and not Proved — a reason string with no consequence is not a coverage failure", res) + } + + // The consequence: decide() must turn this into exit 2, never a pass. + defs := []Definition{def} + drt := map[string]ruleTimings{def.UID: rt} + gt := globalTimings{transitionGrace: grace} + pol := Policy{From: from, To: to} + dres, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, &sentinel, defs, drt, gt, pol) + if err == nil { + t.Fatalf("decide() err = nil, want non-nil: a sentinel short of to+grace must fail the run") + } + if len(dres.Verdicts) != 1 || dres.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want one unobservable verdict", dres.Verdicts) + } } func TestProveCoverage_SentinelExactlyAtGraceIsFine(t *testing.T) { @@ -107,6 +123,22 @@ func TestProveCoverage_FromBeforeRecordIsUnobservable(t *testing.T) { if res.Reason != ReasonFromBeforeRecord { t.Fatalf("Reason = %q, want from_before_record", res.Reason) } + if !res.Unobservable || res.Proved { + t.Fatalf("res = %+v, want Unobservable and not Proved — a reason string with no consequence is not a coverage failure", res) + } + + // The consequence: decide() must turn this into exit 2, never a pass. + defs := []Definition{def} + drt := map[string]ruleTimings{def.UID: rt} + gt := globalTimings{} + pol := Policy{From: from, To: to} + dres, err := decide(Header{StartedAt: started}, nil, &sentinel, defs, drt, gt, pol) + if err == nil { + t.Fatalf("decide() err = nil, want non-nil: `from` before the recording started must fail the run") + } + if len(dres.Verdicts) != 1 || dres.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want one unobservable verdict", dres.Verdicts) + } } // --- Check 3: heartbeat continuity (§6) --- @@ -388,7 +420,7 @@ func denseHealthyPolls(uid string, from, to time.Time, every time.Duration) []Po // --- Check 9: KeepLast (§10.2) --- -func TestProveCoverage_KeepLastIsNoteOnly(t *testing.T) { +func TestProveCoverage_KeepLastObservedIsNoteOnly(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(10 * time.Minute) rt := newRuleTimings(30*time.Second, 60) @@ -414,6 +446,40 @@ func TestProveCoverage_KeepLastIsNoteOnly(t *testing.T) { } } +// §22.2/§10.2: "KeepLast in the configuration gives a note" — a DIFFERENT +// claim from the observed-reason test above. A rule DECLARED with +// no_data_state or exec_err_state = KeepLast is a standing blind spot +// whether or not any poll ever actually reports the reason, so the note +// must fire off the definition alone, over an otherwise perfectly healthy +// window with zero KeepLast reasons anywhere in it. +func TestProveCoverage_KeepLastConfiguredIsNoteOnly(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + rt := newRuleTimings(30*time.Second, 60) + + tests := []struct { + name string + def Definition + }{ + {"no_data_state", Definition{UID: "r1", Title: "R1", NoDataState: "KeepLast"}}, + {"exec_err_state", Definition{UID: "r1", Title: "R1", ExecErrState: "KeepLast"}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + polls := denseHealthyPolls("r1", from, to, 30*time.Second) + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, tc.def, from, to, 0) + if !res.Proved { + t.Fatalf("Proved = false, want true: a declared KeepLast is a note, never fatal: %+v", res) + } + if !anyContains(res.Notes, "KeepLast") { + t.Fatalf("Notes = %v, want a KeepLast note from the definition alone, with zero KeepLast reasons observed", res.Notes) + } + }) + } +} + // --- Clock domains (§16) --- // TestProveCoverage_SkewTranslationAtWindowBoundary pins §16's "Clock diff --git a/grafana-alertcheck/internal/gate/parse_state_test.go b/grafana-alertcheck/internal/gate/parse_state_test.go index 7cf336526..1ab6c620d 100644 --- a/grafana-alertcheck/internal/gate/parse_state_test.go +++ b/grafana-alertcheck/internal/gate/parse_state_test.go @@ -283,6 +283,59 @@ func TestInstanceKey(t *testing.T) { } } +// minimalStateBody is the smallest H1-legal state response: one group, one +// rule, no optional keys at all, plus whatever extra is spliced in verbatim +// before the rule's closing brace — for isolating one optional key at a time +// rather than relying on a fixture that removes several together. +func minimalStateBody(extraRuleJSON string) []byte { + return fmt.Appendf(nil, + `{"status":"success","data":{"groups":[{"file":"F","name":"G","interval":60,`+ + `"rules":[{"uid":"r1","name":"R1","state":"inactive","health":"ok","isPaused":false,`+ + `"lastEvaluation":"2026-01-01T00:00:00Z"%s}]}]}}`, extraRuleJSON) +} + +// §22.2: keepFiringFor is named alongside alerts/totals/labels as an optional +// key (§3.1), but state_missing_optional.json removes it together with +// everything else — never in isolation, so a regression that made it +// required specifically would not be caught by that fixture alone. +func TestParseState_KeepFiringForIsOptional(t *testing.T) { + tests := []struct { + name string + extra string + }{ + {"present", `,"keepFiringFor":300`}, + {"absent", ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + rules, err := ParseState(minimalStateBody(tc.extra)) + if err != nil { + t.Fatalf("ParseState: %v", err) + } + if len(rules) != 1 { + t.Fatalf("rules = %+v, want one", rules) + } + }) + } +} + +// §22.2: labels is optional at the INSTANCE level (opt(m, "labels", ...) in +// parseInstance), distinct from the rule-level labels state_missing_optional.json +// already covers — an instance can exist with no labels of its own. +func TestParseState_InstanceWithoutLabelsParses(t *testing.T) { + body := minimalStateBody(`,"alerts":[{"state":"Normal","activeAt":"2026-01-01T00:00:00Z"}]`) + rules, err := ParseState(body) + if err != nil { + t.Fatalf("ParseState: %v", err) + } + if len(rules) != 1 || len(rules[0].Instances) != 1 { + t.Fatalf("rules = %+v, want one rule with one instance", rules) + } + if got := rules[0].Instances[0].Labels; len(got) != 0 { + t.Errorf("Instance.Labels = %v, want empty/nil", got) + } +} + // synthesizeHighCardinalityState builds a state response with a single rule // holding `alerting` Alerting instances and `normal` Normal instances, by // cloning the one real instance in state_one_instance.json. It is never diff --git a/grafana-alertcheck/internal/gate/resolve_test.go b/grafana-alertcheck/internal/gate/resolve_test.go index 77fd3bb5d..192ed3dc2 100644 --- a/grafana-alertcheck/internal/gate/resolve_test.go +++ b/grafana-alertcheck/internal/gate/resolve_test.go @@ -233,6 +233,26 @@ func TestResolve_CollapseByUIDGivesNoteNotError(t *testing.T) { } } +// §22.2: "the same rule with two identical names ... must collapse to one +// rule" — the literal exact-duplicate-string case, distinct from the +// different-spellings case above. +func TestResolve_IdenticalDuplicateNameCollapsesWithNote(t *testing.T) { + defs := rulerDefs(t) + resolved, notes, err := Resolve(defs, []string{ + "example_workflow_paused_rule", + "example_workflow_paused_rule", + }, "") + if err != nil { + t.Fatalf("Resolve: unexpected error: %v", err) + } + if len(resolved) != 1 || resolved[0].UID != "rule0000007" { + t.Fatalf("resolved = %+v, want exactly one rule0000007", resolved) + } + if len(notes) != 1 { + t.Fatalf("notes = %v, want exactly one collapse note", notes) + } +} + func TestResolve_MinObservedCountIsPostCollapse(t *testing.T) { defs := rulerDefs(t) names := []string{ diff --git a/grafana-alertcheck/internal/gate/schedule_test.go b/grafana-alertcheck/internal/gate/schedule_test.go index 1a992f8bd..4e4023152 100644 --- a/grafana-alertcheck/internal/gate/schedule_test.go +++ b/grafana-alertcheck/internal/gate/schedule_test.go @@ -69,6 +69,29 @@ func TestDeriveTimings_TransitionGraceExcludesSkippedRule(t *testing.T) { } } +// §22.2/§22.3: `for: 1d` and `for: 1w` parse correctly (parse_ruler_test.go), +// but that alone never proves they flow into transitionGrace — a Prometheus +// duration parser that silently truncated to time.Duration's other units, or +// a transitionGrace derivation that only ever saw hand-built values, could +// each pass every existing test and still be wrong together. This drives the +// real ruler_rules.json fixture (rule0000010, for:1w, DERIVED to exercise the +// w unit — testdata/README.md) through ParseDefinitions and DeriveTimings. +func TestDeriveTimings_RealForOneWeekRuleSetsTransitionGrace(t *testing.T) { + defs := rulerDefs(t) + _, global, notes := DeriveTimings(defs, 0) + if len(notes) != 0 { + t.Fatalf("notes = %v, want none: no --poll-interval override is given, so no override note should fire", notes) + } + + want := 7*24*time.Hour + 60*time.Second // rule0000010: for=1w, intervalSeconds=60 + if global.transitionGrace != want { + t.Fatalf("transitionGrace = %s, want %s (rule0000010's for:1w plus its interval)", global.transitionGrace, want) + } + if !strings.Contains(global.graceSource, "Example Failure Ratio Above 10 Percent Weekly") { + t.Errorf("graceSource = %q, want it to name rule0000010", global.graceSource) + } +} + func TestDeriveTimings_TransitionGraceZeroWhenAllSkipped(t *testing.T) { defs := []Definition{{UID: "r1", Title: "R1", IntervalSeconds: 60, For: time.Hour, IsPaused: true}} _, global, _ := DeriveTimings(defs, 0) @@ -395,6 +418,34 @@ func TestStartupSummary_WarningWhenGraceTooLarge(t *testing.T) { } } +// §22.3: "a rule with for: 15m in a 10-minute window gives the warning about +// a large grace period" — no such rule exists in the real capture +// (testdata/README.md), so the test above pins the mechanism with a +// hand-built globalTimings. This drives the same warning off the real +// ruler_rules.json fixture's for:1w rule instead, tying ParseDefinitions and +// DeriveTimings into the warning end to end, not just the warning formula in +// isolation. +func TestStartupSummary_RealForOneWeekRuleTriggersWarning(t *testing.T) { + defs := rulerDefs(t) + _, global, notes := DeriveTimings(defs, 0) + if len(notes) != 0 { + t.Fatalf("notes = %v, want none: no --poll-interval override is given, so no override note should fire", notes) + } + + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) // transitionGrace (>1w) dwarfs 1/4 of this window + summary, warning := StartupSummary(from, to, global) + if !strings.Contains(summary, "planned run time") { + t.Errorf("summary = %q, want it to name the planned run time", summary) + } + if warning == "" { + t.Fatal("warning = \"\", want one: a real for:1w rule's transitionGrace vastly exceeds 1/4 of a 10m window") + } + if !strings.Contains(warning, "Example Failure Ratio Above 10 Percent Weekly") { + t.Errorf("warning = %q, want it to name rule0000010", warning) + } +} + func TestStartupSummary_NoWarningWhenGraceSmall(t *testing.T) { from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) to := from.Add(time.Hour) diff --git a/grafana-alertcheck/internal/gate/source_test.go b/grafana-alertcheck/internal/gate/source_test.go index d25f1f2ec..2a4b9dc71 100644 --- a/grafana-alertcheck/internal/gate/source_test.go +++ b/grafana-alertcheck/internal/gate/source_test.go @@ -349,6 +349,59 @@ func TestHTTPSource_ObservationTiming(t *testing.T) { } } +// §22.7/§16: a genuinely discriminating regression for "the gate compares +// staleness against the Date header, never the runner's clock." lastEvaluation +// sits 100s behind Grafana's TRUE now (obs.GrafanaNow, from the Date header) +// — under the 120s evalStaleAfter limit — but 130s behind the RUNNER's clock. +// An implementation that leaked the runner's clock into the staleness +// comparison, instead of the Date header, would report a false violation +// here; coverage_test.go's TestProveCoverage_SkewTranslationAtWindowBoundary +// cannot catch that, because it sets LastEvaluation equal to GrafanaNow on +// every poll, making staleness zero regardless of which clock is used. +func TestHTTPSourceStalenessNeverFalsePositiveUnderSkew(t *testing.T) { + runnerNow := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + clock := newFakeClock(runnerNow) + const skew = 30 * time.Second // the runner's clock reads 30s ahead of Grafana's + serverDate := runnerNow.Add(-skew) + lastEval := serverDate.Add(-100 * time.Second) + + def := Definition{UID: "r1", Title: "Rule One"} + srv := rawHTTPServer(t, func(r *http.Request) []byte { + body := fmt.Sprintf(`{"status":"success","data":{"groups":[{"file":"F","name":"G","interval":60,"rules":[`+ + `{"uid":%q,"name":%q,"state":"inactive","health":"ok","isPaused":false,"lastEvaluation":%q}`+ + `]}]}}`, def.UID, def.Title, lastEval.UTC().Format(time.RFC3339)) + return rawResponse(200, "OK", map[string]string{ + "Content-Type": "application/json", + "Date": serverDate.UTC().Format(http.TimeFormat), + }, body) + }) + + src := NewHTTPSource(srv.URL, "", clock) + obs, err := src.RuleState(context.Background(), def.Title) + if err != nil { + t.Fatalf("RuleState(): %v", err) + } + if !obs.GrafanaNow.Equal(serverDate) { + t.Fatalf("GrafanaNow = %s, want the Date header %s, never the runner's clock %s", obs.GrafanaNow, serverDate, runnerNow) + } + if len(obs.Rules) != 1 { + t.Fatalf("Rules = %+v, want exactly one", obs.Rules) + } + + rt := newRuleTimings(30*time.Second, 60) // evalStaleAfter = 120s + from := serverDate.Add(-10 * time.Minute) + to := serverDate + polls := denseHealthyPolls(def.UID, from, to, 30*time.Second) + polls[len(polls)-1].LastEvaluation = obs.Rules[0].LastEvaluation // the real, HTTP-sourced value + sentinel := to + + res := proveCoverage(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, rt, def, from, to, 0) + if res.Unobservable { + t.Fatalf("Coverage = %+v, want no violation: 100s behind Grafana's TRUE now is under the 120s limit — "+ + "only a runner-clock leak (skewed +30s here) would push this over", res) + } +} + func TestHTTPSource_Retry_TransientRecovers(t *testing.T) { var mu sync.Mutex calls := 0 diff --git a/grafana-alertcheck/internal/gate/watch_test.go b/grafana-alertcheck/internal/gate/watch_test.go index cecd3a45e..85d9f5185 100644 --- a/grafana-alertcheck/internal/gate/watch_test.go +++ b/grafana-alertcheck/internal/gate/watch_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "maps" "os" "path/filepath" "slices" @@ -429,6 +430,12 @@ func TestPrepareWatchDoesNotWaitForPausedRules(t *testing.T) { if !polls[0].Found || !polls[0].GrafanaNow.Equal(testNow) { t.Errorf("first poll = %+v, want a found observation at %s", polls[0], testNow) } + // §22.3: "the poll record holds the state histogram. Assert that watch + // writes it" — through a real prepareWatch()/Reducer call, not just + // log_test.go's hand-built Writer/ReadLog round trip. + if want := map[string]int{"normal": 1}; !maps.Equal(polls[0].Histogram, want) { + t.Errorf("Histogram = %v, want %v: watch must record the state histogram on every poll it writes", polls[0].Histogram, want) + } if !strings.Contains(notes.String(), watchPausedTitle) || !strings.Contains(notes.String(), "paused") { t.Errorf("notes do not mention the paused rule:\n%s", notes.String()) }