diff --git a/grafana-alertcheck/internal/gate/classify.go b/grafana-alertcheck/internal/gate/classify.go new file mode 100644 index 000000000..11ddc8ba2 --- /dev/null +++ b/grafana-alertcheck/internal/gate/classify.go @@ -0,0 +1,601 @@ +package gate + +import ( + "fmt" + "slices" + "strings" + "time" +) + +// ReasonNodata is decide's own unobservable reason (§10.1/§10.2): proveCoverage +// (P7) deliberately never sets it — health=nodata is a note there, never fatal, +// because escalating it needs Policy.NodataIsUnobservable, and the pure +// coverage layer has no Policy to consult (coverage.go, check 5). decide is +// the seam that DOES have a Policy, so the escalation lives here. +const ReasonNodata UnobservableReason = "nodata" + +// Outcome is the verdict of one instance's timeline, and — after decide takes +// the worst across a rule's instances — of the rule itself (§9). It is a +// published JSON output (§19.0): the three fail values stay distinct even +// though v1 maps all three to exit 1, because a later reason string cannot +// recover the information a single "fail" value would have thrown away, and +// because splitting them later would break a published interface for no gain. +type Outcome string + +const ( + OutcomeClean Outcome = "clean" + OutcomeNewlyBad Outcome = "newly_bad" + OutcomeRecovered Outcome = "recovered" + OutcomePersistentlyBad Outcome = "persistently_bad" + OutcomeFlapping Outcome = "flapping" + OutcomeSkipped Outcome = "skipped" + OutcomeUnobservable Outcome = "unobservable" +) + +// PreexistingPolicy governs only the ONE ambiguous case in the outcome table: +// an instance that was already bad when the window opened. A newly_bad or +// flapping instance is a fail under every policy (§11.3) — the plan lists +// them among the outcomes that "do not change" — so this type only ever +// changes how `recovered` and `persistently_bad` are judged (isViolation +// below). +type PreexistingPolicy string + +const ( + // PreexistingFailUnlessRecovered is the default (§11.7): a preexisting + // instance that clears and stays clear is a pass (`recovered`); one that + // never clears is still a fail (`persistently_bad`). + PreexistingFailUnlessRecovered PreexistingPolicy = "fail-unless-recovered" + // PreexistingFail makes ANY preexisting instance a fail, even one that + // recovers — for a user who wants no benefit of the doubt for a + // condition this release did not cause. + PreexistingFail PreexistingPolicy = "fail" + // PreexistingIgnore disregards a preexisting instance entirely, whether + // it recovers or stays bad for the whole window: only a genuinely NEW + // bad episode (newly_bad or flapping) can fail the rule. + PreexistingIgnore PreexistingPolicy = "ignore" +) + +// Violation is one instance whose timeline outcome counts against the run, +// after the preexisting policy has been applied (isViolation below). +type Violation struct { + Alert, RuleUID string + Outcome Outcome + State State + Health string // raw, reporting-only, like Poll.Health (P1.2a) + LastError string + // FirstSeen is the episode's onset, in the runner domain (§16): activeAt + // translated by its poll's own skew when the episode opened strictly + // inside the window, or `from` itself when the instance was already bad + // at window-open (preexisting) — never a raw, untranslated Grafana + // timestamp. + FirstSeen time.Time + // ClearedAt is zero unless the episode closed via a genuine Cleared + // event, also translated to the runner domain. + ClearedAt time.Time + InstanceLabels map[string]string + // Note carries an explanation for a Violation that has no instance + // behind it — the synthetic MinObserved shortfall entry decide emits + // when the deficit exceeds what any named paused rule explains (§12). + // LastError is reporting-only rule state from a real poll and must not + // double as a message field for a Violation that never touched one. + Note string +} + +// RuleVerdict is one rule's worst-of outcome (§9), always present for every +// resolved rule — Verdicts includes the passes, not only the failures — so a +// human reading the table sees every alert that was asked for, not only the +// ones that misbehaved. +type RuleVerdict struct { + Alert, RuleUID string + Outcome Outcome + BadFor time.Duration // total wall-clock time any instance was bad inside the window, overlaps merged + PollEvery time.Duration + Note string +} + +// Policy is decide's narrowed, pure-layer view of Config/Cfg (§9's P9 +// comment): the classification knobs and the window, nothing else. No URL, +// no token, no I/O handles — those never reach the pure layer. +type Policy struct { + States []State + Preexisting PreexistingPolicy + MinObserved int + AllowPaused, NodataIsUnobservable bool + From, To time.Time +} + +// Result is decide's whole answer: everything §20.2's table and the action's +// JSON outputs need. Coverage carries one CoverageResult per non-skipped +// rule — no separate Interval type anywhere in the project (§2's +// simplification table). +type Result struct { + From, To time.Time + GrafanaVersion string + ClockSkew time.Duration // the largest |skew| across every poll decide was given, not only the ones a rule's window actually used + Coverage map[string]CoverageResult + Verdicts []RuleVerdict + Violations []Violation +} + +// episode is one contiguous, policy-bad span of one instance's timeline, +// already resolved to the runner domain and clamped to [from, windowEnd]. It +// never crosses a genuine Cleared event (H2): a Vanished marker freezes the +// state instead of closing the episode, which is what keeps a vanish from +// ever reading as a recovery. +type episode struct { + start, end time.Time + closedByRealClear bool +} + +// instanceTimeline accumulates one instance's walk across a rule's in-window +// polls. preexisting is decided once, the first time this key is seen bad: +// by the translated ActiveAt against `from` (§16), never by which poll +// happened to report it first — a poll's own cadence is not evidence of when +// the condition actually began (F1/F2). +type instanceTimeline struct { + labels map[string]string + preexisting bool + seen bool + badOpen bool + episodeStart time.Time + lastState State + lastHealth string + lastError string + episodes []episode +} + +// runnerTime translates a Grafana-domain timestamp recorded on poll p into +// the runner domain, undoing that poll's own measured skew (§16). GrafanaNow +// and ActiveAt come from the same response, so the same poll's skew applies +// to both. This is the single implementation of that translation for the +// package (same drift argument as pollsForRule, F5): coverage.go's window +// membership test and heartbeat boundary segments call it too, rather than +// each keeping its own copy of `p.GrafanaNow.Add(-p.Skew())` that could +// silently diverge from this one. +func runnerTime(p Poll, grafanaDomain time.Time) time.Time { + return grafanaDomain.Add(-p.Skew()) +} + +// classifyRule builds every instance timeline for one rule across +// [from, windowEnd] and reduces them to the rule's worst outcome (§9), its +// merged BadFor, and the Violations the preexisting policy actually charges +// against the run. It is PURE: no I/O, no clock reads (§2) — decide supplies +// windowEnd (to + transitionGrace) rather than this function deriving it, so +// a test can pin the boundary directly. +// +// polls need not be pre-filtered to this rule, matching proveCoverage's own +// contract (§14.5): selection is by def.UID. +func classifyRule(def Definition, polls []Poll, from, windowEnd time.Time, badStates map[State]bool, pol PreexistingPolicy) (Outcome, time.Duration, []Violation) { + rulePolls := pollsForRule(polls, def.UID) + inWindow := inWindowPolls(rulePolls, from, windowEnd) + + timelines := make(map[string]*instanceTimeline) + order := make([]string, 0) + + // get backfills labels the first time a real Instance is seen (F4): a key + // can be created earlier by a bare Cleared/Vanished marker, which carries + // no labels of its own, and the instance later re-firing must not report + // an empty InstanceLabels just because of which event happened to create + // the timeline first. + get := func(key string, labels map[string]string) *instanceTimeline { + tl, ok := timelines[key] + if !ok { + tl = &instanceTimeline{labels: labels} + timelines[key] = tl + order = append(order, key) + return tl + } + if tl.labels == nil && labels != nil { + tl.labels = labels + } + return tl + } + + openEpisode := func(tl *instanceTimeline, start time.Time) { + tl.badOpen = true + tl.episodeStart = start + } + closeEpisode := func(tl *instanceTimeline, end time.Time, real bool) { + // inWindowPolls admits a poll whose translated time is up to its own + // skew bound PAST windowEnd (the membership test widens the boundary + // outward, §16). Without this clamp a genuine Cleared event on such a + // poll would produce an episode.end slightly beyond windowEnd, + // contradicting the episode type's own "clamped to + // [from, windowEnd]" contract. + if end.After(windowEnd) { + end = windowEnd + } + // Different polls can carry different measured skews. In theory a + // closing poll's translated time could land before the opening + // poll's — skew is capped at skewHardLimit (60s), so this is remote, + // not impossible — and a negative span would feed mergeDurations a + // duration that subtracts instead of adds. Clamp rather than trust + // the arithmetic never to invert. + if end.Before(tl.episodeStart) { + end = tl.episodeStart + } + tl.episodes = append(tl.episodes, episode{start: tl.episodeStart, end: end, closedByRealClear: real}) + tl.badOpen = false + } + // onsetOf resolves a fresh episode's start: the instance's own ActiveAt, + // translated to the runner domain by this poll's skew, clamped so it + // never reads as starting before the window opened. + onsetOf := func(p Poll, inst Instance) time.Time { + start := runnerTime(p, inst.ActiveAt) + if start.Before(from) { + start = from + } + return start + } + + for _, p := range inWindow { + byKey := make(map[string]Instance, len(p.Abnormal)) + for _, inst := range p.Abnormal { + byKey[instanceKey(inst.Labels)] = inst + } + + for key, inst := range byKey { + tl := get(key, inst.Labels) + bad := badStates[inst.State] + switch { + case !tl.seen: + tl.seen = true + if bad { + // Fail-closed (§16): only call an onset "preexisting" + // when even the worst-case skew error still puts it at + // or before `from`. An onset that might really have + // landed just inside the window must classify as a new + // episode, never earn the `recovered` benefit of the + // doubt it would get if it later clears (F1/F2). + activeAtRunner := runnerTime(p, inst.ActiveAt) + tl.preexisting = !activeAtRunner.Add(p.SkewBound()).After(from) + if tl.preexisting { + openEpisode(tl, from) + } else { + openEpisode(tl, onsetOf(p, inst)) + } + } + case bad && !tl.badOpen: + openEpisode(tl, onsetOf(p, inst)) + case !bad && tl.badOpen: + closeEpisode(tl, runnerTime(p, p.GrafanaNow), true) + } + tl.lastState, tl.lastHealth, tl.lastError = inst.State, p.Health, p.LastError + } + + for _, key := range p.Cleared { + tl := get(key, nil) + if !tl.seen { + // Cleared on the very first mention means the transition + // happened between the poll just before this one (possibly + // pre-window) and this one: there is no window-internal + // evidence that it was ever bad, so it is neither + // preexisting nor a new episode. + tl.seen = true + continue + } + if tl.badOpen { + closeEpisode(tl, runnerTime(p, p.GrafanaNow), true) + } + tl.lastHealth, tl.lastError = p.Health, p.LastError + } + + // Vanished is a deliberate no-op (H2): freeze whatever badOpen/preexisting + // already holds. An instance that vanishes while bad must stay bad, and + // one that vanishes while never having been bad must stay uninteresting. + for _, key := range p.Vanished { + tl := get(key, nil) + tl.seen = true + tl.lastHealth = p.Health + } + } + + // Multiple instances can appear for the first time within the same poll, + // and map iteration order is nondeterministic; sort so this pure + // function's Violations/BadFor output is stable across runs given the + // same input, like log.go sorts Cleared/Vanished for the same reason. + slices.Sort(order) + + var ( + outcome Outcome = OutcomeClean + badFor []episode + viols []Violation + ) + + for _, key := range order { + tl := timelines[key] + if tl.badOpen { + closeEpisode(tl, windowEnd, false) + } + if len(tl.episodes) == 0 { + continue + } + + var instOutcome Outcome + switch { + case len(tl.episodes) > 1: + instOutcome = OutcomeFlapping + case tl.preexisting: + if tl.episodes[0].closedByRealClear { + instOutcome = OutcomeRecovered + } else { + instOutcome = OutcomePersistentlyBad + } + default: + // A genuinely new onset always fails, whether or not it later + // clears within the window (§11.4 point 3): only a PREEXISTING + // condition earns the benefit of `recovered`. + instOutcome = OutcomeNewlyBad + } + + if outcomeRank(instOutcome) > outcomeRank(outcome) { + outcome = instOutcome + } + badFor = append(badFor, tl.episodes...) + + if isViolation(instOutcome, pol) { + var clearedAt time.Time + last := tl.episodes[len(tl.episodes)-1] + if last.closedByRealClear { + clearedAt = last.end + } + viols = append(viols, Violation{ + Alert: def.Title, + RuleUID: def.UID, + Outcome: instOutcome, + State: tl.lastState, + Health: tl.lastHealth, + LastError: tl.lastError, + FirstSeen: tl.episodes[0].start, + ClearedAt: clearedAt, + InstanceLabels: tl.labels, + }) + } + } + + return outcome, mergeDurations(badFor), viols +} + +// isViolation decides whether one instance's outcome counts against the run, +// once the preexisting policy is applied. newly_bad and flapping always do +// (§11.3): both contain a genuinely new bad episode, so no policy forgives +// them. recovered and persistently_bad are, by classifyRule's construction, +// ALWAYS preexisting (a non-preexisting single episode is newly_bad instead, +// regardless of whether it clears) — so these are the only two policy can +// change, and isViolation needs no separate preexisting flag to know that. +func isViolation(o Outcome, pol PreexistingPolicy) bool { + switch o { + case OutcomeNewlyBad, OutcomeFlapping: + return true + case OutcomePersistentlyBad: + return pol != PreexistingIgnore + case OutcomeRecovered: + return pol == PreexistingFail + default: + return false + } +} + +// outcomeRank orders outcomes for classifyRule's worst-of reduction across a +// rule's instances (§9). The three fail values, and recovered above clean, +// give it exactly the ordering the table requires — +// "unobservable > {flapping, persistently_bad, newly_bad} > recovered > +// skipped > clean" — with unobservable and skipped applied outside this +// function (decide owns both: unobservable from CoverageResult, skipped from +// Definition.IsPaused). The table does not distinguish among the three fail +// values, so their relative order here (flapping above persistently_bad +// above newly_bad) is an arbitrary but fixed and documented tie-break, not a +// claim that one is worse than another. +func outcomeRank(o Outcome) int { + switch o { + case OutcomeFlapping: + return 4 + case OutcomePersistentlyBad: + return 3 + case OutcomeNewlyBad: + return 2 + case OutcomeRecovered: + return 1 + default: // OutcomeClean + return 0 + } +} + +// mergeDurations sums the wall-clock time covered by a set of episodes, +// merging overlaps so a rule with several simultaneously-bad instances is +// not reported as bad for longer than it actually was. +func mergeDurations(eps []episode) time.Duration { + if len(eps) == 0 { + return 0 + } + sorted := slices.Clone(eps) + slices.SortStableFunc(sorted, func(a, b episode) int { return a.start.Compare(b.start) }) + + var total time.Duration + cur := sorted[0] + for _, e := range sorted[1:] { + if e.start.After(cur.end) { + total += cur.end.Sub(cur.start) + cur = e + continue + } + if e.end.After(cur.end) { + cur.end = e.end + } + } + total += cur.end.Sub(cur.start) + return total +} + +// pollsForRule filters polls to one rule and sorts them by GrafanaNow, the +// same selection proveCoverage uses (§14.5: selection is by UID, never by +// title) — stable, because two polls sharing a coarse Date header must not +// reorder nondeterministically in a pure function. This is the single +// filter+sort implementation for the package (F5): proveCoverage calls it +// too, rather than keeping its own copy that could silently drift from this +// one's membership test. +func pollsForRule(polls []Poll, uid string) []Poll { + var out []Poll + for _, p := range polls { + if p.RuleUID == uid { + out = append(out, p) + } + } + slices.SortStableFunc(out, func(a, b Poll) int { return a.GrafanaNow.Compare(b.GrafanaNow) }) + return out +} + +// badStateSet turns Policy.States into a lookup set, defaulting to {firing} +// (§13) when the caller leaves States empty — decide applies the default +// itself so a test can pass a zero-value Policy and get v1's real default, +// rather than relying on a CLI layer that does not exist yet. +func badStateSet(states []State) map[State]bool { + if len(states) == 0 { + states = []State{StateFiring} + } + set := make(map[State]bool, len(states)) + for _, s := range states { + set[s] = true + } + return set +} + +// decide is the pure seam between the collected evidence and the CLI's exit +// code: nearly every §22 test targets this function, not Check (P9). It +// combines proveCoverage's nine checks with classifyRule's timelines under +// one Policy, and OWNS the H6 mapping: any unobservable rule makes decide +// return a non-nil error, which P10's CLI maps to exit 2 unconditionally +// (H7) — never to 0 or 1, and never suppressed by a real violation found +// alongside it. +// +// Result is fully populated even when the returned error is non-nil: H7's +// "err != nil, the violation list is irrelevant" means the CALLER must not +// use Violations to second-guess the error, not that Result stops being +// useful for the human table on exit 2. +func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, + rt map[string]ruleTimings, gt globalTimings, pol Policy) (Result, error) { + + badStates := badStateSet(pol.States) + + result := Result{ + From: pol.From, + To: pol.To, + GrafanaVersion: h.GrafanaVersion, + Coverage: make(map[string]CoverageResult), + } + for _, p := range polls { + s := p.Skew() + if s < 0 { + s = -s + } + if s > result.ClockSkew { + result.ClockSkew = s + } + } + + minObserved := pol.MinObserved + if minObserved == 0 { + minObserved = len(defs) + } + + windowEnd := pol.To.Add(gt.transitionGrace) + + var ( + skippedRules []Definition + watchedCount int + anyUnobservable bool + unobservableNames []string + ) + + for _, def := range defs { + if def.IsPaused { + skippedRules = append(skippedRules, def) + result.Verdicts = append(result.Verdicts, RuleVerdict{ + Alert: def.Title, RuleUID: def.UID, Outcome: OutcomeSkipped, + PollEvery: rt[def.UID].pollEvery, + Note: "paused before the window opened", + }) + continue + } + watchedCount++ + + t := rt[def.UID] + cov := proveCoverage(h, polls, sentinel, t, def, pol.From, pol.To, gt.transitionGrace) + + if pol.NodataIsUnobservable && !cov.Unobservable { + inWindow := inWindowPolls(pollsForRule(polls, def.UID), pol.From, windowEnd) + if runLen, sawAny := longestHealthRun(inWindow, "nodata"); sawAny && runLen > t.healthGrace { + cov.Unobservable = true + cov.Proved = false + if cov.Reason == "" { + cov.Reason = ReasonNodata + } + cov.Notes = append(cov.Notes, fmt.Sprintf( + "rule %q: health=nodata for %s exceeds healthGrace %s and --nodata-is-unobservable is set", + def.Title, runLen, t.healthGrace)) + } + } + result.Coverage[def.UID] = cov + + outcome, badFor, viols := classifyRule(def, polls, pol.From, windowEnd, badStates, pol.Preexisting) + if cov.Unobservable { + outcome = OutcomeUnobservable + anyUnobservable = true + unobservableNames = append(unobservableNames, fmt.Sprintf("%s (%s)", def.Title, cov.Reason)) + } + result.Violations = append(result.Violations, viols...) + result.Verdicts = append(result.Verdicts, RuleVerdict{ + Alert: def.Title, RuleUID: def.UID, Outcome: outcome, BadFor: badFor, + PollEvery: t.pollEvery, Note: strings.Join(cov.Notes, "; "), + }) + } + + // MinObserved (§12): default len(defs) after the collapse (already done + // by Resolve before decide ever sees defs). skipped rules count against + // it unless AllowPaused says otherwise. A shortfall counts toward exit 1 + // (§9.1), never exit 2 — decide never returns an error for this — and H7 + // requires it to surface through Violations like any other fail reason, + // so a shortfall always produces at least one, even when no rule is + // paused at all (an operator-supplied MinObserved that simply exceeds + // what could ever be resolved). + counted := watchedCount + var attributable []Definition + if pol.AllowPaused { + counted += len(skippedRules) + } else { + attributable = skippedRules + } + if shortfall := minObserved - counted; shortfall > 0 { + attributed := 0 + for _, def := range attributable { + if attributed >= shortfall { + break + } + // §12.1 requires the paused rule and --allow-paused both be + // named to the user; naming the rule is this Violation's job, + // the --allow-paused hint is the CLI table/renderer's (P10) — + // tracked here so it is not dropped when that phase is built. + result.Violations = append(result.Violations, Violation{ + Alert: def.Title, RuleUID: def.UID, Outcome: OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set", + }) + attributed++ + } + for ; attributed < shortfall; attributed++ { + // No named rule explains this part of the deficit — e.g. an + // operator-supplied --min-observed above what could ever be + // resolved. Note, not LastError: LastError is reporting-only + // rule state read from a real poll, and this Violation never + // touched one. + result.Violations = append(result.Violations, Violation{ + Outcome: OutcomeSkipped, + Note: fmt.Sprintf("min-observed %d exceeds the %d rule(s) counted as observed", minObserved, counted), + }) + } + } + + if anyUnobservable { + return result, fmt.Errorf("gate: %d rule(s) unobservable: %s", len(unobservableNames), strings.Join(unobservableNames, "; ")) + } + return result, nil +} diff --git a/grafana-alertcheck/internal/gate/classify_test.go b/grafana-alertcheck/internal/gate/classify_test.go new file mode 100644 index 000000000..7176a9167 --- /dev/null +++ b/grafana-alertcheck/internal/gate/classify_test.go @@ -0,0 +1,859 @@ +package gate + +import ( + "testing" + "time" +) + +func lbl(name string) map[string]string { return map[string]string{"instance": name} } + +// abnormalPoll builds one Poll carrying a single abnormal instance, with the +// bookkeeping classifyRule needs (RuleUID, GrafanaNow, Health, Abnormal). +func abnormalPoll(uid string, at time.Time, state State, labels map[string]string, activeAt time.Time) Poll { + return Poll{ + RuleUID: uid, + GrafanaNow: at, + Found: true, + Health: "ok", + LastEvaluation: at, + Abnormal: []Instance{{Labels: labels, State: state, ActiveAt: activeAt}}, + } +} + +func clearedPoll(uid string, at time.Time, cleared ...string) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at, Cleared: cleared} +} + +func vanishedPoll(uid string, at time.Time, vanished ...string) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at, Vanished: vanished} +} + +func quietPoll(uid string, at time.Time) Poll { + return Poll{RuleUID: uid, GrafanaNow: at, Found: true, Health: "ok", LastEvaluation: at} +} + +var defaultBad = badStateSet(nil) // {firing} + +// --- clean / newly_bad --- + +func TestClassifyRule_NoEvidenceIsClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{quietPoll("r1", from), quietPoll("r1", to)} + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeClean || badFor != 0 || len(viols) != 0 { + t.Fatalf("outcome=%v badFor=%v viols=%v, want clean/0/none", outcome, badFor, viols) + } +} + +func TestClassifyRule_NewOnsetInsideWindowIsNewlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + abnormalPoll("r1", to, StateFiring, lbl("a"), onset), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeNewlyBad { + t.Fatalf("outcome = %v, want newly_bad", outcome) + } + if want := to.Sub(onset); badFor != want { + t.Fatalf("badFor = %v, want %v", badFor, want) + } + if len(viols) != 1 || viols[0].Outcome != OutcomeNewlyBad { + t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) + } +} + +// TestClassifyRule_NewOnsetThatClearsStillFails pins §11.4 point 3: a +// genuinely new bad episode fails even if it clears again before the window +// ends — only a PREEXISTING condition earns the benefit of `recovered`. +func TestClassifyRule_NewOnsetThatClearsStillFails(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(2 * time.Minute) + clearAt := from.Add(3 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, instanceKey(lbl("a"))), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeNewlyBad { + t.Fatalf("outcome = %v, want newly_bad even though it cleared", outcome) + } + if len(viols) != 1 { + t.Fatalf("viols = %+v, want one violation", viols) + } +} + +// --- recovered / persistently_bad (preexisting) --- + +func TestClassifyRule_PreexistingThatRecoversIsRecoveredAndNotAViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + clearAt := from.Add(8 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeRecovered { + t.Fatalf("outcome = %v, want recovered", outcome) + } + if want := clearAt.Sub(from); badFor != want { + t.Fatalf("badFor = %v, want %v", badFor, want) + } + if len(viols) != 0 { + t.Fatalf("viols = %+v, want none: default policy passes a recovered preexisting instance", viols) + } +} + +func TestClassifyRule_PreexistingStillBadAtWindowEndIsPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad", outcome) + } + if badFor != to.Sub(from) { + t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) + } + if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { + t.Fatalf("viols = %+v, want one persistently_bad violation", viols) + } +} + +// --- flapping --- + +func TestClassifyRule_ClearThenBadAgainIsFlapping(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from), + clearedPoll("r1", from.Add(2*time.Minute), key), + abnormalPoll("r1", from.Add(5*time.Minute), StateFiring, lbl("a"), from.Add(5*time.Minute)), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeFlapping { + t.Fatalf("outcome = %v, want flapping", outcome) + } + if len(viols) != 1 || viols[0].Outcome != OutcomeFlapping { + t.Fatalf("viols = %+v, want one flapping violation, always a fail regardless of policy", viols) + } +} + +// --- H2: vanished is a discontinuity, never a clear --- + +func TestClassifyRule_VanishedWhileBadStaysPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Minute)), + vanishedPoll("r1", from.Add(5*time.Minute), key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad: a vanish must never read as a recovery (H2)", outcome) + } + if badFor != to.Sub(from) { + t.Fatalf("badFor = %v, want the full window %v: the freeze must hold the episode open to windowEnd", badFor, to.Sub(from)) + } + if len(viols) != 1 { + t.Fatalf("viols = %+v, want one violation", viols) + } +} + +func TestClassifyRule_VanishedWhileNeverBadIsUninteresting(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + // Pending is abnormal (non-normal) but not in the default {firing} bad + // set, so its vanish must stay uninteresting too. + polls := []Poll{ + abnormalPoll("r1", from, StatePending, lbl("a"), from), + vanishedPoll("r1", from.Add(5*time.Minute), key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeClean || badFor != 0 || len(viols) != 0 { + t.Fatalf("outcome=%v badFor=%v viols=%v, want clean/0/none", outcome, badFor, viols) + } +} + +// --- preexisting policy --- + +func TestClassifyRule_PreexistingPolicyFailFailsARecoveredInstance(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + clearedPoll("r1", from.Add(2*time.Minute), key), + quietPoll("r1", to), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFail) + if outcome != OutcomeRecovered { + t.Fatalf("outcome = %v, want recovered — the descriptive outcome does not change under policy=fail", outcome) + } + if len(viols) != 1 || viols[0].Outcome != OutcomeRecovered { + t.Fatalf("viols = %+v, want one violation: policy=fail gives no benefit of the doubt to a preexisting instance", viols) + } +} + +func TestClassifyRule_PreexistingPolicyIgnoreForgivesPersistentlyBad(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + abnormalPoll("r1", to, StateFiring, lbl("a"), from.Add(-time.Hour)), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad — the descriptive outcome does not change under policy=ignore", outcome) + } + if len(viols) != 0 { + t.Fatalf("viols = %+v, want none: policy=ignore disregards a preexisting instance even if it never recovers", viols) + } +} + +func TestClassifyRule_PreexistingPolicyIgnoreStillFailsANewOnset(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + abnormalPoll("r1", to, StateFiring, lbl("a"), onset), + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingIgnore) + if outcome != OutcomeNewlyBad || len(viols) != 1 { + t.Fatalf("outcome=%v viols=%v, want newly_bad/1: ignore only forgives PREEXISTING badness", outcome, viols) + } +} + +// --- worst-of across instances --- + +func TestClassifyRule_WorstOfMultipleInstancesWins(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + { + RuleUID: "r1", GrafanaNow: from, Found: true, Health: "ok", + Abnormal: []Instance{ + {Labels: lbl("a"), State: StateFiring, ActiveAt: from.Add(-time.Hour)}, // preexisting, will recover + {Labels: lbl("b"), State: StateFiring, ActiveAt: from}, // preexisting, will stay bad + }, + }, + clearedPoll("r1", from.Add(2*time.Minute), instanceKey(lbl("a"))), + { + RuleUID: "r1", GrafanaNow: to, Found: true, Health: "ok", + Abnormal: []Instance{{Labels: lbl("b"), State: StateFiring, ActiveAt: from}}, + }, + } + outcome, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad: the worse of {recovered, persistently_bad}", outcome) + } + if len(viols) != 1 || viols[0].Outcome != OutcomePersistentlyBad { + t.Fatalf("viols = %+v, want exactly the persistently_bad instance's violation", viols) + } +} + +// --- decide(): skipped rules, unobservable (H6), MinObserved, exit mapping (H7) --- + +func TestDecide_SkippedRuleNeverReachesProveCoverage(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1", IsPaused: true} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to, AllowPaused: true} + + // No polls, no sentinel at all: a heartbeat_gap/no_sentinel misclassification + // here would mean proveCoverage ran for a skipped rule (§4.3's obligation). + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil: a rule paused before the window is skipped, not unobservable", err) + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeSkipped { + t.Fatalf("Verdicts = %+v, want exactly one skipped verdict", res.Verdicts) + } + if _, ok := res.Coverage["r1"]; ok { + t.Fatalf("Coverage[r1] present, want absent: a skipped rule has no coverage to prove") + } +} + +func TestDecide_UnobservableRuleAlwaysReturnsAnError(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + // No sentinel at all: check 1 fails, so the rule is unobservable + // regardless of anything else. + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, nil, nil, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want non-nil: H6/H7 require an unobservable rule to always fail the run") + } + if len(res.Verdicts) != 1 || res.Verdicts[0].Outcome != OutcomeUnobservable { + t.Fatalf("Verdicts = %+v, want exactly one unobservable verdict", res.Verdicts) + } +} + +// TestDecide_UnobservableWinsEvenAlongsideARealViolation pins H6 exactly: +// "Any unobservable rule -> exit 2, no exception, even alongside a real +// newly_bad." +func TestDecide_UnobservableWinsEvenAlongsideARealViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(5 * time.Minute) + + defBroken := Definition{UID: "broken", Title: "Broken"} + defBad := Definition{UID: "bad", Title: "Bad"} + defs := []Definition{defBroken, defBad} + rt := map[string]ruleTimings{ + "broken": newRuleTimings(30*time.Second, 60), + "bad": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + if ts.Equal(onset) || ts.After(onset) { + polls = append(polls, abnormalPoll("bad", ts, StateFiring, lbl("a"), onset)) + } else { + polls = append(polls, quietPoll("bad", ts)) + } + } + // "broken" gets no polls at all: no sentinel, no heartbeats -> unobservable. + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want non-nil: one rule is unobservable") + } + var gotBroken, gotBad Outcome + for _, v := range res.Verdicts { + switch v.RuleUID { + case "broken": + gotBroken = v.Outcome + case "bad": + gotBad = v.Outcome + } + } + if gotBroken != OutcomeUnobservable { + t.Fatalf("broken.Outcome = %v, want unobservable", gotBroken) + } + if gotBad != OutcomeNewlyBad { + t.Fatalf("bad.Outcome = %v, want newly_bad: classification still runs and is still visible in Verdicts (H5)", gotBad) + } + if len(res.Violations) == 0 { + t.Fatalf("Violations empty, want the newly_bad instance still reported even though the run fails on the unobservable rule") + } +} + +func TestDecide_CleanWindowIsAPass(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if len(res.Violations) != 0 { + t.Fatalf("Violations = %+v, want none: H7 says a pass is exactly len(Violations)==0 && err==nil", res.Violations) + } + if res.Verdicts[0].Outcome != OutcomeClean { + t.Fatalf("Outcome = %v, want clean", res.Verdicts[0].Outcome) + } +} + +// --- MinObserved shortfall (§12) --- + +func TestDecide_SkippedOnlyShortfallProducesAViolationWithoutAnError(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + watched := Definition{UID: "watched", Title: "Watched"} + paused := Definition{UID: "paused", Title: "Paused", IsPaused: true} + defs := []Definition{watched, paused} + rt := map[string]ruleTimings{ + "watched": newRuleTimings(30*time.Second, 60), + "paused": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + // MinObserved defaults to len(defs) = 2, but only "watched" is observable. + pol := Policy{From: from, To: to} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("watched", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil: a shortfall caused only by a skipped rule is exit 1, not exit 2 (§9.1)", err) + } + if len(res.Violations) != 1 { + t.Fatalf("Violations = %+v, want exactly one: H7 needs the shortfall visible through Violations to keep its equivalence", res.Violations) + } + if v := res.Violations[0]; v.Outcome != OutcomeSkipped || v.RuleUID != "paused" || v.Alert != "Paused" { + t.Fatalf("Violations[0] = %+v, want Outcome=skipped naming the paused rule (§12.1: the message names the paused rule)", v) + } + if res.Violations[0].Note == "" { + t.Fatalf("Violations[0].Note is empty, want an explanation: the shortfall reason must not be smuggled into LastError, " + + "which is reporting-only rule state from a real poll this synthetic Violation never touched") + } +} + +// TestDecide_ExplicitMinObservedShortfallWithNoPausedRuleStillProducesAViolation +// pins F3: an operator-supplied MinObserved that exceeds what could ever be +// resolved is still a shortfall, even with zero paused rules to blame it on +// — H7 must not let this silently read as a pass. +func TestDecide_ExplicitMinObservedShortfallWithNoPausedRuleStillProducesAViolation(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to, MinObserved: 3} // only one rule will ever be resolved + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("r1", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil: an unmet MinObserved is exit 1, never exit 2", err) + } + if len(res.Violations) != 2 { + t.Fatalf("Violations = %+v, want two: the shortfall (3-1=2) is not explained by any paused rule, "+ + "so H7 requires it to surface directly rather than pass silently", res.Violations) + } + for _, v := range res.Violations { + if v.Outcome != OutcomeSkipped { + t.Fatalf("Violations = %+v, want Outcome=skipped on the synthetic shortfall entries", res.Violations) + } + } +} + +func TestDecide_AllowPausedSuppressesTheShortfall(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + + watched := Definition{UID: "watched", Title: "Watched"} + paused := Definition{UID: "paused", Title: "Paused", IsPaused: true} + defs := []Definition{watched, paused} + rt := map[string]ruleTimings{ + "watched": newRuleTimings(30*time.Second, 60), + "paused": newRuleTimings(30*time.Second, 60), + } + gt := globalTimings{} + pol := Policy{From: from, To: to, AllowPaused: true} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, quietPoll("watched", ts)) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if len(res.Violations) != 0 { + t.Fatalf("Violations = %+v, want none: --allow-paused must suppress the shortfall entirely", res.Violations) + } +} + +// --- nodata escalation (decide's own Policy-driven check) --- + +func TestDecide_NodataIsUnobservableEscalatesASustainedRun(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} // healthGrace = max(60s,60s) = 60s + gt := globalTimings{} + pol := Policy{From: from, To: to, NodataIsUnobservable: true} + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err == nil { + t.Fatalf("err = nil, want non-nil: a sustained nodata run must be unobservable under --nodata-is-unobservable") + } + if res.Coverage["r1"].Reason != ReasonNodata { + t.Fatalf("Reason = %q, want %q", res.Coverage["r1"].Reason, ReasonNodata) + } +} + +func TestDecide_NodataIsANoteByDefault(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + defs := []Definition{def} + rt := map[string]ruleTimings{"r1": newRuleTimings(30*time.Second, 60)} + gt := globalTimings{} + pol := Policy{From: from, To: to} // NodataIsUnobservable defaults to false + + var polls []Poll + for ts := from; !ts.After(to); ts = ts.Add(30 * time.Second) { + polls = append(polls, Poll{RuleUID: "r1", GrafanaNow: ts, Found: true, Health: "nodata", LastEvaluation: ts}) + } + sentinel := to + + res, err := decide(Header{StartedAt: from.Add(-time.Hour)}, polls, &sentinel, defs, rt, gt, pol) + if err != nil { + t.Fatalf("err = %v, want nil: 96%% of the fleet runs no_data_state:OK and must not fail by default", err) + } + if res.Coverage["r1"].Unobservable { + t.Fatalf("Coverage[r1].Unobservable = true, want false by default") + } +} + +// --- F1/F2 regressions: preexisting is decided by ActiveAt, not poll timing --- + +// TestClassifyRule_OnsetBetweenFromAndFirstPollIsNewlyBadNotRecovered pins +// F1: an instance whose true onset (ActiveAt) falls strictly inside the +// window — even though the first poll that happens to observe it already +// shows it bad — must never be treated as preexisting. If it then clears, +// the plan requires newly_bad (exit 1), not recovered (exit 0). +func TestClassifyRule_OnsetBetweenFromAndFirstPollIsNewlyBadNotRecovered(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(1 * time.Minute) // the true onset, strictly after `from` + firstPoll := from.Add(2 * time.Minute) // the first poll that happens to observe it + clearAt := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", firstPoll, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeNewlyBad { + t.Fatalf("outcome = %v, want newly_bad: the onset is after `from`, so it is not preexisting even though "+ + "the FIRST in-window poll already observes it bad (F1)", outcome) + } + if len(viols) != 1 || viols[0].Outcome != OutcomeNewlyBad { + t.Fatalf("viols = %+v, want one newly_bad violation: a policy=fail-unless-recovered default must still fail this", viols) + } + if want := clearAt.Sub(onset); badFor != want { + t.Fatalf("badFor = %v, want %v: BadFor must count from the true onset, not from `from` (F1's overcount bug)", badFor, want) + } +} + +// TestClassifyRule_OnsetJustBeforeFromIsPreexisting is the mirror check: an +// onset at or before `from` (even if the first poll is later) is genuinely +// preexisting and, if it clears, is `recovered`. +func TestClassifyRule_OnsetJustBeforeFromIsPreexisting(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(-time.Minute) + firstPoll := from.Add(2 * time.Minute) + clearAt := from.Add(5 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + abnormalPoll("r1", firstPoll, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, key), + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeRecovered { + t.Fatalf("outcome = %v, want recovered: the onset is at/before `from`, genuinely preexisting", outcome) + } + if len(viols) != 0 { + t.Fatalf("viols = %+v, want none: default policy passes a recovered preexisting instance", viols) + } + if want := clearAt.Sub(from); badFor != want { + t.Fatalf("badFor = %v, want %v: a preexisting episode's BadFor is clamped to window-open, not backdated past it", badFor, want) + } +} + +// TestClassifyRule_SkewTranslatesActiveAtAcrossTheWindowBoundary pins F2: a +// poll carrying a nonzero skew must have its ActiveAt (and GrafanaNow) +// translated to the runner domain before comparing against `from` — a raw, +// untranslated comparison would land on the wrong side of the F1 boundary +// check. +func TestClassifyRule_SkewTranslatesActiveAtAcrossTheWindowBoundary(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + // Grafana's clock reads 90s ahead of the runner's (skew = +90s). The + // poll's raw GrafanaNow/ActiveAt both sit 90s past `from` in Grafana's + // domain, but translate to exactly `from` in the runner domain — genuinely + // preexisting once translated, and wrongly "newly_bad" if the skew is + // ignored. + skew := 90 * time.Second + rawActiveAt := from.Add(skew) + poll := Poll{ + RuleUID: "r1", GrafanaNow: from.Add(skew), Found: true, Health: "ok", + LastEvaluation: from.Add(skew), SkewMS: skew.Milliseconds(), + Abnormal: []Instance{{Labels: lbl("a"), State: StateFiring, ActiveAt: rawActiveAt}}, + } + stillBad := poll + stillBad.GrafanaNow = to.Add(skew) + stillBad.LastEvaluation = to.Add(skew) + + outcome, badFor, _ := classifyRule(def, []Poll{poll, stillBad}, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomePersistentlyBad { + t.Fatalf("outcome = %v, want persistently_bad: a +90s skew must translate ActiveAt back to exactly `from` (F2)", outcome) + } + if badFor != to.Sub(from) { + t.Fatalf("badFor = %v, want the full window %v", badFor, to.Sub(from)) + } +} + +// --- F4: InstanceLabels must survive a timeline first created by a bare marker --- + +func TestClassifyRule_LabelsSurviveWhenTimelineStartsFromAClearedMarker(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + newOnset := from.Add(5 * time.Minute) + + polls := []Poll{ + // The very first mention of this key is a bare Cleared marker (its + // prior bad episode, if any, started before the window) — no labels + // travel with a Cleared/Vanished event. + clearedPoll("r1", from.Add(1*time.Minute), key), + abnormalPoll("r1", newOnset, StateFiring, lbl("a"), newOnset), + quietPoll("r1", to), + } + _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if len(viols) != 1 { + t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) + } + if viols[0].InstanceLabels == nil || viols[0].InstanceLabels["instance"] != "a" { + t.Fatalf("InstanceLabels = %+v, want {instance: a}: labels must backfill even though the "+ + "timeline was first created by a label-less Cleared marker (F4)", viols[0].InstanceLabels) + } +} + +// TestClassifyRule_ViolationFieldsArePrecise pins FirstSeen/ClearedAt exactly, +// not just that a violation exists (F7). +func TestClassifyRule_ViolationFieldsArePrecise(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onset := from.Add(2 * time.Minute) + clearAt := from.Add(3 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onset, StateFiring, lbl("a"), onset), + clearedPoll("r1", clearAt, instanceKey(lbl("a"))), + quietPoll("r1", to), + } + _, _, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if len(viols) != 1 { + t.Fatalf("viols = %+v, want exactly one violation", viols) + } + v := viols[0] + if !v.FirstSeen.Equal(onset) { + t.Fatalf("FirstSeen = %v, want %v", v.FirstSeen, onset) + } + if !v.ClearedAt.Equal(clearAt) { + t.Fatalf("ClearedAt = %v, want %v", v.ClearedAt, clearAt) + } + if v.InstanceLabels["instance"] != "a" { + t.Fatalf("InstanceLabels = %+v, want {instance: a}", v.InstanceLabels) + } +} + +// TestClassifyRule_ClearedEventPastWindowEndClampsToWindowEnd pins the +// episode.end clamp: inWindowPolls admits a poll up to its own skew bound +// past windowEnd (§16's widened membership test), so a genuine Cleared event +// on such a poll must not leave the episode extending beyond windowEnd. +func TestClassifyRule_ClearedEventPastWindowEndClampsToWindowEnd(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + bound := 30 * time.Second + clearedAt := to.Add(20 * time.Second) // past windowEnd, but within the skew bound + + polls := []Poll{ + abnormalPoll("r1", from, StateFiring, lbl("a"), from.Add(-time.Hour)), + { + RuleUID: "r1", GrafanaNow: clearedAt, Found: true, Health: "ok", + SkewBoundMS: bound.Milliseconds(), Cleared: []string{key}, + }, + } + outcome, badFor, _ := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeRecovered { + t.Fatalf("outcome = %v, want recovered", outcome) + } + if badFor != to.Sub(from) { + t.Fatalf("badFor = %v, want the window %v exactly: the episode end must clamp to windowEnd, "+ + "not extend to the late Cleared event's raw time", badFor, to.Sub(from)) + } +} + +// TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean pins the fail-closed +// reading of the upper boundary: an instance whose runner-domain onset lands +// only slightly past windowEnd (to + transitionGrace) is reachable at all only +// because inWindowPolls widens the boundary outward by the skew bound, so the +// gate cannot PROVE it belongs to the next window. It is charged as newly_bad — +// with BadFor truncated to zero — rather than silently forgiven as clean. +func TestClassifyRule_OnsetJustPastWindowEndIsNewlyBadNotClean(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + grace := time.Minute + windowEnd := to.Add(grace) + def := Definition{UID: "r1", Title: "R1"} + + // A poll admitted only by its own skew bound: its GrafanaNow sits 20s past + // windowEnd, inside the 30s tolerance. It carries an instance whose onset + // is 10s past windowEnd — still "after the grace", but only by less than + // the measurement's own uncertainty. + bound := 30 * time.Second + poll := Poll{ + RuleUID: "r1", GrafanaNow: windowEnd.Add(20 * time.Second), Found: true, Health: "ok", + LastEvaluation: windowEnd.Add(20 * time.Second), SkewBoundMS: bound.Milliseconds(), + Abnormal: []Instance{{Labels: lbl("a"), State: StateFiring, ActiveAt: windowEnd.Add(10 * time.Second)}}, + } + + outcome, badFor, viols := classifyRule(def, []Poll{poll}, from, windowEnd, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeNewlyBad { + t.Fatalf("outcome = %v, want newly_bad: an onset past windowEnd seen only via the skew bound must fail closed", outcome) + } + if badFor != 0 { + t.Fatalf("badFor = %v, want 0: the zero-length episode must truncate to the window end", badFor) + } + if len(viols) != 1 { + t.Fatalf("viols = %+v, want exactly one newly_bad violation", viols) + } +} + +// TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative pins the +// end-before-start clamp: two polls with different measured skews can +// translate so that a closing poll's runner-domain time lands before the +// opening poll's, which — unclamped — would feed mergeDurations a negative +// span. +func TestClassifyRule_CloseBeforeOpenClampsToZeroNotNegative(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(10 * time.Minute) + onsetPoll := from.Add(5 * time.Minute) + closePoll := from.Add(6 * time.Minute) + closeSkew := 2 * time.Minute // translates closePoll back to from+4min, before onsetPoll's from+5min + def := Definition{UID: "r1", Title: "R1"} + key := instanceKey(lbl("a")) + + polls := []Poll{ + quietPoll("r1", from), + abnormalPoll("r1", onsetPoll, StateFiring, lbl("a"), onsetPoll), // skew 0 + { + RuleUID: "r1", GrafanaNow: closePoll, Found: true, Health: "ok", + SkewMS: closeSkew.Milliseconds(), Cleared: []string{key}, + }, + quietPoll("r1", to), + } + outcome, badFor, viols := classifyRule(def, polls, from, to, defaultBad, PreexistingFailUnlessRecovered) + if outcome != OutcomeNewlyBad { + t.Fatalf("outcome = %v, want newly_bad", outcome) + } + if badFor < 0 { + t.Fatalf("badFor = %v, want a non-negative duration even though the closing poll's translated "+ + "time landed before the opening poll's", badFor) + } + if badFor != 0 { + t.Fatalf("badFor = %v, want 0: the clamp collapses the inverted span to a zero-length episode", badFor) + } + if len(viols) != 1 { + t.Fatalf("viols = %+v, want one violation", viols) + } +} + +// --- mergeDurations --- + +func TestMergeDurations_OverlappingEpisodesCountOnce(t *testing.T) { + from := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + eps := []episode{ + {start: from, end: from.Add(5 * time.Minute)}, + {start: from.Add(2 * time.Minute), end: from.Add(8 * time.Minute)}, // overlaps the first + {start: from.Add(20 * time.Minute), end: from.Add(21 * time.Minute)}, // disjoint + } + got := mergeDurations(eps) + want := 8*time.Minute + 1*time.Minute // [0,8) merged = 8m, plus the disjoint 1m + if got != want { + t.Fatalf("mergeDurations = %v, want %v: two simultaneously-bad instances must not double-count their overlap", got, want) + } +} + +func TestMergeDurations_Empty(t *testing.T) { + if got := mergeDurations(nil); got != 0 { + t.Fatalf("mergeDurations(nil) = %v, want 0", got) + } +} diff --git a/grafana-alertcheck/internal/gate/coverage.go b/grafana-alertcheck/internal/gate/coverage.go index 36d2c3dac..3b5425b6d 100644 --- a/grafana-alertcheck/internal/gate/coverage.go +++ b/grafana-alertcheck/internal/gate/coverage.go @@ -2,7 +2,6 @@ package gate import ( "fmt" - "slices" "time" ) @@ -82,16 +81,12 @@ func proveCoverage(h Header, polls []Poll, sentinel *time.Time, t ruleTimings, d windowEnd := to.Add(grace) - var rulePolls []Poll - for _, p := range polls { - if p.RuleUID == def.UID { - rulePolls = append(rulePolls, p) - } - } - // Stable, not sort.Slice: two polls sharing a GrafanaNow (a coarse Date - // header, or a corrupted/replayed log) must not reorder nondeterministically - // in a function that promises to be pure. - slices.SortStableFunc(rulePolls, func(a, b Poll) int { return a.GrafanaNow.Compare(b.GrafanaNow) }) + // pollsForRule (classify.go) is the single filter+sort implementation for + // "select one rule's polls, stably ordered by GrafanaNow" — proveCoverage + // and classifyRule must never carry two independent copies of this + // selection, or one drifting from the other becomes exactly the kind of + // silent membership mismatch this file's checks exist to prevent. + rulePolls := pollsForRule(polls, def.UID) var res CoverageResult fail := func(reason UnobservableReason, note string) { @@ -264,7 +259,7 @@ func inWindowPolls(polls []Poll, from, windowEnd time.Time) []Poll { var out []Poll for _, p := range polls { bound := p.SkewBound() - runner := p.GrafanaNow.Add(-p.Skew()) + runner := runnerTime(p, p.GrafanaNow) if runner.Before(from.Add(-bound)) || runner.After(windowEnd.Add(bound)) { continue } @@ -294,7 +289,7 @@ func ruleHeartbeatGap(in []Poll, from, windowEnd time.Time) (largestGap time.Dur return windowEnd.Sub(from), from } - runnerOf := func(p Poll) time.Time { return p.GrafanaNow.Add(-p.Skew()) } + runnerOf := func(p Poll) time.Time { return runnerTime(p, p.GrafanaNow) } first := in[0] if gap := runnerOf(first).Sub(from) + first.SkewBound(); gap > largestGap {