Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
//go:build unix

package gate

import (
Expand All @@ -12,10 +10,6 @@ import (
// point (§8): a second writer must fail immediately with an error the operator
// sees, not queue behind the first and start appending to a log somebody else
// already finished.
//
// There is deliberately no Windows implementation — runners are Linux and
// goreleaser builds linux+darwin only (P6, P12) — so the package does not
// build there at all rather than silently skipping the lock.
func lockExclusive(f *os.File) error {
if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
return fmt.Errorf("flock: %w", err)
Expand Down
28 changes: 28 additions & 0 deletions grafana-alertcheck/internal/gate/log.go
Original file line number Diff line number Diff line change
Expand Up @@ -237,6 +237,34 @@ func (r *Reducer) Reduce(uid string, obs Observation) Poll {
return p
}

// seedFrom restores the marker state above from polls that are already in the
// log, so the first poll a NEW Reducer produces compares against the last poll
// the previous one wrote instead of against an empty set.
//
// It exists for the one place a recording changes hands: watch's parent takes
// the first observation of every rule and its detached child continues from
// there (P6). Without the seed, an instance that is abnormal in the parent's
// observation and gone by the child's first poll produces no marker at all —
// it leaves the record as though it had never been bad, which is H2's
// fail-open reached through the handoff rather than through a reason string.
//
// Not-found polls are skipped, mirroring Reduce: an absent rule leaves the
// previous abnormal set untouched rather than emptying it.
func (r *Reducer) seedFrom(polls []Poll) {
r.mu.Lock()
defer r.mu.Unlock()
for _, p := range polls {
if !p.Found {
continue
}
keys := make(map[string]struct{}, len(p.Abnormal))
for _, inst := range p.Abnormal {
keys[instanceKey(inst.Labels)] = struct{}{}
}
r.prevAbnormal[p.RuleUID] = keys
}
}

// reasonNames reports whether reason names want. Newer Grafana versions
// comma-join several reasons into one string, so this tests membership rather
// than equality (P7 check 9 needs the same test for KeepLast).
Expand Down
44 changes: 33 additions & 11 deletions grafana-alertcheck/internal/gate/schedule.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,21 +193,27 @@ type Scheduler struct {
every map[string]time.Duration
}

// NewScheduler builds a Scheduler over rules (keyed by UID), staggering each
// rule's initial next-due time across [0, pollEvery) so the fleet does not
// start phase-aligned (§5's burst-bound proof depends on this: an
// already-staggered fleet only re-aligns by chance, briefly, not by
// NewScheduler builds a Scheduler over per-rule cadences (keyed by UID),
// staggering each rule's initial next-due time across [0, pollEvery) so the
// fleet does not start phase-aligned (§5's burst-bound proof depends on this:
// an already-staggered fleet only re-aligns by chance, briefly, not by
// construction).
func NewScheduler(rules map[string]ruleTimings, now time.Time) *Scheduler {
//
// It takes cadences rather than whole ruleTimings on purpose: a scheduler
// decides when to poll and nothing else, so it must not be handed maxGap,
// healthGrace or evalStaleAfter. Those are coverage thresholds, they are
// applied by the pure layer at classification time, and the recorder that
// drives this scheduler never applies them at all.
func NewScheduler(every map[string]time.Duration, now time.Time) *Scheduler {
s := &Scheduler{
next: make(map[string]time.Time, len(rules)),
every: make(map[string]time.Duration, len(rules)),
next: make(map[string]time.Time, len(every)),
every: make(map[string]time.Duration, len(every)),
}
for uid, rt := range rules {
s.every[uid] = rt.pollEvery
for uid, pollEvery := range every {
s.every[uid] = pollEvery
var offset time.Duration
if rt.pollEvery > 0 {
offset = rand.N(rt.pollEvery)
if pollEvery > 0 {
offset = rand.N(pollEvery)
}
s.next[uid] = now.Add(offset)
}
Expand Down Expand Up @@ -247,6 +253,22 @@ func (s *Scheduler) Mark(uid string, now time.Time) {
s.next[uid] = now.Add(s.every[uid])
}

// earliestDue returns the earliest scheduled next-due time, and false when the
// scheduler holds no rules at all. The recorder's loop (P6) waits exactly that
// long instead of waking on a fixed tick: a fixed tick either polls a slack
// rule early — spending request budget the §5 formulas already accounted for —
// or wakes too late for the tightest rule and opens a gap inside its own
// maxGap.
func (s *Scheduler) earliestDue() (time.Time, bool) {
var earliest time.Time
for _, t := range s.next {
if earliest.IsZero() || t.Before(earliest) {
earliest = t
}
}
return earliest, !earliest.IsZero()
}

// CheckBudget applies §5's error-at-start check to a fully resolved schedule.
// t and measured are both keyed by rule UID; measured must carry every UID in
// t; a rule this run never measured can't have its budget proved, and a
Expand Down
8 changes: 4 additions & 4 deletions grafana-alertcheck/internal/gate/schedule_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,9 @@ func TestScheduler_MarkAdvancesNextDue(t *testing.T) {
// forced onto the tight rule's cadence.
func TestScheduler_PerRuleCadenceOverTime(t *testing.T) {
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
rules := map[string]ruleTimings{
"tight": {pollEvery: 10 * time.Second},
"slack": {pollEvery: 300 * time.Second},
rules := map[string]time.Duration{
"tight": 10 * time.Second,
"slack": 300 * time.Second,
}
s := NewScheduler(rules, start)

Expand Down Expand Up @@ -196,7 +196,7 @@ func TestScheduler_PerRuleCadenceOverTime(t *testing.T) {

func TestNewScheduler_StaggersWithinPollEvery(t *testing.T) {
now := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
rules := map[string]ruleTimings{"r1": {pollEvery: 100 * time.Second}}
rules := map[string]time.Duration{"r1": 100 * time.Second}
s := NewScheduler(rules, now)
offset := s.next["r1"].Sub(now)
if offset < 0 || offset >= 100*time.Second {
Expand Down
40 changes: 37 additions & 3 deletions grafana-alertcheck/internal/gate/source_fake_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@ import (
// That is sufficient here: every retry/backoff test in this phase only needs
// to avoid a real sleep. It is NOT sufficient for a test that must prove a
// wait did not fire early — e.g. a P4 scheduler test asserting Due() doesn't
// return a rule before its next-due time. That needs a clock with a real
// waiter list keyed off Advance, which does not exist yet; build it when a
// phase actually needs it rather than guessing its shape now.
// return a rule before its next-due time. Use virtualClock below for that: it
// is the clock P6's recorder-loop tests needed, and it makes a wait and the
// passage of time the same event.
type fakeClock struct {
mu sync.Mutex
now time.Time
Expand Down Expand Up @@ -45,6 +45,40 @@ func (c *fakeClock) After(d time.Duration) <-chan time.Time {
return ch
}

// virtualClock is a Clock in which time moves only when something waits for
// it: After(d) jumps Now() forward by d and fires at once. That makes a
// recorder-loop test both instant and exact — a loop that waits for its next
// scheduled poll gets that poll's time, never an early or a late wake — and it
// terminates, which a clock whose After fires without advancing Now does not
// (the loop would spin forever on a rule that never comes due).
//
// It is goroutine-safe, but a test that advances time from two goroutines gets
// what it deserves: use it from the loop under test only.
type virtualClock struct {
mu sync.Mutex
now time.Time
}

func newVirtualClock(now time.Time) *virtualClock { return &virtualClock{now: now} }

func (c *virtualClock) Now() time.Time {
c.mu.Lock()
defer c.mu.Unlock()
return c.now
}

func (c *virtualClock) After(d time.Duration) <-chan time.Time {
c.mu.Lock()
if d > 0 {
c.now = c.now.Add(d)
}
fireAt := c.now
c.mu.Unlock()
ch := make(chan time.Time, 1)
ch <- fireAt
return ch
}

// steppingClock advances by a fixed step on every Now() call, so a test can
// assert exact latency/skew-bound arithmetic (doRequest's three clock reads
// per attempt) without depending on real wall-clock timing.
Expand Down
Loading
Loading