From dd089cd059ce435122ed11f567a4626685200fb3 Mon Sep 17 00:00:00 2001 From: Bartek Tofel Date: Mon, 31 Aug 2026 18:53:07 +0200 Subject: [PATCH] Wire watch/check subcommands to the gate library, with a table+JSON renderer and H6/H7 exit-code mapping. Extend Result with per-rule/global thresholds and a real skew bound; export SkewHardLimit; reject --states normal. --- .../cmd/grafana-alertcheck/check.go | 148 ++++++++++++++++++ .../cmd/grafana-alertcheck/check_test.go | 147 +++++++++++++++++ .../cmd/grafana-alertcheck/common.go | 108 +++++++++++++ .../cmd/grafana-alertcheck/main.go | 13 +- .../cmd/grafana-alertcheck/table.go | 136 ++++++++++++++++ .../cmd/grafana-alertcheck/table_test.go | 114 ++++++++++++++ .../cmd/grafana-alertcheck/watch.go | 148 ++++++++++++++++++ .../cmd/grafana-alertcheck/watch_test.go | 99 ++++++++++++ grafana-alertcheck/internal/gate/check.go | 5 +- grafana-alertcheck/internal/gate/classify.go | 69 +++++++- grafana-alertcheck/internal/gate/schedule.go | 10 +- grafana-alertcheck/internal/gate/source.go | 6 +- 12 files changed, 981 insertions(+), 22 deletions(-) create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/check.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/check_test.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/common.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/table.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/table_test.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/watch.go create mode 100644 grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check.go b/grafana-alertcheck/cmd/grafana-alertcheck/check.go new file mode 100644 index 000000000..f4e9dab26 --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/check.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "io" + "os/signal" + "syscall" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +const checkUsage = "usage: grafana-alertcheck check [--in ] [--pidfile F] --from RFC3339 --to RFC3339 " + + "[--alerts ...] [--folder F] [--states ...] [--preexisting ...] [--min-observed N] [--allow-paused] " + + "[--nodata-is-unobservable] [--concurrency N] [--output json]" + +// runCheck is the classify step's CLI surface: parse flags into a +// gate.Config, run gate.Check, and translate its (Result, error) into +// §20.2/§20.3's output and exit code. All of the correctness lives in +// gate.Check (P9) and decide (P8) — this file's only job is presentation and +// the H6/H7 exit-code mapping, which exitCode below keeps as one pure +// function so it can be tested without a network. +func runCheck(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("check", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprintln(stderr, checkUsage) } + + common := registerCommon(fs) + in := fs.String("in", "", "path of a log recorded by watch; empty selects single-step mode (§9)") + pidfile := fs.String("pidfile", "", "pidfile of the recorder to stop before reading --in (default .pid)") + from := fs.String("from", "", "the moment the deploy finished, RFC3339 (required in recorder mode, §7)") + to := fs.String("to", "", "the end of the window to classify, RFC3339 (required)") + states := fs.String("states", "", "comma-separated bad states to classify against (default: firing, §13)") + preexisting := fs.String("preexisting", "", "how to judge an instance already bad at `from` (default: fail-unless-recovered, §11.7)") + minObserved := fs.Int("min-observed", 0, "minimum rules that must be observed (default: every resolved rule, §12)") + allowPaused := fs.Bool("allow-paused", false, "do not count a rule paused before the window against --min-observed (§12.1)") + nodataIsUnobservable := fs.Bool("nodata-is-unobservable", false, "treat a sustained health=nodata as unobservable rather than a note (§10.2)") + output := fs.String("output", "", `"json" writes the machine-readable Result to stdout in addition to the table (§20.2); default is the table alone`) + + if err := fs.Parse(args); err != nil { + return 2 + } + if *output != "" && *output != "json" { + fmt.Fprintf(stderr, "--output: unknown value %q (only \"json\" is supported)\n", *output) + return 2 + } + + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + alerts, err := readAlerts(stdin, *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + stateList, err := parseStates(*states) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + preexistingPolicy, err := parsePreexisting(*preexisting) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + cfg := gate.Config{ + URL: url, + Token: token, + Alerts: alerts, + Folder: *common.folder, + States: stateList, + Preexisting: preexistingPolicy, + MinObserved: *minObserved, + AllowPaused: *allowPaused, + NodataIsUnobservable: *nodataIsUnobservable, + Log: *in, + PidFile: *pidfile, + Concurrency: *common.concurrency, + Clock: gate.SystemClock{}, + Notes: stderr, + } + if *to == "" { + fmt.Fprintln(stderr, "check: --to is required (§7)") + return 2 + } + t, err := time.Parse(time.RFC3339, *to) + if err != nil { + fmt.Fprintf(stderr, "--to: %v\n", err) + return 2 + } + cfg.To = t + if *from != "" { + f, err := time.Parse(time.RFC3339, *from) + if err != nil { + fmt.Fprintf(stderr, "--from: %v\n", err) + return 2 + } + cfg.From = f + } + + // SIGINT/SIGTERM cancel the run cleanly rather than leaving an operator's + // Ctrl-C to kill the process mid-collection: Check's collection loop and + // drain wait both already select on ctx.Done() (check.go), so this makes + // an interrupted run fail the way every other could-not-check path does + // — exit 2, never a silently truncated pass. + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + result, checkErr := gate.Check(ctx, cfg) + + if err := renderTable(stderr, result); err != nil { + fmt.Fprintln(stderr, err) + } + if checkErr != nil { + fmt.Fprintln(stderr, checkErr) + } + if *output == "json" { + enc := json.NewEncoder(stdout) + enc.SetIndent("", " ") + if err := enc.Encode(result); err != nil { + fmt.Fprintf(stderr, "encode --output json: %v\n", err) + return 2 + } + } + return exitCode(result, checkErr) +} + +// exitCode is §20.3/H6/H7's whole mapping, kept as one pure function of +// exactly what Check returns so it is testable without a network: err != nil +// is exit 2 UNCONDITIONALLY — never 0 and never 1, even alongside real +// violations, because inability beats violation (H6) and an error is never a +// pass (H7). Violations without an error is exit 1. Neither is exit 0. +func exitCode(res gate.Result, err error) int { + switch { + case err != nil: + return 2 + case len(res.Violations) > 0: + return 1 + default: + return 0 + } +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go new file mode 100644 index 000000000..dbf81d3a9 --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/check_test.go @@ -0,0 +1,147 @@ +package main + +import ( + "bytes" + "errors" + "os" + "strings" + "testing" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// TestExitCode pins §20.3/H6/H7's mapping directly against exitCode, with no +// network involved: err != nil is exit 2 even alongside violations (H6 — +// inability beats violation), violations alone are exit 1, and neither is 0. +func TestExitCode(t *testing.T) { + tests := []struct { + name string + res gate.Result + err error + want int + }{ + {"pass", gate.Result{}, nil, 0}, + {"violation", gate.Result{Violations: []gate.Violation{{}}}, nil, 1}, + {"error alone", gate.Result{}, errors.New("boom"), 2}, + {"error beats violation", gate.Result{Violations: []gate.Violation{{}}}, errors.New("boom"), 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := exitCode(tt.res, tt.err); got != tt.want { + t.Fatalf("exitCode(...) = %d, want %d", got, tt.want) + } + }) + } +} + +func writeTempAlerts(t *testing.T) string { + t.Helper() + path := t.TempDir() + "/alerts.txt" + if err := os.WriteFile(path, []byte("Some Alert\n"), 0o644); err != nil { + t.Fatal(err) + } + return path +} + +// TestRunCheck_FlagValidation is the flag-validation matrix: every one of +// these must fail before any network call, because Config.validate() (P9) +// runs first — an unreachable GRAFANA_URL succeeding or timing out is a +// different test than these, which check pure input validation. +func TestRunCheck_FlagValidation(t *testing.T) { + tests := []struct { + name string + env bool + args func(t *testing.T) []string + wantErr string + }{ + {"missing env", false, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--alerts", writeTempAlerts(t)} + }, "GRAFANA_URL"}, + {"missing to", true, func(t *testing.T) []string { + return []string{"--alerts", writeTempAlerts(t)} + }, "--to"}, + {"bad to", true, func(t *testing.T) []string { + return []string{"--to", "not-a-time", "--alerts", writeTempAlerts(t)} + }, "--to"}, + {"bad from", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--from", "not-a-time", "--alerts", writeTempAlerts(t)} + }, "--from"}, + {"bad output", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--output", "xml", "--alerts", writeTempAlerts(t)} + }, "--output"}, + {"bad states", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--states", "bogus", "--alerts", writeTempAlerts(t)} + }, "--states"}, + {"states normal is rejected", true, func(t *testing.T) []string { + // normal is the good state, never a state to classify AS bad + // (R1): accepting it would make --states normal fail every + // healthy instance, the fail-open shape H7 exists to prevent. + return []string{"--to", "2026-01-01T00:00:00Z", "--states", "normal", "--alerts", writeTempAlerts(t)} + }, "--states"}, + {"bad preexisting", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--preexisting", "bogus", "--alerts", writeTempAlerts(t)} + }, "--preexisting"}, + {"alerts with in", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z", "--in", "some.jsonl", "--alerts", writeTempAlerts(t)} + }, "refused"}, + {"no alerts no in", true, func(t *testing.T) []string { + return []string{"--to", "2026-01-01T00:00:00Z"} + }, "no alert names"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + } else { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + } + var stdout, stderr bytes.Buffer + args := append([]string{"check"}, tt.args(t)...) + code := run(args, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), tt.wantErr) { + t.Fatalf("stderr = %q, want it to contain %q", stderr.String(), tt.wantErr) + } + }) + } +} + +// TestRunCheck_ToInPastNoLog pins §4.2's refusal: a `to` already in the past +// with no recorded log cannot be classified from anything, because nothing +// ever observed the window. +func TestRunCheck_ToInPastNoLog(t *testing.T) { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + code := run([]string{"check", + "--from", "1999-01-01T00:00:00Z", "--to", "2000-01-01T00:00:00Z", + "--alerts", writeTempAlerts(t), + }, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "already passed") { + t.Fatalf("stderr = %q, want the §4.2 refusal", stderr.String()) + } +} + +// TestRunCheck_NoResultOnConfigError pins §20.2: --output json never writes +// to stdout when Check was never reached, because there is no Result to +// encode — only the table (on stderr) can report a configuration failure. +func TestRunCheck_NoResultOnConfigError(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + var stdout, stderr bytes.Buffer + code := run([]string{"check", "--to", "2026-01-01T00:00:00Z", "--output", "json"}, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2", code) + } + if stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/common.go b/grafana-alertcheck/cmd/grafana-alertcheck/common.go new file mode 100644 index 000000000..e605c029b --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/common.go @@ -0,0 +1,108 @@ +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// commonFlags is registerCommon's result: the exactly three flags watch and +// check share (§20). Connection details are never flags (§20.2) and states / +// poll-interval are deliberately NOT here — states is check-only because +// recording is unfiltered (P6), and poll-interval is watch-only because check +// reads the cadence from the log header (P5). Putting either here would +// silently reinstate a knob this plan removed. +type commonFlags struct { + folder *string + concurrency *int + alerts *string +} + +func registerCommon(fs *flag.FlagSet) *commonFlags { + return &commonFlags{ + folder: fs.String("folder", "", "default folder to scope an unqualified alert name to (§17)"), + concurrency: fs.Int("concurrency", 1, "maximum concurrent requests to Grafana"), + alerts: fs.String("alerts", "", "path to a file of alert names, one per line, or - for stdin"), + } +} + +// readAlerts reads §17's alert names, one per line, from a file or from +// stdin when path is "-". An empty path is not an error here — watch and +// check each decide for themselves whether an empty list is allowed +// (log mode never wants one; single-step / record mode always does). +func readAlerts(stdin io.Reader, path string) ([]string, error) { + if path == "" { + return nil, nil + } + var r io.Reader + if path == "-" { + r = stdin + } else { + f, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("read --alerts %s: %w", path, err) + } + defer f.Close() + r = f + } + var lines []string + sc := bufio.NewScanner(r) + for sc.Scan() { + lines = append(lines, sc.Text()) + } + if err := sc.Err(); err != nil { + return nil, fmt.Errorf("read --alerts %s: %w", path, err) + } + return lines, nil +} + +// parseStates parses check's --states flag: a comma-separated list of the +// "bad" state vocabulary Config.States matches against (§13, classify.go's +// badStateSet). An empty string is not resolved here — it means "use the +// library default of {firing}" — so this returns nil, nil for "" rather than +// an error. +// +// normal is deliberately NOT accepted: the v2 plan fixes this vocabulary to +// firing | pending | nodata | error (line 378) precisely because "normal" is +// the good state, never a bad one to classify against. Accepting it here +// would let --states normal turn every healthy instance into a violation and +// fail every healthy fleet — the exact fail-open shape H7 exists to prevent. +func parseStates(s string) ([]gate.State, error) { + if strings.TrimSpace(s) == "" { + return nil, nil + } + var out []gate.State + for _, part := range strings.Split(s, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + switch gate.State(part) { + case gate.StateFiring, gate.StatePending, gate.StateNodata, gate.StateError: + out = append(out, gate.State(part)) + default: + return nil, fmt.Errorf("--states: unknown state %q (want any of: firing, pending, nodata, error)", part) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("--states: %q named no state", s) + } + return out, nil +} + +// parsePreexisting parses check's --preexisting flag (§11.7). +func parsePreexisting(s string) (gate.PreexistingPolicy, error) { + switch gate.PreexistingPolicy(s) { + case "": + return gate.PreexistingFailUnlessRecovered, nil + case gate.PreexistingFailUnlessRecovered, gate.PreexistingFail, gate.PreexistingIgnore: + return gate.PreexistingPolicy(s), nil + default: + return "", fmt.Errorf("--preexisting: unknown policy %q (want one of: fail-unless-recovered, fail, ignore)", s) + } +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/main.go b/grafana-alertcheck/cmd/grafana-alertcheck/main.go index d7968f5a8..73d30f3f3 100644 --- a/grafana-alertcheck/cmd/grafana-alertcheck/main.go +++ b/grafana-alertcheck/cmd/grafana-alertcheck/main.go @@ -1,8 +1,5 @@ -// Command grafana-alertcheck is the CLI entry point for the gate. P3 wires -// only the `list` subcommand — enough to validate auth, the ruler parse, and -// resolution against a real Grafana before any coverage logic exists (§9 rule -// 4, "reach runnable at PR 5"). P10 extends this file with `watch` and -// `check`. +// Command grafana-alertcheck is the CLI entry point for the gate: `list` +// (P3), `watch` (record, P10) and `check` (classify, P10). package main import ( @@ -15,7 +12,7 @@ func main() { os.Exit(run(os.Args[1:], os.Stdout, os.Stderr)) } -const usage = "usage: grafana-alertcheck " +const usage = "usage: grafana-alertcheck " // run is the whole of main's testable surface: parse the subcommand, dispatch, // return the process exit code. Exit codes below 2 (pass/violations) belong to @@ -36,6 +33,10 @@ func run(args []string, stdout, stderr io.Writer) int { switch args[0] { case "list": return runList(args[1:], stdout, stderr) + case "watch": + return runWatch(args[1:], os.Stdin, stdout, stderr) + case "check": + return runCheck(args[1:], os.Stdin, stdout, stderr) case "-h", "-help", "--help": fmt.Fprintln(stdout, usage) return 0 diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table.go b/grafana-alertcheck/cmd/grafana-alertcheck/table.go new file mode 100644 index 000000000..0e4c93c67 --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/table.go @@ -0,0 +1,136 @@ +package main + +import ( + "fmt" + "io" + "sort" + "text/tabwriter" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// renderTable is §20.2's required human table. It always writes to the +// writer it is given, which the caller (runCheck) always points at +// stderr — the human table is not the machine output §20.2 reserves stdout +// for. +// +// Three sections, in order: +// +// 1. one line per rule: outcome, BadFor, pollEvery, proved-or-not with the +// largest gap; +// 2. one line per Violation (R2): a rule's worst-of outcome does not carry +// the State/Health of the instance that actually caused it — Violation +// does — so this is also where those two columns appear, sorted after +// the rule table rather than folded into it, and it is the only place an +// operator running WITHOUT --output json sees the §12.1 --allow-paused +// hint that Violation.Note already carries (classify.go); +// 3. a footer with the per-rule thresholds and the run-wide numbers §20.2 +// says are the answer to "why" on exit 2: each non-skipped rule's +// maxGap/healthGrace/evalStaleAfter, the global transitionGrace and +// drainTimeout, and the largest measured clock skew alongside its own +// error bound (RTT/2) — SkewHardLimit is a separate, fixed input +// threshold and is reported next to it, never as if it were that bound. +func renderTable(w io.Writer, res gate.Result) error { + alertOf := make(map[string]string, len(res.Verdicts)) + for _, v := range res.Verdicts { + alertOf[v.RuleUID] = v.Alert + } + + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "ALERT\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE") + for _, v := range sortedVerdicts(res.Verdicts) { + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", + v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second), + provedLabel(res.Coverage[v.RuleUID]), v.Note) + } + if err := tw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + + if len(res.Violations) > 0 { + fmt.Fprintln(w, "\nVIOLATIONS") + vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(vtw, "ALERT\tOUTCOME\tSTATE\tHEALTH\tNOTE") + for _, v := range sortedViolations(res.Violations) { + fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\n", alertLabel(v, alertOf), v.Outcome, v.State, v.Health, v.Note) + } + if err := vtw.Flush(); err != nil { + return fmt.Errorf("render table: %w", err) + } + } + + fmt.Fprintln(w) + for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) { + t := res.Thresholds[uid] + fmt.Fprintf(w, "rule %s: maxGap=%s healthGrace=%s evalStaleAfter=%s\n", + alertOr(uid, alertOf), t.MaxGap, t.HealthGrace, t.EvalStaleAfter) + } + fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n", + res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout) + fmt.Fprintf(w, "violations: %d, largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n", + len(res.Violations), res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond), + gate.SkewHardLimit, res.GrafanaVersion) + return nil +} + +// provedLabel is the table's PROVED column: "yes" for a clean coverage +// proof, "no" with the reason and largest gap for an unobservable rule, and +// "-" for a rule decide never asked proveCoverage about at all (skipped — +// paused before the window opened, §12). +func provedLabel(cov gate.CoverageResult) string { + if cov.Reason == "" && !cov.Unobservable && !cov.Proved { + return "-" + } + if cov.Unobservable { + if cov.LargestGap > 0 { + return fmt.Sprintf("no (%s; largest gap %s at %s)", cov.Reason, + cov.LargestGap.Round(time.Second), cov.LargestGapAt.Format(time.RFC3339)) + } + return fmt.Sprintf("no (%s)", cov.Reason) + } + return "yes" +} + +// alertLabel resolves a Violation's alert name. Most violations already +// carry it directly; the synthetic MinObserved-shortfall entry with no named +// rule (classify.go) has an empty Alert and an empty RuleUID, so alertOf +// cannot resolve it either — "-" says plainly that this row is not about a +// specific rule. +func alertLabel(v gate.Violation, alertOf map[string]string) string { + if v.Alert != "" { + return v.Alert + } + if a, ok := alertOf[v.RuleUID]; ok { + return a + } + return "-" +} + +func alertOr(uid string, alertOf map[string]string) string { + if a, ok := alertOf[uid]; ok { + return a + } + return uid +} + +func sortedVerdicts(in []gate.RuleVerdict) []gate.RuleVerdict { + out := append([]gate.RuleVerdict(nil), in...) + sort.Slice(out, func(i, j int) bool { return out[i].Alert < out[j].Alert }) + return out +} + +func sortedViolations(in []gate.Violation) []gate.Violation { + out := append([]gate.Violation(nil), in...) + sort.SliceStable(out, func(i, j int) bool { return out[i].Alert < out[j].Alert }) + return out +} + +func sortedThresholdUIDs(thresholds map[string]gate.RuleThresholds, alertOf map[string]string) []string { + uids := make([]string, 0, len(thresholds)) + for uid := range thresholds { + uids = append(uids, uid) + } + sort.Slice(uids, func(i, j int) bool { return alertOr(uids[i], alertOf) < alertOr(uids[j], alertOf) }) + return uids +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go new file mode 100644 index 000000000..106760fbb --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/table_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "bytes" + "strings" + "testing" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +// TestRenderTable is the golden table test: a fixed Result renders a +// deterministic, ordered rule table, a violations section (R2) and a footer +// carrying the per-rule and global thresholds plus the skew and its bound +// (R3) — with no live Check involved. +func TestRenderTable(t *testing.T) { + gapAt := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + res := gate.Result{ + GrafanaVersion: "13.1.0", + ClockSkew: 1500 * time.Millisecond, + ClockSkewBound: 250 * time.Millisecond, + Verdicts: []gate.RuleVerdict{ + {Alert: "Zebra Alert", RuleUID: "uid-z", Outcome: gate.OutcomeClean, PollEvery: 30 * time.Second}, + {Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable, + PollEvery: 30 * time.Second, Note: "gap of 5m0s starting at 2026-01-01T12:00:00Z exceeds maxGap 1m0s"}, + {Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"}, + }, + Violations: []gate.Violation{ + {Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable, State: gate.StateFiring, Health: "error", Note: "unobservable"}, + {Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped, + Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"}, + }, + Coverage: map[string]gate.CoverageResult{ + "uid-z": {Proved: true}, + "uid-a": {Unobservable: true, Reason: gate.ReasonHeartbeatGap, LargestGap: 5 * time.Minute, LargestGapAt: gapAt}, + }, + Thresholds: map[string]gate.RuleThresholds{ + "uid-z": {MaxGap: time.Minute, HealthGrace: time.Minute, EvalStaleAfter: time.Minute}, + "uid-a": {MaxGap: time.Minute, HealthGrace: 2 * time.Minute, EvalStaleAfter: time.Minute}, + }, + Global: gate.GlobalThresholds{ + TransitionGrace: 5 * time.Minute, + GraceSource: `Ape Alert (for=5m)`, + DrainTimeout: 2 * time.Minute, + }, + } + + var buf bytes.Buffer + if err := renderTable(&buf, res); err != nil { + t.Fatalf("renderTable: %v", err) + } + out := buf.String() + + // Rule table: Ape sorts before Zebra sorts before... Paused is skipped and + // carries no coverage entry, so it renders "-" for PROVED. + if !strings.Contains(out, "Ape Alert") || !strings.Contains(out, "unobservable") { + t.Fatalf("out = %q, want Ape's unobservable row", out) + } + if !strings.Contains(out, "heartbeat_gap") || !strings.Contains(out, "largest gap 5m0s") { + t.Fatalf("out = %q, want the coverage reason and largest gap", out) + } + if !strings.Contains(out, "Zebra Alert") || !strings.Contains(out, "clean") { + t.Fatalf("out = %q, want Zebra's clean row", out) + } + + // Violations section (R2): must show up even without --output json, and + // must carry the §12.1 --allow-paused hint text verbatim. + if !strings.Contains(out, "VIOLATIONS") { + t.Fatalf("out = %q, want a VIOLATIONS section", out) + } + if !strings.Contains(out, "--allow-paused") { + t.Fatalf("out = %q, want the §12.1 --allow-paused hint in the human table", out) + } + if !strings.Contains(out, "STATE") || !strings.Contains(out, "HEALTH") { + t.Fatalf("out = %q, want the violations table to have STATE and HEALTH columns", out) + } + if !strings.Contains(out, string(gate.StateFiring)) || !strings.Contains(out, "error") { + t.Fatalf("out = %q, want Ape's violation State/Health", out) + } + + // Footer (R3): per-rule thresholds, global thresholds, and skew with its + // own bound rather than the fixed hard limit. + if !strings.Contains(out, "Ape Alert: maxGap=1m0s healthGrace=2m0s evalStaleAfter=1m0s") { + t.Fatalf("out = %q, want Ape's per-rule thresholds", out) + } + if !strings.Contains(out, "Zebra Alert: maxGap=1m0s healthGrace=1m0s evalStaleAfter=1m0s") { + t.Fatalf("out = %q, want Zebra's per-rule thresholds", out) + } + if strings.Contains(out, "Paused Alert: maxGap") { + t.Fatalf("out = %q, a skipped rule must not report thresholds it never had (§12)", out) + } + if !strings.Contains(out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s") { + t.Fatalf("out = %q, want the global thresholds line", out) + } + if !strings.Contains(out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)") { + t.Fatalf("out = %q, want the skew and its own bound, not the hard limit misused as one", out) + } + if !strings.Contains(out, "violations: 2") { + t.Fatalf("out = %q, want the violation count", out) + } + if !strings.Contains(out, "13.1.0") { + t.Fatalf("out = %q, want the grafana version", out) + } +} + +// TestProvedLabel_Skipped pins the "-" case: a rule decide never asked +// proveCoverage about (paused before the window opened, §12) has an empty +// CoverageResult and must not be reported as either proved or unobservable. +func TestProvedLabel_Skipped(t *testing.T) { + if got := provedLabel(gate.CoverageResult{}); got != "-" { + t.Fatalf("provedLabel(zero value) = %q, want \"-\"", got) + } +} diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch.go b/grafana-alertcheck/cmd/grafana-alertcheck/watch.go new file mode 100644 index 000000000..f3ad57d92 --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/watch.go @@ -0,0 +1,148 @@ +package main + +import ( + "context" + "flag" + "fmt" + "io" + "time" + + "github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate" +) + +const watchUsage = "usage: grafana-alertcheck watch --out [--pidfile F] [--daemon-log F] " + + "--alerts [--folder F] [--poll-interval D] [--concurrency N] [--until RFC3339]" + +// runWatch is the record step's entire CLI surface, split in two by one flag +// set — gate.DaemonChildFlag ("--daemon-child") and gate.ReadyFDFlag +// ("--ready-fd") select which side of P6's parent/child split this +// invocation is: +// +// - without them: the record command an operator types. It parses --out, +// --alerts and the rest, builds a gate.WatchConfig and calls gate.Watch, +// which resolves, records the first observation of every rule, and +// detaches the recorder before returning (§4.3). +// - with them: the detached recorder itself. gate.Watch's own childArgs +// (watch_unix.go) is the only thing that ever sets them — an operator +// never types "--daemon-child" and it does not appear in watchUsage — +// and this dispatches straight to gate.RunDaemonChild. P6's integration +// test already covers the spawn; this is the one new test P10 owns: that +// seeing the flag reaches RunDaemonChild. +// +// Both flags live in the SAME flag set as the operator-facing ones rather +// than a second, hidden set: the child is started with childArgs' exact +// argv, e.g. "watch --daemon-child --out log.jsonl --ready-fd 3 +// [--until ...] [--concurrency ...]", and a second parser would have to stay +// byte-for-byte in sync with that slice to accept it. +func runWatch(args []string, stdin io.Reader, stdout, stderr io.Writer) int { + fs := flag.NewFlagSet("watch", flag.ContinueOnError) + fs.SetOutput(stderr) + fs.Usage = func() { fmt.Fprintln(stderr, watchUsage) } + + common := registerCommon(fs) + out := fs.String("out", "", "JSONL log path to record to") + pidfile := fs.String("pidfile", "", "pidfile path (default .pid)") + daemonLog := fs.String("daemon-log", "", "stdout/stderr sink for the detached recorder (default .daemon.log)") + until := fs.String("until", "", "optional hard stop, RFC3339 (default: run until check stops it)") + pollInterval := fs.String("poll-interval", "", "override every rule's poll cadence (default: half its own evaluation interval)") + + // Hidden: never in watchUsage, never typed by an operator (see doc comment). + daemonChild := fs.Bool(gate.DaemonChildFlag[2:], false, "") + readyFD := fs.Int(gate.ReadyFDFlag[2:], 0, "") + + if err := fs.Parse(args); err != nil { + return 2 + } + + if *daemonChild { + return runDaemonChild(*out, *until, *common.concurrency, *readyFD, stderr) + } + + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + alerts, err := readAlerts(stdin, *common.alerts) + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + + cfg := gate.WatchConfig{ + URL: url, + Token: token, + Alerts: alerts, + Folder: *common.folder, + Out: *out, + PidFile: *pidfile, + DaemonLog: *daemonLog, + Concurrency: *common.concurrency, + Clock: gate.SystemClock{}, + Notes: stderr, + } + if *until != "" { + t, err := time.Parse(time.RFC3339, *until) + if err != nil { + fmt.Fprintf(stderr, "--until: %v\n", err) + return 2 + } + cfg.Until = t + } + if *pollInterval != "" { + d, err := time.ParseDuration(*pollInterval) + if err != nil { + fmt.Fprintf(stderr, "--poll-interval: %v\n", err) + return 2 + } + cfg.PollEvery = d + } + + if err := gate.Watch(context.Background(), cfg); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + return 0 +} + +// runDaemonChild is the detached recorder's whole entry point (P6's +// obligation on this phase). Its stdout and stderr are already the daemon +// log file — spawnChild (watch_unix.go) redirects both before Start — so +// writing to stderr here lands exactly where waitForChildReady's failure +// path quotes from. +func runDaemonChild(out, until string, concurrency, readyFD int, stderr io.Writer) int { + url, token, err := grafanaEnv() + if err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + cfg := gate.DaemonChildConfig{ + URL: url, + Token: token, + Out: out, + Concurrency: concurrency, + Clock: gate.SystemClock{}, + ReadyFD: readyFD, + } + if until != "" { + t, err := time.Parse(time.RFC3339, until) + if err != nil { + fmt.Fprintf(stderr, "--until: %v\n", err) + return 2 + } + cfg.Until = t + } + if err := gate.RunDaemonChild(context.Background(), cfg); err != nil { + fmt.Fprintln(stderr, err) + return 2 + } + return 0 +} + +// The daemon-child and ready-fd flags registered above must keep matching +// watch_unix.go's childArgs, which names exactly --daemon-child, --out, +// --ready-fd, --until and --concurrency and nothing else: that function +// builds this process's own argv when it re-execs itself as the detached +// recorder, so a flag added to one side without the other means the child +// fails on its very first flag.Parse. diff --git a/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go b/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go new file mode 100644 index 000000000..267a312a2 --- /dev/null +++ b/grafana-alertcheck/cmd/grafana-alertcheck/watch_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "bytes" + "os" + "strings" + "testing" +) + +// TestRunWatch_FlagValidation is the record step's flag-validation matrix. +// Every case fails inside gate.WatchConfig.validate() (P6) or before it, so +// none needs a reachable Grafana. +func TestRunWatch_FlagValidation(t *testing.T) { + tests := []struct { + name string + env bool + args func(t *testing.T) []string + wantErr string + }{ + {"missing env", false, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t)} + }, "GRAFANA_URL"}, + {"missing out", true, func(t *testing.T) []string { + return []string{"--alerts", writeTempAlerts(t)} + }, "no log path"}, + {"missing alerts", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl"} + }, "no alert names"}, + {"bad until format", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--until", "not-a-time"} + }, "--until"}, + {"until in the past", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--until", "2000-01-01T00:00:00Z"} + }, "not in the future"}, + {"bad poll-interval", true, func(t *testing.T) []string { + return []string{"--out", t.TempDir() + "/log.jsonl", "--alerts", writeTempAlerts(t), "--poll-interval", "not-a-duration"} + }, "--poll-interval"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.env { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + } else { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + } + var stdout, stderr bytes.Buffer + args := append([]string{"watch"}, tt.args(t)...) + code := run(args, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), tt.wantErr) { + t.Fatalf("stderr = %q, want it to contain %q", stderr.String(), tt.wantErr) + } + }) + } +} + +// TestRunWatch_DaemonChildDispatch pins P6's obligation on this phase: seeing +// gate.DaemonChildFlag must dispatch to gate.RunDaemonChild, and the flag +// must never appear in watchUsage (an operator never types it). +func TestRunWatch_DaemonChildDispatch(t *testing.T) { + t.Setenv("GRAFANA_URL", "http://example.invalid") + t.Setenv("GRAFANA_TOKEN", "test-token") + + var stdout, stderr bytes.Buffer + // No log at this path: RunDaemonChild fails trying to read it, which is + // enough to prove dispatch happened without needing a real recording. + missing := os.DevNull + ".missing" + code := run([]string{"watch", "--daemon-child", "--out", missing, "--ready-fd", "0"}, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), missing) { + t.Fatalf("stderr = %q, want RunDaemonChild's read failure naming %q", stderr.String(), missing) + } + if strings.Contains(watchUsage, "daemon-child") { + t.Fatalf("watchUsage = %q, must never name --daemon-child", watchUsage) + } + if strings.Contains(watchUsage, "ready-fd") { + t.Fatalf("watchUsage = %q, must never name --ready-fd", watchUsage) + } +} + +func TestRunWatch_DaemonChild_MissingEnv(t *testing.T) { + t.Setenv("GRAFANA_URL", "") + t.Setenv("GRAFANA_TOKEN", "") + + var stdout, stderr bytes.Buffer + code := run([]string{"watch", "--daemon-child", "--out", "log.jsonl"}, &stdout, &stderr) + if code != 2 { + t.Fatalf("code = %d, want 2; stderr = %q", code, stderr.String()) + } + if !strings.Contains(stderr.String(), "GRAFANA_URL") { + t.Fatalf("stderr = %q, want it to name the missing env var", stderr.String()) + } +} diff --git a/grafana-alertcheck/internal/gate/check.go b/grafana-alertcheck/internal/gate/check.go index 6c3bf8805..710bedef9 100644 --- a/grafana-alertcheck/internal/gate/check.go +++ b/grafana-alertcheck/internal/gate/check.go @@ -21,8 +21,9 @@ import ( // not return a code, because a code is a presentation decision and the // library must not make it. // - §12.1 wants the paused rule AND --allow-paused both named to the user. -// decide names the rule in the shortfall Violation's Note (classify.go); -// the flag hint belongs to the CLI's renderer. +// decide names both in the shortfall Violation's Note (classify.go), and +// P10's renderer prints every Violation, including Note, in the human +// table — not only in --output json. // - Config.Notes carries the running commentary (§13.2's planned run time, // the countdown, the blind-interval warning). §20.2 puts the human output // on stderr and reserves stdout for --output json, so the CLI must pass diff --git a/grafana-alertcheck/internal/gate/classify.go b/grafana-alertcheck/internal/gate/classify.go index c6b32fc25..d15abaeba 100644 --- a/grafana-alertcheck/internal/gate/classify.go +++ b/grafana-alertcheck/internal/gate/classify.go @@ -104,6 +104,28 @@ type Policy struct { From, To time.Time } +// RuleThresholds is one non-skipped rule's resolved coverage thresholds +// (§5/§10.1/§14.1), carried on Result so the CLI's table (P10, §20.2) can +// print the numbers that answer "why" on exit 2 without decide exposing the +// unexported ruleTimings type itself. +type RuleThresholds struct { + MaxGap time.Duration + HealthGrace time.Duration + EvalStaleAfter time.Duration +} + +// GlobalThresholds is the run-wide half of the same information (§13.1, +// §19): transitionGrace and drainTimeout apply once, across every +// non-skipped watched rule, not per rule (globalTimings). +type GlobalThresholds struct { + TransitionGrace time.Duration + // GraceSource names, and already carries the `for` value of, the rule + // that set TransitionGrace (§13.2 requires printing both). "none" when no + // rule contributed (TransitionGrace is then 0). + GraceSource string + DrainTimeout time.Duration +} + // Result is decide's whole answer: everything §20.2's table and the action's // JSON outputs need. Coverage carries one CoverageResult per non-skipped // rule — no separate Interval type anywhere in the project (§2's @@ -112,9 +134,22 @@ type Result struct { From, To time.Time GrafanaVersion string ClockSkew time.Duration // the largest |skew| across every poll decide was given, not only the ones a rule's window actually used + // ClockSkewBound is the skew BOUND (RTT/2, §16) of that SAME poll — not + // the largest bound seen overall, which would pair a wide bound from an + // unrelated slow request with the worst skew and misstate how tightly + // that skew is actually known. SkewHardLimit is a separate, fixed input + // validation threshold (source.go) and is not an error bound on this + // value; the CLI prints both, but must not conflate them. + ClockSkewBound time.Duration Coverage map[string]CoverageResult - Verdicts []RuleVerdict - Violations []Violation + // Thresholds carries one RuleThresholds per rule Coverage also covers — + // every non-skipped rule, keyed by UID. A skipped rule has neither: it + // was never scheduled, so it has no maxGap/healthGrace/evalStaleAfter to + // report (§12). + Thresholds map[string]RuleThresholds + Global GlobalThresholds + Verdicts []RuleVerdict + Violations []Violation } // episode is one contiguous, policy-bad span of one instance's timeline, @@ -207,7 +242,7 @@ func classifyRule(def Definition, polls []Poll, from, windowEnd time.Time, badSt } // Different polls can carry different measured skews. In theory a // closing poll's translated time could land before the opening - // poll's — skew is capped at skewHardLimit (60s), so this is remote, + // poll's — skew is capped at SkewHardLimit (60s), so this is remote, // not impossible — and a negative span would feed mergeDurations a // duration that subtracts instead of adds. Clamp rather than trust // the arithmetic never to invert. @@ -482,14 +517,29 @@ func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, To: pol.To, GrafanaVersion: h.GrafanaVersion, Coverage: make(map[string]CoverageResult), + Thresholds: make(map[string]RuleThresholds), + Global: GlobalThresholds{ + TransitionGrace: gt.transitionGrace, + GraceSource: gt.graceSource, + DrainTimeout: gt.drainTimeout, + }, } + skewSeen := false for _, p := range polls { s := p.Skew() if s < 0 { s = -s } - if s > result.ClockSkew { + // The bound travels with ITS OWN poll's skew, never the largest bound + // seen overall (Result.ClockSkewBound's doc comment) — so it is only + // ever overwritten in lockstep with ClockSkew, on the same poll. >= + // rather than > on top of skewSeen: a strict > would never assign the + // bound at all when every poll's skew is exactly 0, understating the + // real measurement uncertainty as an unearned "bound ±0s". + if !skewSeen || s > result.ClockSkew { result.ClockSkew = s + result.ClockSkewBound = p.SkewBound() + skewSeen = true } } @@ -545,6 +595,11 @@ func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, } } result.Coverage[def.UID] = cov + result.Thresholds[def.UID] = RuleThresholds{ + MaxGap: t.maxGap, + HealthGrace: t.healthGrace, + EvalStaleAfter: t.evalStaleAfter, + } outcome, badFor, viols := classifyRule(def, polls, pol.From, windowEnd, badStates, pol.Preexisting) if cov.Unobservable { @@ -581,9 +636,9 @@ func decide(h Header, polls []Poll, sentinel *time.Time, defs []Definition, break } // §12.1 requires the paused rule and --allow-paused both be - // named to the user; naming the rule is this Violation's job, - // the --allow-paused hint is the CLI table/renderer's (P10) — - // tracked here so it is not dropped when that phase is built. + // named to the user; both live in this one Violation, in Note — + // P10's renderer prints Note verbatim rather than re-deriving + // the hint, so the exact wording here is what an operator reads. result.Violations = append(result.Violations, Violation{ Alert: def.Title, RuleUID: def.UID, Outcome: OutcomeSkipped, Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set", diff --git a/grafana-alertcheck/internal/gate/schedule.go b/grafana-alertcheck/internal/gate/schedule.go index 87b014375..32fd93df4 100644 --- a/grafana-alertcheck/internal/gate/schedule.go +++ b/grafana-alertcheck/internal/gate/schedule.go @@ -8,16 +8,18 @@ import ( "time" ) -// skewHardLimit is one of §5's filled-in values (basis: §16; §22.11 asserts +// SkewHardLimit is one of §5's filled-in values (basis: §16; §22.11 asserts // 120s errors, 30s does not). Defined here, in schedule.go's named-constants // block, per §5's instruction — it moved out of source.go now that P4 exists; -// P2 needed it before this file did, so it started there. -const skewHardLimit = 60 * time.Second +// P2 needed it before this file did, so it started there. Exported (P10) so +// the CLI can report it verbatim next to a measured skew instead of keeping +// its own mirrored copy. +const SkewHardLimit = 60 * time.Second // fromFutureTolerance is how far ahead of the runner's own clock a supplied // `from` may sit before check refuses it (§7: "from in the future, more than // the skew tolerance — error"). §7 names no number, so this is the judgment -// call §5's table records: the same 60s as skewHardLimit, because the only +// call §5's table records: the same 60s as SkewHardLimit, because the only // legitimate reason for a `from` in the future is clock disagreement between // the deploy step and the check step, and that is bounded by the same figure. // It is once-per-run input validation, not a per-rule coverage check, so diff --git a/grafana-alertcheck/internal/gate/source.go b/grafana-alertcheck/internal/gate/source.go index 656c80558..58008087f 100644 --- a/grafana-alertcheck/internal/gate/source.go +++ b/grafana-alertcheck/internal/gate/source.go @@ -263,7 +263,7 @@ type requestResult struct { // doRequest performs one HTTP GET and classifies the outcome (§14.5, §16): // a network failure, a non-2xx status, or a body-read failure is retryable // (*TransportError); a missing or unparseable Date header, or a skew beyond -// skewHardLimit, is a hard error — retrying can never fix either, so neither +// SkewHardLimit, is a hard error — retrying can never fix either, so neither // may enter the backoff loop (H4). // // The Date-header/skew check runs for every endpoint this hits, including @@ -317,8 +317,8 @@ func (s *httpSource) doRequest(ctx context.Context, path string) (requestResult, if absSkew < 0 { absSkew = -absSkew } - if absSkew > skewHardLimit { - return requestResult{}, fmt.Errorf("%s: clock skew %s exceeds hard limit %s (§16)", path, absSkew, skewHardLimit) + if absSkew > SkewHardLimit { + return requestResult{}, fmt.Errorf("%s: clock skew %s exceeds hard limit %s (§16)", path, absSkew, SkewHardLimit) } return requestResult{Body: b, ServerDate: serverDate, Skew: signedSkew, SkewBound: bound, Latency: latency}, nil