diff --git a/packages/viewer/e2e/overlap-baseline.json b/packages/viewer/e2e/overlap-baseline.json new file mode 100644 index 0000000..1cacc00 --- /dev/null +++ b/packages/viewer/e2e/overlap-baseline.json @@ -0,0 +1,26 @@ +{ + "_comment": [ + "Known-failing boards for the rendered-geometry overlap guard. Every board NOT listed here must", + "render clean; a new name appearing in a run is a regression and fails CI. This exists because the", + "guard had been red on master since the SGCR migration, which meant it gated nothing at all — three", + "PRs merged on top of a red guard without anyone being able to tell whether they made it worse.", + "", + "Entries are board-level, not violation-level, on purpose: the exact node ids differ between macOS", + "and Linux CI (text metrics move the layout), so pinning ids would fail constantly off-CI. The", + "`observed` lists are documentation of what CI saw when the entry was added, not an assertion —", + "read them when a listed board changes to judge whether it got better or worse.", + "", + "This list is a debt ledger and should only ever shrink. Delete a board the moment it renders clean", + "on CI; the guard prints a reminder when a listed board passes." + ], + "_source": "CI run 31290712587 (master @ fb6ff4a, ubuntu-latest)", + "known_failing": { + "ex_architecture-zones": { "observed": ["node-node:web|bff", "edge-over-node:worker"] }, + "ex_data-lineage": { "observed": ["edge-over-node:events"] }, + "ex_dataeng-etl": { "observed": ["edge-over-node:kafka"] }, + "ex_k8s-topology": { "observed": ["edge-over-node:ingress,cm"] }, + "ex_support-escalation": { "observed": ["edge-over-node:created,triage,l2inv"] }, + "ex_swimlane": { "observed": ["edge-over-node:checkout"] }, + "syn_groups_long": { "observed": ["edge-over-node:a1,a2"] } + } +} diff --git a/packages/viewer/e2e/overlap.e2e.mjs b/packages/viewer/e2e/overlap.e2e.mjs index 4ce0837..1e169db 100644 --- a/packages/viewer/e2e/overlap.e2e.mjs +++ b/packages/viewer/e2e/overlap.e2e.mjs @@ -5,8 +5,13 @@ // stressors, each rendered twice to prove the layout is deterministic (not flaky). // // npm run test:overlap (build first) | test:overlap:nobuild (dist already built) -// Exit 0 = all clean + deterministic, 1 = a failure. The deliberately-pathological topologies -// (skip-rank edges, cycles, self-loops) live in the one-off measurement harness, not here. +// The deliberately-pathological topologies (skip-rank edges, cycles, self-loops) live in the +// one-off measurement harness, not here. +// +// Boards listed in overlap-baseline.json are known-bad and don't fail the build; everything else +// must render clean. Exit 0 = no board outside the baseline broke, 1 = a regression. Violations are +// always printed in full, baselined or not, so a known-bad board getting WORSE is visible in review +// even though it doesn't flip the exit code. import { chromium } from "playwright"; import { spawn } from "node:child_process"; import { readFileSync, readdirSync } from "node:fs"; @@ -22,6 +27,10 @@ const BASE = `http://127.0.0.1:${PORT}`; const WSID = "overlap"; const U = `${BASE}/w/${WSID}/`; const RUNS = 2; // each case rendered twice → determinism check +// Boards already known to render with overlaps. They still run and still print their violations; +// they just don't fail the build. Anything NOT listed must be clean. See the file's own _comment. +const BASELINE_REL = "e2e/overlap-baseline.json"; +const BASELINE = JSON.parse(readFileSync(join(HERE, "overlap-baseline.json"), "utf8")).known_failing; const lbl = (s) => ({ data: { label: s } }); // A few CLEAN stressors (the risk areas: long labels in zones, swimlanes, wide fan-out). @@ -75,7 +84,8 @@ const push = (a, t, c) => fetch(`${U}push`, { method: "POST", headers: { "conten const srv = spawn("node", [join(VIEWER, "dist", "server.js")], { env: { ...process.env, PORT: String(PORT), PUSH_TOKEN: TOKEN }, stdio: ["ignore", "ignore", "inherit"] }); let browser; -const fails = []; +const fails = []; // boards that broke and are NOT in the baseline → CI failure +const fixed = []; // boards in the baseline that now render clean → prompt to shrink the ledger try { await waitUp(); for (const c of cases) { const r = await push(c.name, c.type, c.content); if (!r.ok) throw new Error(`push ${c.name} -> ${r.status}`); } @@ -98,13 +108,22 @@ try { } const m = runs[0]; const issues = []; - if (m.nn.length) issues.push(`node-node:${m.nn.slice(0, 3).join(",")}`); - if (m.cc.length) issues.push(`container:${m.cc.slice(0, 3).join(",")}`); - if (m.sp.length) issues.push(`spillout:${m.sp.slice(0, 3).join(",")}`); - if (m.en.length) issues.push(`edge-over-node:${m.en.slice(0, 3).join(",")}`); - if (!runs.every((r) => r.sig === m.sig)) issues.push("FLAKY (geometry differs across runs)"); - const ok = issues.length === 0; - console.log(`${ok ? "PASS" : "FAIL"} ${c.name} (leaves=${m.leaves})${ok ? "" : " → " + issues.join("; ")}`); + // Full lists, not the first 3: when a known-failing board changes, the detail is the only way + // to tell "same breakage" from "got worse", and a truncated list hides the difference. + if (m.nn.length) issues.push(`node-node:${m.nn.join(",")}`); + if (m.cc.length) issues.push(`container:${m.cc.join(",")}`); + if (m.sp.length) issues.push(`spillout:${m.sp.join(",")}`); + if (m.en.length) issues.push(`edge-over-node:${m.en.join(",")}`); + const flaky = !runs.every((r) => r.sig === m.sig); + if (flaky) issues.push("FLAKY (geometry differs across runs)"); + const clean = issues.length === 0; + const baselined = Object.hasOwn(BASELINE, c.name); + // Flakiness is never excusable: a baseline entry records a bad-but-STABLE layout, and a board + // whose geometry moves between runs can't be reasoned about at all. + const ok = clean || (baselined && !flaky); + const tag = clean ? (baselined ? "FIXED" : "PASS") : ok ? "KNOWN" : "FAIL"; + console.log(`${tag} ${c.name} (leaves=${m.leaves})${clean ? "" : " → " + issues.join("; ")}`); + if (clean && baselined) fixed.push(c.name); if (!ok) fails.push(c.name); } } catch (e) { @@ -114,5 +133,15 @@ try { if (browser) await browser.close(); srv.kill("SIGTERM"); } -console.log(`\n=== ${cases.length - fails.length}/${cases.length} clean & deterministic ===`); +const knownCount = cases.filter((c) => Object.hasOwn(BASELINE, c.name)).length - fixed.length; +console.log(`\n=== ${cases.length - fails.length - knownCount}/${cases.length} clean, ${knownCount} known-bad, ${fails.length} regressed ===`); +if (fixed.length) { + console.log(`\n${fixed.length} board(s) now render clean: ${fixed.join(", ")}`); + console.log(`Remove them from ${BASELINE_REL} so they stay fixed — the ledger only shrinks.`); +} +if (fails.length) { + console.log(`\nREGRESSED (not in the baseline): ${fails.join(", ")}`); + console.log(`These boards are expected to render with zero overlaps. Fix the layout, or — if this`); + console.log(`is a deliberate, reviewed trade — add the board to ${BASELINE_REL} with a reason.`); +} process.exit(fails.length ? 1 : 0);