chore: merge Batch A ACKs (#67 #61 #62 #64 #48)

CHANGELOG.md

@@ -52,6 +52,7 @@ Format: version sections are listed newest first.
 ### Added
 - **EXL3 live tok/s** — detect ExLlamaV3 `tools/serve_openai.py` (`owned_by: exl3` or `/health` `{ok, busy}`) instead of mislabeling it as vLLM. Generation and prefill tok/s come from `/health` cumulative token counters (no Prometheus `/metrics`).
 - **Tailnet monitoring** — opt-in per unit (`tailscaleMonitoring`, default **off**); `tailscale status --json` on the host and a Tailnet card under Resources. Flags a unit that is healthy on the LAN but off its tailnet. ([#43](https://github.com/MiaAI-Lab/sparkDash/pull/43))
+- **NV_ERR_NO_MEMORY on the GPU panel** — count of NVRM `NV_ERR_NO_MEMORY` kernel log lines since boot (shown when > 0). Journal is scanned at most once a minute, not on the 2s poll. Replaces the approach in [#40](https://github.com/MiaAI-Lab/sparkDash/pull/40).
 
 ### Security
 - **`BIND_HOST` now defaults to `127.0.0.1` (loopback) instead of `0.0.0.0`** — the dashboard is unauthenticated and can SSH into and power off Sparks, so it is no longer reachable on the LAN by default. Set `BIND_HOST` to the host's LAN IP (or `0.0.0.0`) to opt in to remote access. **Migration:** if you access sparkDash from another machine via bare-metal `npm start`, set `BIND_HOST` explicitly. Production and dev Compose both set `BIND_HOST=0.0.0.0` (`network_mode: host`). Startup now also warns when bound to a non-loopback address. ([#35](https://github.com/MiaAI-Lab/sparkDash/pull/35))

Dockerfile

@@ -51,6 +51,7 @@ COPY --from=builder /app/package.json ./package.json
 COPY --from=builder /app/package-lock.json ./package-lock.json
 COPY --from=builder /app/server ./server
 COPY --from=builder /app/src/shared ./src/shared
+COPY --from=builder /app/src/components/ShowcasePage/showcasePrompts.ts ./src/components/ShowcasePage/showcasePrompts.ts
 COPY --from=builder /app/config ./config
 
 # Volume for persistent sparks.json

README.md

@@ -398,6 +398,7 @@ Copy `.env.example` to `.env` if needed:
 | `POLL_INTERVAL_HERMES` | `600000` | Hermes Agent update check poll (ms) |
 | `POLL_INTERVAL_TAILSCALE` | `30000` | Tailnet probe poll (ms) |
 | `TAILSCALE_PROBE_TIMEOUT_MS` | `8000` | Timeout for `tailscale status --json` (ms) |
+| `POLL_INTERVAL_NVERR` | `60000` | Kernel journal scan for NVRM `NV_ERR_NO_MEMORY` (ms) |
 | `HERMES_UPDATE_TIMEOUT_MS` | `600000` | Hard timeout for running `hermes update` over SSH (ms) |
 | `POLL_INTERVAL_LIVENESS` | `5000` | Online/SSH liveness check (ms) |
 | `SPARKDASH_SECRETS_KEY` | _(auto)_ | Passphrase or 64-char hex for secret encryption |

server/collectors/SystemCollector.js

@@ -1,9 +1,24 @@
 import fs from "fs";
 import path from "path";
-import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS } from "../config.js";
+import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS, POLL_INTERVAL_NVERR } from "../config.js";
 import { normalizeMac, WOL_INTERFACE } from "../wol.js";
 import { sshExec } from "./ssh.js";
 
+const NVERR_JOURNAL_CMD =
+  'journalctl -k --no-pager -q --grep=NV_ERR_NO_MEMORY 2>/dev/null | grep -c NV_ERR_NO_MEMORY || true';
+
+/**
+ * Parse `grep -c` stdout into a non-negative integer. Exported for tests.
+ * @param {unknown} raw
+ * @returns {number}
+ */
+export function parseNvErrNoMemoryCount(raw) {
+  const line = String(raw ?? "").trim().split("\n").pop() ?? "";
+  const n = Number.parseInt(line, 10);
+  if (!Number.isFinite(n) || n < 0) return 0;
+  return n;
+}
+
 /**
  * SystemCollector — collects hardware metrics for a Spark.
  * In Phase 2, this is the LOCAL path only (no SSH).
@@ -32,6 +47,8 @@ export class SystemCollector {
 
     // Cached hardware info
     this._hardwareInfo = null;
+    /** Cached NVRM NV_ERR_NO_MEMORY count (slow journal scan). */
+    this._nvErrCache = { count: 0, at: 0 };
   }
 
   /** Collect GPU metrics (temperature, usage, power, VRAM). */
@@ -157,6 +174,7 @@ export class SystemCollector {
       vram,
       processes,
       throttle: gpu.throttle,
+      nvErrNoMemory: await this._nvErrNoMemory(),
     };
   }
 
@@ -994,6 +1012,7 @@ export class SystemCollector {
         vram: { used: usedMB, total: totalMB, percentage, available: availableMB },
         processes,
         throttle: gpu.throttle,
+        nvErrNoMemory: await this._nvErrNoMemory(),
       };
     } catch (err) {
       console.error(`[SystemCollector] Remote GPU error for ${this.spark.id}:`, err.message);
@@ -1448,7 +1467,7 @@ export class SystemCollector {
         });
       }
     }
-    return this._readHostFile(`/proc/net/${relPath}`);
+    return fs.readFileSync(`/proc/net/${relPath}`, "utf-8");
   }
 
   /** Lightweight liveness for local Sparks. */
@@ -1499,6 +1518,34 @@ export class SystemCollector {
     return fs.promises.statfs(dir);
   }
 
+  /**
+   * Count NVRM `NV_ERR_NO_MEMORY` lines in the kernel journal since boot.
+   * Cached for POLL_INTERVAL_NVERR — never on the 2s GPU/memory loop uncached.
+   * @returns {Promise<number>}
+   */
+  async _nvErrNoMemory() {
+    const now = Date.now();
+    if (this._nvErrCache.at > 0 && now - this._nvErrCache.at < POLL_INTERVAL_NVERR) {
+      return this._nvErrCache.count;
+    }
+    try {
+      let out;
+      if (this.spark.isLocal) {
+        out = this._hasHostProc()
+          ? await this._execOnHost(NVERR_JOURNAL_CMD)
+          : await this._exec(NVERR_JOURNAL_CMD);
+      } else {
+        out = await sshExec(this.spark, NVERR_JOURNAL_CMD, { timeoutMs: 8000 });
+      }
+      const count = parseNvErrNoMemoryCount(out);
+      this._nvErrCache = { count, at: now };
+      return count;
+    } catch {
+      this._nvErrCache.at = now;
+      return this._nvErrCache.count;
+    }
+  }
+
   // ─── Default metrics ─────────────────────────────────────
   _defaultGpu() {
     return {
@@ -1508,6 +1555,7 @@ export class SystemCollector {
       vram: { used: 0, total: 0, percentage: 0, available: 0 },
       processes: [],
       throttle: this._defaultThrottle(),
+      nvErrNoMemory: 0,
     };
   }
 

server/collectors/__tests__/LlmProbe.exl3.test.js

@@ -121,13 +121,15 @@ test("_applyExl3Health: counter diffs → tok/s; idle → 0", () => {
   assert.equal(probe.prefillTps, 40);
 });
 
-test("probe: exl3 path does not mislabel as vllm", async () => {
+test("probe: exl3 path does not mislabel as vllm", async (t) => {
+  const now = 10_000;
+  t.mock.method(Date, "now", () => now);
   const probe = new LlmProbe({ lanIp: "127.0.0.1" }, 8888);
   probe.serverIsOpenAI = true;
   probe.backendType = "exl3";
   probe.authOpen = true;
-  probe._lastDetectAt = Date.now();
-  probe.lastProbeTime = Date.now() - 2000;
+  probe._lastDetectAt = now;
+  probe.lastProbeTime = now - 2000;
   probe.lastTokenCounts = { input: 100, output: 50 };
   probe._fetch = async (url) => {
     const u = String(url);

server/collectors/__tests__/LlmProbe.vllm-llamacpp.regression.test.js

@@ -36,6 +36,12 @@ function textRes(txt, status = 200) {
   };
 }
 
+function freezeProbeClock(t) {
+  const now = 10_000;
+  t.mock.method(Date, "now", () => now);
+  return now;
+}
+
 test("vLLM detect: /v1/models + vllm /metrics → vllm (not ds4/sglang)", async () => {
   const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8000);
   const hits = [];
@@ -58,13 +64,14 @@ test("vLLM detect: /v1/models + vllm /metrics → vllm (not ds4/sglang)", async
   assert.ok(!hits.includes("/get_server_info") || hits.includes("/metrics"));
 });
 
-test("vLLM probe: counter diffs + tiles; skips get_server_info when known vllm", async () => {
+test("vLLM probe: counter diffs + tiles; skips get_server_info when known vllm", async (t) => {
+  const now = freezeProbeClock(t);
   const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8000);
   probe.serverIsOpenAI = true;
   probe.backendType = "vllm";
   probe.authOpen = true;
-  probe._lastDetectAt = Date.now();
-  probe.lastProbeTime = Date.now() - 2000;
+  probe._lastDetectAt = now;
+  probe.lastProbeTime = now - 2000;
   probe.lastTokenCounts = { input: 1000, output: 500 };
   const hits = [];
   probe._fetch = async (url) => {
@@ -219,13 +226,14 @@ test("llama.cpp detect: /slots array wins over OpenAI paths", async () => {
   assert.equal(probe.backendType, "llama.cpp");
 });
 
-test("llama.cpp probe: slot deltas → tok/s; props for model", async () => {
+test("llama.cpp probe: slot deltas → tok/s; props for model", async (t) => {
+  const now = freezeProbeClock(t);
   const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080);
   probe.serverIsOpenAI = false;
   probe.backendType = "llama.cpp";
   probe.authOpen = true;
-  probe._lastDetectAt = Date.now();
-  probe.lastProbeTime = Date.now() - 2000;
+  probe._lastDetectAt = now;
+  probe.lastProbeTime = now - 2000;
   probe.slotState.set(0, { decoded: 10, prompted: 5 });
   probe._fetch = async (url) => {
     const u = String(url);
@@ -268,13 +276,14 @@ test("llama.cpp probe: slot deltas → tok/s; props for model", async () => {
   assert.equal(snap.uncachedPrefillTps, null);
 });
 
-test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", async () => {
+test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", async (t) => {
+  const now = freezeProbeClock(t);
   const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080);
   probe.serverIsOpenAI = false;
   probe.backendType = "llama.cpp";
   probe.authOpen = true;
-  probe._lastDetectAt = Date.now();
-  probe.lastProbeTime = Date.now() - 2000;
+  probe._lastDetectAt = now;
+  probe.lastProbeTime = now - 2000;
   probe.slotState.set(0, { decoded: 10, prompted: 5 });
   probe.lastPrefillKinds = { cached: 10, computed: 5 };
   probe._fetch = async (url) => {
@@ -299,13 +308,14 @@ test("llama.cpp probe: n_prompt_tokens_cache → cached vs uncached prefill", as
   assert.equal(snap.cachedPrefillTps, 15); // (40-10)/2
 });
 
-test("llama.cpp: n_prompt_tokens_processed 0 is not treated as missing", async () => {
+test("llama.cpp: n_prompt_tokens_processed 0 is not treated as missing", async (t) => {
+  const now = freezeProbeClock(t);
   const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8080);
   probe.serverIsOpenAI = false;
   probe.backendType = "llama.cpp";
   probe.authOpen = true;
-  probe._lastDetectAt = Date.now();
-  probe.lastProbeTime = Date.now() - 2000;
+  probe._lastDetectAt = now;
+  probe.lastProbeTime = now - 2000;
   probe.slotState.set(0, { decoded: 10, prompted: 0 });
   probe.lastPrefillKinds = { cached: 10, computed: 0 };
   probe._fetch = async (url) => {

server/collectors/__tests__/SystemCollector.hostNet.test.js

@@ -0,0 +1,29 @@
+import assert from "node:assert/strict";
+import fs from "node:fs";
+import test from "node:test";
+
+import { SystemCollector } from "../SystemCollector.js";
+
+function localSpark() {
+  return {
+    id: "local-test",
+    name: "Local Test",
+    isLocal: true,
+    lanIp: "127.0.0.1",
+  };
+}
+
+test("host network file fallback reads container proc without recursing", async (t) => {
+  const collector = new SystemCollector(localSpark());
+  collector._hasHostProc = () => false;
+  const reads = [];
+  t.mock.method(fs, "readFileSync", (filePath, encoding) => {
+    reads.push({ filePath, encoding });
+    return "Inter-| Receive | Transmit\n";
+  });
+
+  const contents = await collector._readHostNetFile("dev");
+
+  assert.match(contents, /Inter-\|/);
+  assert.deepEqual(reads, [{ filePath: "/proc/net/dev", encoding: "utf-8" }]);
+});

server/collectors/__tests__/SystemCollector.nvErr.test.js

@@ -0,0 +1,16 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import { parseNvErrNoMemoryCount } from "../SystemCollector.js";
+
+test("parseNvErrNoMemoryCount reads grep -c output", () => {
+  assert.equal(parseNvErrNoMemoryCount("12"), 12);
+  assert.equal(parseNvErrNoMemoryCount("0"), 0);
+  assert.equal(parseNvErrNoMemoryCount(" 43\n"), 43);
+});
+
+test("parseNvErrNoMemoryCount defaults invalid input to 0", () => {
+  assert.equal(parseNvErrNoMemoryCount(""), 0);
+  assert.equal(parseNvErrNoMemoryCount("not-a-number"), 0);
+  assert.equal(parseNvErrNoMemoryCount(undefined), 0);
+  assert.equal(parseNvErrNoMemoryCount("-3"), 0);
+});

server/collectors/__tests__/showcasePrompts.test.js

@@ -91,12 +91,18 @@ test("catalog prompts exist for text, structural, and mixed pickers", () => {
   assert.ok(textCount >= 2);
 });
 
-test("DecodeBench uses Showcase structural prompts at temperature 0, thinking off", () => {
-  assert.match(benchSrc, /pickShowcasePrompts\("structural"/);
-  assert.match(benchSrc, /withFillToMaxInstruction/);
+test("DecodeBench uses the lab structured protocol at temperature 0, thinking off", () => {
+  // 1.8.3 moved DecodeBench off the Showcase structural catalog + fill-to-max
+  // and onto the lab structured protocol (count 1 -> 200); 1.8.4 added the
+  // output-type picker. These assertions still referenced the removed API.
+  assert.match(benchSrc, /pickDecodeBenchPrompts\(/);
+  assert.match(benchSrc, /decodeBenchPromptForType\(/);
+  assert.match(benchSrc, /normalizeDecodeBenchType\(/);
   assert.match(benchSrc, /temperature:\s*0/);
+  assert.match(benchSrc, /top_p:\s*1/);
   assert.match(benchSrc, /applyThinkingFlags\(body,\s*modelId,\s*false\)/);
   assert.match(benchSrc, /min_tokens:\s*maxTokens/);
+  assert.doesNotMatch(benchSrc, /withFillToMaxInstruction/);
   assert.doesNotMatch(benchSrc, /uniquePrefillPrefix/);
   assert.doesNotMatch(benchSrc, /BENCH_PROMPTS/);
 });

server/config.js

@@ -34,6 +34,8 @@ const POLL_INTERVAL_LLM = parseInt(process.env.POLL_INTERVAL_LLM || "2000", 10);
 const POLL_INTERVAL_COMFY = parseInt(process.env.POLL_INTERVAL_COMFY || "2000", 10);
 // Tailnet membership changes slowly; each poll is an SSH round-trip.
 const POLL_INTERVAL_TAILSCALE = parseInt(process.env.POLL_INTERVAL_TAILSCALE || "30000", 10);
+// Kernel journal scan for NV_ERR_NO_MEMORY — not on the 2s GPU loop.
+const POLL_INTERVAL_NVERR = parseInt(process.env.POLL_INTERVAL_NVERR || "60000", 10);
 // dmon -c 1 -d 1 blocks ~1s; default 2s avoids stacking with in-flight guards
 const POLL_INTERVAL_BANDWIDTH = parseInt(process.env.POLL_INTERVAL_BANDWIDTH || "2000", 10);
 // Dedicated liveness (sshTest / local ping) cadence — not a metric domain.
@@ -106,6 +108,7 @@ export {
   POLL_INTERVAL_LLM,
   POLL_INTERVAL_COMFY,
   POLL_INTERVAL_TAILSCALE,
+  POLL_INTERVAL_NVERR,
   POLL_INTERVAL_BANDWIDTH,
   POLL_INTERVAL_LIVENESS,
   POLL_INTERVAL_HERMES,

src/api/types.ts

@@ -211,6 +211,8 @@ export interface GpuMetrics {
   processes?: Array<{ pid: number; name: string; vramMB: number }>;
   /** NVIDIA clock throttle / thermal slowdown state from nvidia-smi. */
   throttle?: GpuThrottle | null;
+  /** Kernel NVRM NV_ERR_NO_MEMORY count since boot (cached ~60s). */
+  nvErrNoMemory?: number;
 }
 
 // ─── CPU metrics ─────────────────────────────────────────

src/components/SparkPage/GpuPanel.tsx

@@ -202,6 +202,18 @@ export function GpuPanel({ gpu, cpu, sparkId, temperatureUnit, className }: GpuP
         </div>
       )}
 
+      {(gpu?.nvErrNoMemory ?? 0) > 0 && (
+        <div
+          className="flex items-center justify-between text-sm"
+          title="NVRM kernel NV_ERR_NO_MEMORY lines since boot (journal). GPU memory allocation failures under pressure."
+        >
+          <span className="text-muted">NV_ERR_NO_MEMORY</span>
+          <span className="font-tabular text-sm font-semibold text-danger">
+            {gpu?.nvErrNoMemory}
+          </span>
+        </div>
+      )}
+
       {/* Top GPU processes by VRAM usage */}
       {gpu && gpu.processes && gpu.processes.length > 0 && (
         <div className="space-y-1.5 border-t border-border pt-3">