diff --git a/Dockerfile b/Dockerfile index 8a19596a..b7e64a7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,11 @@ COPY package.json package-lock.json* ./ RUN npm ci --no-audit --no-fund \ || (echo "npm ci failed once — retrying…" && npm cache clean --force && npm ci --no-audit --no-fund) -# Copy source and build +# Copy source and build. VITE_HISTORY_HOURS sets the frontend metrics-history +# retention window (see src/hooks/metricsStore.ts); override via +# `docker compose build --build-arg VITE_HISTORY_HOURS=4` or the env in compose. +ARG VITE_HISTORY_HOURS=8 +ENV VITE_HISTORY_HOURS=${VITE_HISTORY_HOURS} COPY . . RUN npm run build diff --git a/docker-compose.yml b/docker-compose.yml index 3811d7eb..44e7e368 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,9 @@ services: dockerfile: Dockerfile platforms: - linux/arm64 + args: + # Frontend metrics-history retention (hours). Rebuild after changing. + VITE_HISTORY_HOURS: ${VITE_HISTORY_HOURS:-8} container_name: sparkDash # always: come back after host reboot even if the container was stopped # before shutdown (unlike unless-stopped, which stays down after a manual stop). diff --git a/server/collectors/LlmProbe.js b/server/collectors/LlmProbe.js index 08522b2a..2d29858f 100644 --- a/server/collectors/LlmProbe.js +++ b/server/collectors/LlmProbe.js @@ -92,6 +92,8 @@ export class LlmProbe { this.lastPrefillKinds = null; /** Previous vLLM TTFT histogram `_sum` (seconds). null until first sample. */ this.lastTtftSum = null; + /** Previous vLLM TTFT histogram `_count` (requests). null until first sample. */ + this.lastTtftCount = null; /** Previous `vllm:iteration_tokens_total_sum` (engine-step tokens). */ this.lastIterSum = null; this.lastProbeTime = 0; @@ -105,6 +107,8 @@ export class LlmProbe { this.requestsRunning = null; this.requestsWaiting = null; this.ttftP95Seconds = null; + /** Live recent-window mean TTFT (seconds) from histogram sum/count deltas. null when unavailable. */ + this.ttftSeconds = null; this.preemptionsTotal = null; // cumulative counter /** Prefix cache hit rate 0–1 (hits/queries). */ this.prefixCacheHitRate = null; @@ -242,6 +246,7 @@ export class LlmProbe { this.requestsRunning = null; this.requestsWaiting = null; this.ttftP95Seconds = null; + this.ttftSeconds = null; this.preemptionsTotal = null; this.prefixCacheHitRate = null; this.e2eP95Seconds = null; @@ -251,6 +256,7 @@ export class LlmProbe { this.lastTokenCounts = { input: 0, output: 0 }; this.lastPrefillKinds = null; this.lastTtftSum = null; + this.lastTtftCount = null; this.lastIterSum = null; this._sglangStickyTps = null; } @@ -608,6 +614,7 @@ export class LlmProbe { this.kvCacheUsage = null; this.requestsWaiting = null; this.ttftP95Seconds = null; + this.ttftSeconds = null; this.preemptionsTotal = null; this.e2eP95Seconds = null; this.itlP95Seconds = null; @@ -632,6 +639,7 @@ export class LlmProbe { this.kvCacheUsage = null; this.requestsWaiting = null; this.ttftP95Seconds = null; + this.ttftSeconds = null; this.preemptionsTotal = null; this.prefixCacheHitRate = null; this.e2eP95Seconds = null; @@ -706,6 +714,22 @@ export class LlmProbe { Math.round((livePrefill > 0 ? livePrefill : finishedPrefill) * 100) / 100 ); } + // Live mean TTFT over the last poll window from histogram sum/count deltas. + // Computed BEFORE lastTtftSum is advanced so the delta is real, not 0. + const ttftCount = this._getVllmMetric(txt, "time_to_first_token_seconds_count"); + if (ttftCount != null) { + const deltaSum = + ttftSum != null && this.lastTtftSum != null ? ttftSum - this.lastTtftSum : null; + const deltaCount = + this.lastTtftCount != null ? ttftCount - this.lastTtftCount : null; + this.ttftSeconds = + deltaSum != null && deltaCount != null && deltaCount > 0 && deltaSum >= 0 + ? Math.round((deltaSum / deltaCount) * 1000) / 1000 + : null; + this.lastTtftCount = ttftCount; + } else { + this.ttftSeconds = null; + } if (ttftSum != null) this.lastTtftSum = ttftSum; } if (iterSum != null) this.lastIterSum = iterSum; @@ -1374,6 +1398,7 @@ export class LlmProbe { requestsRunning: this.requestsRunning, requestsWaiting: this.requestsWaiting, ttftP95Seconds: this.ttftP95Seconds, + ttftSeconds: this.ttftSeconds, preemptionsTotal: this.preemptionsTotal, prefixCacheHitRate: this.prefixCacheHitRate, e2eP95Seconds: this.e2eP95Seconds, @@ -1403,6 +1428,7 @@ export class LlmProbe { requestsRunning: null, requestsWaiting: null, ttftP95Seconds: null, + ttftSeconds: null, preemptionsTotal: null, prefixCacheHitRate: null, e2eP95Seconds: null, diff --git a/src/api/types.ts b/src/api/types.ts index 9f1be881..39e70cfe 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -307,6 +307,8 @@ export interface LlmMetrics { requestsWaiting?: number | null; /** vLLM time-to-first-token p95 in seconds. null when unavailable. */ ttftP95Seconds?: number | null; + /** Live recent-window mean TTFT (seconds) from vLLM histogram sum/count deltas. null when unavailable. */ + ttftSeconds?: number | null; /** vLLM cumulative preemption count. null when unavailable. */ preemptionsTotal?: number | null; /** vLLM prefix-cache hit rate (hits/queries, 0–1). null when unavailable. */ diff --git a/src/components/SparkPage/LlmPanel.tsx b/src/components/SparkPage/LlmPanel.tsx index 29c9013c..f6230b29 100644 --- a/src/components/SparkPage/LlmPanel.tsx +++ b/src/components/SparkPage/LlmPanel.tsx @@ -1,14 +1,19 @@ -import { useState, useEffect, useRef, useCallback } from "react"; +import { useState, useEffect, useRef, useCallback, useMemo } from "react"; import type { LlmMetrics, LlmBenchTarget } from "../../api/types"; import { setLlmApiKey, updateLlmPort, updateLlmPorts } from "../../api/client"; import { Sparkline } from "../ui/Sparkline"; import { Panel } from "../ui/Panel"; import { BotIcon, GearIcon, InfoIcon } from "../ui/icons"; -import { useMetricsHistoryTail } from "../../hooks/metricsStore"; +import { + useMetricsHistory, + useMetricsHistoryTail, + avgPositive, +} from "../../hooks/metricsStore"; import { BenchmarkDialog } from "./BenchmarkDialog"; import { PrefillBenchDialog } from "./PrefillBenchDialog"; import { LlmDailyChart } from "./LlmDailyChart"; import { parseLlmTargetInput } from "../../shared/llmTarget.js"; +import { LlmTrendChart } from "./LlmTrendChart"; interface LlmPanelProps { llm: LlmMetrics | null; @@ -384,6 +389,16 @@ export function LlmPanel({ const prefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefill`); const cachedPrefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefillCached`); const uncachedPrefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefillUncached`); + + // Full series (~1 h) for running averages over busy (>0) samples only. + const genFull = useMetricsHistory(sparkId, `llm:${llmPort}.tps`); + const prefillFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefill`); + const cachedFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefillCached`); + const uncachedFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefillUncached`); + const genAvg = useMemo(() => avgPositive(genFull), [genFull]); + const prefillAvg = useMemo(() => avgPositive(prefillFull), [prefillFull]); + const cachedPrefillAvg = useMemo(() => avgPositive(cachedFull), [cachedFull]); + const uncachedPrefillAvg = useMemo(() => avgPositive(uncachedFull), [uncachedFull]); const [showSettings, setShowSettings] = useState(false); const [portDraft, setPortDraft] = useState(String(llmPort)); const [apiKeyDraft, setApiKeyDraft] = useState(""); @@ -684,9 +699,16 @@ export function LlmPanel({ Generation tok/s
- - {generationTps.toFixed(1)} - +
+
+ {generationTps.toFixed(1)} +
+ {genAvg != null && ( +
+ avg {genAvg >= 100 ? genAvg.toFixed(0) : genAvg.toFixed(1)} +
+ )} +
Prefill tok/s
- - {prefillTps.toFixed(1)} - +
+
+ {prefillTps.toFixed(1)} +
+ {prefillAvg != null && ( +
+ avg {prefillAvg >= 100 ? prefillAvg.toFixed(0) : prefillAvg.toFixed(1)} +
+ )} +
{showPrefillSplit && ( @@ -710,9 +739,16 @@ export function LlmPanel({ Cached prefill tok/s
- - {cachedPrefillTps.toFixed(1)} - +
+
+ {cachedPrefillTps.toFixed(1)} +
+ {cachedPrefillAvg != null && ( +
+ avg {cachedPrefillAvg >= 100 ? cachedPrefillAvg.toFixed(0) : cachedPrefillAvg.toFixed(1)} +
+ )} +
Uncached prefill tok/s
- - {uncachedPrefillTps.toFixed(1)} - +
+
+ {uncachedPrefillTps.toFixed(1)} +
+ {uncachedPrefillAvg != null && ( +
+ avg {uncachedPrefillAvg >= 100 ? uncachedPrefillAvg.toFixed(0) : uncachedPrefillAvg.toFixed(1)} +
+ )} +
)} +
diff --git a/src/components/SparkPage/LlmTrendChart.tsx b/src/components/SparkPage/LlmTrendChart.tsx new file mode 100644 index 00000000..dbc19598 --- /dev/null +++ b/src/components/SparkPage/LlmTrendChart.tsx @@ -0,0 +1,171 @@ +import { useMemo } from "react"; +import { HISTORY_MAX, useMetricsHistory, avgPositive } from "../../hooks/metricsStore"; + +const VIEW_W = 300; +const VIEW_H = 64; +const PAD = 2; +/** + * Fixed display window: 30 minutes of 2 s samples. The x-axis is anchored to + * this constant — never the current sample count — so the line grows into the + * chart left-to-right and then scrolls, instead of re-stretching (rewriting + * history) on every tick. Averages below still span full HISTORY_MAX retention. + */ +const DISPLAY_WINDOW = 900; + +function fmt(n: number | null): string { + if (n == null || !Number.isFinite(n)) return "—"; + return n >= 100 ? n.toFixed(0) : n.toFixed(1); +} + +/** + * Polyline points for one series, normalised to the shared max. x maps onto a + * FIXED window: the newest sample sits at the right edge once the window is + * full; while filling, points occupy only the left fraction and the line grows. + */ +function buildPoints(raw: readonly number[], max: number): string { + // Only the newest DISPLAY_WINDOW samples are drawn; older ones still feed + // the averages below. Once full, the window scrolls (newest at right edge). + const data = raw.length > DISPLAY_WINDOW ? raw.slice(-DISPLAY_WINDOW) : raw; + if (data.length < 2) return ""; + const span = max || 1; + const pts = data.map((v, i) => { + const x = (i / (DISPLAY_WINDOW - 1)) * VIEW_W; + const y = VIEW_H - PAD - (Math.min(v, max) / span) * (VIEW_H - PAD * 2); + return `${x.toFixed(1)},${y.toFixed(1)}`; + }); + return pts.join(" "); +} + +function areaPath(points: string): string { + const seg = points.split(" "); + const first = seg[0]?.split(",")[0] ?? "0"; + const last = seg[seg.length - 1]?.split(",")[0] ?? first; + return `M${first},${VIEW_H} L${points} L${last},${VIEW_H} Z`; +} + +/** Human label: chart shows the last window; averages span full retention. */ +function fmtSpan(seconds: number): string { + if (seconds < 3600) return `${Math.round(seconds / 60)}m`; + const h = seconds / 3600; + return `${h % 1 === 0 ? h : h.toFixed(1)}h`; +} + +function historyLabel(): string { + return `chart ~${fmtSpan(DISPLAY_WINDOW * 2)} · avgs ~${fmtSpan(HISTORY_MAX * 2)} · 2s samples`; +} + +/** Newest DISPLAY_WINDOW samples — the slice the chart draws. */ +function windowed(data: readonly number[]): readonly number[] { + return data.length > DISPLAY_WINDOW ? data.slice(-DISPLAY_WINDOW) : data; +} + +/** + * tok/s trend chart for one LLM port. The x-axis is a FIXED 30-minute window: + * the line grows left-to-right while filling, then scrolls — history already + * drawn never re-stretches, so the chart can't "rewrite" its own past. The + * averages below span the full retention (VITE_HISTORY_HOURS, default 8 h). + * + * TTFT is deliberately NOT drawn here: vLLM reports it only while serving, so + * the series is sparse and not tick-aligned — overlaying it on this chart would + * misplace it in time. It is also near-redundant with the prefill spikes it + * tracks. The busy-sample TTFT average badge is the useful signal and reads the + * sparse series directly (no x-axis involved). + */ +export function LlmTrendChart({ + sparkId, + llmPort, +}: { + sparkId: string; + llmPort: number; +}) { + const gen = useMetricsHistory(sparkId, `llm:${llmPort}.tps`); + const prefill = useMetricsHistory(sparkId, `llm:${llmPort}.prefill`); + const ttft = useMetricsHistory(sparkId, `llm:${llmPort}.ttft`); + + const genAvg = useMemo(() => avgPositive(gen), [gen]); + const prefillAvg = useMemo(() => avgPositive(prefill), [prefill]); + const ttftAvg = useMemo(() => avgPositive(ttft), [ttft]); + + // Chart draws only the newest hour; averages above use the full series. + const genWin = useMemo(() => windowed(gen), [gen]); + const prefillWin = useMemo(() => windowed(prefill), [prefill]); + + // Normalise each series to its OWN max: prefill (thousands) and generation + // (tens) differ by ~100x, so a shared scale would flatten gen into the floor. + // Max is over the drawn window so old spikes can't squash recent detail. + const genMax = useMemo(() => Math.max(1, ...genWin), [genWin]); + const prefillMax = useMemo(() => Math.max(1, ...prefillWin), [prefillWin]); + const genPts = useMemo(() => buildPoints(genWin, genMax), [genWin, genMax]); + const prefillPts = useMemo(() => buildPoints(prefillWin, prefillMax), [prefillWin, prefillMax]); + + const hasData = genWin.length > 1 || prefillWin.length > 1; + + return ( +
+
+ + tok/s history + + {historyLabel()} +
+ {!hasData ? ( +

No samples yet.

+ ) : ( + + {prefillPts && ( + <> + + + + )} + {genPts && ( + <> + + + + )} + + )} +
+ + Gen avg{" "} + {fmt(genAvg)} + + + Prefill avg{" "} + {fmt(prefillAvg)} + + + TTFT avg{" "} + + {ttftAvg != null ? `${ttftAvg.toFixed(3)}s` : "—"} + + + avg over busy samples only +
+
+ ); +} diff --git a/src/hooks/metricsStore.ts b/src/hooks/metricsStore.ts index 5bb23137..345417c4 100644 --- a/src/hooks/metricsStore.ts +++ b/src/hooks/metricsStore.ts @@ -21,7 +21,12 @@ import type { SparkSnapshot } from "../api/types"; * All listeners are woken on notify; unchanged keys keep the same ref → no render. */ -const HISTORY_MAX = 1800; // 1 h at 2 s poll — the WS interval, not wall-clock guarantees +// History depth, configurable at build time. VITE_HISTORY_HOURS = wall-clock +// hours to retain (default 8 h at the 2 s WS poll ≈ 112 KB per series — the +// browser tab is the only thing that pays for it). +const SAMPLES_PER_HOUR = 1800; // 2 s poll +const HISTORY_HOURS = Number(import.meta.env.VITE_HISTORY_HOURS ?? 8) || 8; +export const HISTORY_MAX = Math.round(SAMPLES_PER_HOUR * HISTORY_HOURS); /** Samples shown in inline sparklines (≈1 min at 2 s poll). Full series stays in HISTORY_MAX. */ export const SPARKLINE_TAIL = 30; @@ -33,6 +38,23 @@ const listeners = new Set<() => void>(); const EMPTY: readonly number[] = Object.freeze([] as number[]); +/** + * Mean of a metric series over samples where the reading is > 0, or null when + * there are no busy samples. Skipping zeros keeps idle/off phases from dragging + * the average toward zero (a prefill that runs at 500 tok/s is "500", not 0.5). + */ +export function avgPositive(values: readonly number[]): number | null { + let sum = 0; + let count = 0; + for (const v of values) { + if (v > 0) { + sum += v; + count += 1; + } + } + return count > 0 ? sum / count : null; +} + function notify() { for (const l of listeners) l(); } @@ -112,6 +134,13 @@ export function ingestSnapshots(sparks: SparkSnapshot[]): void { const portKey = port != null ? `:${port}` : `:${i}`; pushHistory(`${s.id}:llm${portKey}.tps`, llm.generationTps); pushHistory(`${s.id}:llm${portKey}.prefill`, llm.prefillTps); + // TTFT is sparse: vLLM reports live TTFT only while serving. It is NOT + // index-aligned with the tick-dense series above — that is fine because + // the ttft series feeds only the busy-sample average badge, never the + // overlaid chart (see LlmTrendChart). + if (llm.ttftSeconds != null) { + pushHistory(`${s.id}:llm${portKey}.ttft`, llm.ttftSeconds); + } if (llm.cachedPrefillTps != null) { pushHistory(`${s.id}:llm${portKey}.prefillCached`, llm.cachedPrefillTps); }