Skip to content

feat: LLM panel tok/s & TTFT trend chart with busy-sample averages #75

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
unsaltedbutter-ai wants to merge 3 commits into from
Closed

feat: LLM panel tok/s & TTFT trend chart with busy-sample averages #75

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3cd2f87
feat: LLM panel tok/s & TTFT trend chart with busy-sample averages
unsaltedbutter-ai Aug 31, 2026
89114cf
fix: drop TTFT overlay from tok/s trend chart per review
unsaltedbutter-ai Sep 6, 2026
62907ed
feat: fixed 30-min scrolling chart window; configurable history reten…
unsaltedbutter-ai Sep 6, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,11 @@ COPY package.json package-lock.json* ./
RUN npm ci --no-audit --no-fund \
|| (echo "npm ci failed once — retrying…" && npm cache clean --force && npm ci --no-audit --no-fund)

# Copy source and build
# Copy source and build. VITE_HISTORY_HOURS sets the frontend metrics-history
# retention window (see src/hooks/metricsStore.ts); override via
# `docker compose build --build-arg VITE_HISTORY_HOURS=4` or the env in compose.
ARG VITE_HISTORY_HOURS=8
ENV VITE_HISTORY_HOURS=${VITE_HISTORY_HOURS}
COPY . .
RUN npm run build

Expand Down
3 changes: 3 additions & 0 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@ services:
dockerfile: Dockerfile
platforms:
- linux/arm64
args:
# Frontend metrics-history retention (hours). Rebuild after changing.
VITE_HISTORY_HOURS: ${VITE_HISTORY_HOURS:-8}
container_name: sparkDash
# always: come back after host reboot even if the container was stopped
# before shutdown (unlike unless-stopped, which stays down after a manual stop).
Expand Down
26 changes: 26 additions & 0 deletions server/collectors/LlmProbe.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,8 @@ export class LlmProbe {
this.lastPrefillKinds = null;
/** Previous vLLM TTFT histogram `_sum` (seconds). null until first sample. */
this.lastTtftSum = null;
/** Previous vLLM TTFT histogram `_count` (requests). null until first sample. */
this.lastTtftCount = null;
/** Previous `vllm:iteration_tokens_total_sum` (engine-step tokens). */
this.lastIterSum = null;
this.lastProbeTime = 0;
Expand All @@ -105,6 +107,8 @@ export class LlmProbe {
this.requestsRunning = null;
this.requestsWaiting = null;
this.ttftP95Seconds = null;
/** Live recent-window mean TTFT (seconds) from histogram sum/count deltas. null when unavailable. */
this.ttftSeconds = null;
this.preemptionsTotal = null; // cumulative counter
/** Prefix cache hit rate 0–1 (hits/queries). */
this.prefixCacheHitRate = null;
Expand Down Expand Up @@ -242,6 +246,7 @@ export class LlmProbe {
this.requestsRunning = null;
this.requestsWaiting = null;
this.ttftP95Seconds = null;
this.ttftSeconds = null;
this.preemptionsTotal = null;
this.prefixCacheHitRate = null;
this.e2eP95Seconds = null;
Expand All @@ -251,6 +256,7 @@ export class LlmProbe {
this.lastTokenCounts = { input: 0, output: 0 };
this.lastPrefillKinds = null;
this.lastTtftSum = null;
this.lastTtftCount = null;
this.lastIterSum = null;
this._sglangStickyTps = null;
}
Expand Down Expand Up @@ -608,6 +614,7 @@ export class LlmProbe {
this.kvCacheUsage = null;
this.requestsWaiting = null;
this.ttftP95Seconds = null;
this.ttftSeconds = null;
this.preemptionsTotal = null;
this.e2eP95Seconds = null;
this.itlP95Seconds = null;
Expand All @@ -632,6 +639,7 @@ export class LlmProbe {
this.kvCacheUsage = null;
this.requestsWaiting = null;
this.ttftP95Seconds = null;
this.ttftSeconds = null;
this.preemptionsTotal = null;
this.prefixCacheHitRate = null;
this.e2eP95Seconds = null;
Expand Down Expand Up @@ -706,6 +714,22 @@ export class LlmProbe {
Math.round((livePrefill > 0 ? livePrefill : finishedPrefill) * 100) / 100
);
}
// Live mean TTFT over the last poll window from histogram sum/count deltas.
// Computed BEFORE lastTtftSum is advanced so the delta is real, not 0.
const ttftCount = this._getVllmMetric(txt, "time_to_first_token_seconds_count");
if (ttftCount != null) {
const deltaSum =
ttftSum != null && this.lastTtftSum != null ? ttftSum - this.lastTtftSum : null;
const deltaCount =
this.lastTtftCount != null ? ttftCount - this.lastTtftCount : null;
this.ttftSeconds =
deltaSum != null && deltaCount != null && deltaCount > 0 && deltaSum >= 0
? Math.round((deltaSum / deltaCount) * 1000) / 1000
: null;
this.lastTtftCount = ttftCount;
} else {
this.ttftSeconds = null;
}
if (ttftSum != null) this.lastTtftSum = ttftSum;
}
if (iterSum != null) this.lastIterSum = iterSum;
Expand Down Expand Up @@ -1374,6 +1398,7 @@ export class LlmProbe {
requestsRunning: this.requestsRunning,
requestsWaiting: this.requestsWaiting,
ttftP95Seconds: this.ttftP95Seconds,
ttftSeconds: this.ttftSeconds,
preemptionsTotal: this.preemptionsTotal,
prefixCacheHitRate: this.prefixCacheHitRate,
e2eP95Seconds: this.e2eP95Seconds,
Expand Down Expand Up @@ -1403,6 +1428,7 @@ export class LlmProbe {
requestsRunning: null,
requestsWaiting: null,
ttftP95Seconds: null,
ttftSeconds: null,
preemptionsTotal: null,
prefixCacheHitRate: null,
e2eP95Seconds: null,
Expand Down
2 changes: 2 additions & 0 deletions src/api/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,8 @@ export interface LlmMetrics {
requestsWaiting?: number | null;
/** vLLM time-to-first-token p95 in seconds. null when unavailable. */
ttftP95Seconds?: number | null;
/** Live recent-window mean TTFT (seconds) from vLLM histogram sum/count deltas. null when unavailable. */
ttftSeconds?: number | null;
/** vLLM cumulative preemption count. null when unavailable. */
preemptionsTotal?: number | null;
/** vLLM prefix-cache hit rate (hits/queries, 0–1). null when unavailable. */
Expand Down
72 changes: 58 additions & 14 deletions src/components/SparkPage/LlmPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,19 @@
import { useState, useEffect, useRef, useCallback } from "react";
import { useState, useEffect, useRef, useCallback, useMemo } from "react";
import type { LlmMetrics, LlmBenchTarget } from "../../api/types";
import { setLlmApiKey, updateLlmPort, updateLlmPorts } from "../../api/client";
import { Sparkline } from "../ui/Sparkline";
import { Panel } from "../ui/Panel";
import { BotIcon, GearIcon, InfoIcon } from "../ui/icons";
import { useMetricsHistoryTail } from "../../hooks/metricsStore";
import {
useMetricsHistory,
useMetricsHistoryTail,
avgPositive,
} from "../../hooks/metricsStore";
import { BenchmarkDialog } from "./BenchmarkDialog";
import { PrefillBenchDialog } from "./PrefillBenchDialog";
import { LlmDailyChart } from "./LlmDailyChart";
import { parseLlmTargetInput } from "../../shared/llmTarget.js";
import { LlmTrendChart } from "./LlmTrendChart";

interface LlmPanelProps {
llm: LlmMetrics | null;
Expand Down Expand Up @@ -384,6 +389,16 @@ export function LlmPanel({
const prefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefill`);
const cachedPrefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefillCached`);
const uncachedPrefillHistory = useMetricsHistoryTail(sparkId, `llm:${llmPort}.prefillUncached`);

// Full series (~1 h) for running averages over busy (>0) samples only.
const genFull = useMetricsHistory(sparkId, `llm:${llmPort}.tps`);
const prefillFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefill`);
const cachedFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefillCached`);
const uncachedFull = useMetricsHistory(sparkId, `llm:${llmPort}.prefillUncached`);
const genAvg = useMemo(() => avgPositive(genFull), [genFull]);
const prefillAvg = useMemo(() => avgPositive(prefillFull), [prefillFull]);
const cachedPrefillAvg = useMemo(() => avgPositive(cachedFull), [cachedFull]);
const uncachedPrefillAvg = useMemo(() => avgPositive(uncachedFull), [uncachedFull]);
const [showSettings, setShowSettings] = useState(false);
const [portDraft, setPortDraft] = useState(String(llmPort));
const [apiKeyDraft, setApiKeyDraft] = useState("");
Expand Down Expand Up @@ -684,9 +699,16 @@ export function LlmPanel({
<span className="text-xs text-muted">Generation tok/s</span>
<div className="flex items-center gap-2">
<Sparkline data={genHistory} color="var(--color-accent)" height={24} />
<span className="font-tabular text-sm font-semibold text-accent">
{generationTps.toFixed(1)}
</span>
<div className="text-right">
<div className="font-tabular text-sm font-semibold text-accent">
{generationTps.toFixed(1)}
</div>
{genAvg != null && (
<div className="font-tabular text-[9px] text-muted">
avg {genAvg >= 100 ? genAvg.toFixed(0) : genAvg.toFixed(1)}
</div>
)}
</div>
</div>
</div>
<div
Expand All @@ -696,9 +718,16 @@ export function LlmPanel({
<span className="text-xs text-muted">Prefill tok/s</span>
<div className="flex items-center gap-2">
<Sparkline data={prefillHistory} color="var(--color-text)" height={24} />
<span className="font-tabular text-sm font-semibold text-text">
{prefillTps.toFixed(1)}
</span>
<div className="text-right">
<div className="font-tabular text-sm font-semibold text-text">
{prefillTps.toFixed(1)}
</div>
{prefillAvg != null && (
<div className="font-tabular text-[9px] text-muted">
avg {prefillAvg >= 100 ? prefillAvg.toFixed(0) : prefillAvg.toFixed(1)}
</div>
)}
</div>
</div>
</div>
{showPrefillSplit && (
Expand All @@ -710,9 +739,16 @@ export function LlmPanel({
<span className="text-xs text-muted">Cached prefill tok/s</span>
<div className="flex items-center gap-2">
<Sparkline data={cachedPrefillHistory} color="var(--color-muted)" height={24} />
<span className="font-tabular text-sm font-semibold text-muted">
{cachedPrefillTps.toFixed(1)}
</span>
<div className="text-right">
<div className="font-tabular text-sm font-semibold text-muted">
{cachedPrefillTps.toFixed(1)}
</div>
{cachedPrefillAvg != null && (
<div className="font-tabular text-[9px] text-muted">
avg {cachedPrefillAvg >= 100 ? cachedPrefillAvg.toFixed(0) : cachedPrefillAvg.toFixed(1)}
</div>
)}
</div>
</div>
</div>
<div
Expand All @@ -722,14 +758,22 @@ export function LlmPanel({
<span className="text-xs text-muted">Uncached prefill tok/s</span>
<div className="flex items-center gap-2">
<Sparkline data={uncachedPrefillHistory} color="var(--color-text)" height={24} />
<span className="font-tabular text-sm font-semibold text-text">
{uncachedPrefillTps.toFixed(1)}
</span>
<div className="text-right">
<div className="font-tabular text-sm font-semibold text-text">
{uncachedPrefillTps.toFixed(1)}
</div>
{uncachedPrefillAvg != null && (
<div className="font-tabular text-[9px] text-muted">
avg {uncachedPrefillAvg >= 100 ? uncachedPrefillAvg.toFixed(0) : uncachedPrefillAvg.toFixed(1)}
</div>
)}
</div>
</div>
</div>
</>
)}

<LlmTrendChart sparkId={sparkId} llmPort={llmPort} />
<LlmDailyChart sparkId={sparkId} llmPort={llmPort} />

<div className="grid grid-cols-4 gap-2 border-t border-border pt-3">
Expand Down
Loading