Skip to content

refactor(observability): simplify run outcome tracking, route span attributes and proxy log deprecations #4605

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kwakayama merged 1 commit into from
Sep 27, 2026
Merged

refactor(observability): simplify run outcome tracking, route span attributes and proxy log deprecations #4605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 43 additions & 36 deletions src/internal-agents/run-stream.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,41 @@ function resolveChildRunToolNames(mergedTools: Agent["config"]["tools"]): Set<st
return names;
}

/**
* Counts failed tool calls, and those that ran a child agent, and keeps the
* first stable RunError code from a run's AG-UI events.
*/
function createRunOutcomeTracker(childRunToolNames: ReadonlySet<string>) {
const childRunToolCallIds = new Set<string>();
let terminalErrorCode: string | undefined;
let toolErrorCount = 0;
let childRunErrorCount = 0;
return {
observe(event: string, payload: Record<string, unknown>): void {
const { toolCallId } = payload;
if (
event === "ToolCallStart" && typeof toolCallId === "string" &&
typeof payload.toolCallName === "string" && childRunToolNames.has(payload.toolCallName)
) {
childRunToolCallIds.add(toolCallId);
} else if (event === "ToolCallResult" && payload.isError === true) {
toolErrorCount++;
if (typeof toolCallId === "string" && childRunToolCallIds.has(toolCallId)) {
childRunErrorCount++;
}
} else if (event === "RunError") {
terminalErrorCode ??= toStableRunErrorCode(payload.code);
}
},
terminalErrorCode: () => terminalErrorCode,
spanAttributes: () => ({
"agent.run.tool_error_count": toolErrorCount,
"agent.run.child_run_error_count": childRunErrorCount,
}),
logFields: () => ({ toolErrorCount, childRunErrorCount }),
};
}

function isExplicitlyDeniedToolName(
agent: Agent,
deniedToolNames: ReadonlySet<string>,
Expand Down Expand Up @@ -1401,31 +1436,7 @@ export async function createRuntimeAgentStreamResponse(
"Internal agent runtime stream stopped before EOF",
);
let readerCancellation: Promise<void> | undefined;
let terminalRunErrorCode: string | undefined;
let toolErrorCount = 0;
let childRunErrorCount = 0;
const childRunToolCallIds = new Set<string>();
const observeRunOutcomeEvent = (event: string, payload: Record<string, unknown>) => {
if (
event === "ToolCallStart" && typeof payload.toolCallId === "string" &&
typeof payload.toolCallName === "string" &&
childRunToolNames.has(payload.toolCallName)
) {
childRunToolCallIds.add(payload.toolCallId);
}
if (event === "ToolCallResult" && payload.isError === true) {
toolErrorCount++;
if (
typeof payload.toolCallId === "string" &&
childRunToolCallIds.has(payload.toolCallId)
) {
childRunErrorCount++;
}
}
if (event === "RunError") {
terminalRunErrorCode ??= toStableRunErrorCode(payload.code);
}
};
const runOutcome = createRunOutcomeTracker(childRunToolNames);
let heartbeatTimer: ReturnType<typeof setInterval> | undefined;
stopHeartbeat = () => {
if (heartbeatTimer) {
Expand Down Expand Up @@ -1570,7 +1581,7 @@ export async function createRuntimeAgentStreamResponse(
providerReplayStepOpen = false;
}
prepareToolResultIfNeeded(mappedEvent.event, mappedEvent.payload);
observeRunOutcomeEvent(mappedEvent.event, mappedEvent.payload);
runOutcome.observe(mappedEvent.event, mappedEvent.payload);
enqueueIfAttached(mappedEvent.event, mappedEvent.payload);
};
heartbeatTimer = setInterval(
Expand Down Expand Up @@ -1629,12 +1640,12 @@ export async function createRuntimeAgentStreamResponse(
}

for (const mappedEvent of finalizeRunEvents(state, completedResponse)) {
observeRunOutcomeEvent(mappedEvent.event, mappedEvent.payload);
runOutcome.observe(mappedEvent.event, mappedEvent.payload);
enqueueIfAttached(mappedEvent.event, mappedEvent.payload);
}
const finalStatus = state.sawTerminalError ? "failed" : "completed";
const terminalErrorCode = state.sawTerminalError
? terminalRunErrorCode ?? "AgentRunTerminalError"
? runOutcome.terminalErrorCode() ?? "AgentRunTerminalError"
: undefined;
if (state.sawTerminalError) {
deps.sessionManager.failRun(input.runId);
Expand All @@ -1651,8 +1662,7 @@ export async function createRuntimeAgentStreamResponse(
"agent.run.final_status": finalStatus,
"agent.run.saw_visible_output": state.sawVisibleOutput,
"agent.run.saw_terminal_error": state.sawTerminalError,
"agent.run.tool_error_count": toolErrorCount,
"agent.run.child_run_error_count": childRunErrorCount,
...runOutcome.spanAttributes(),
// The RunError message can carry unclassified framework error text, so only
// the stable code leaves the process.
...(terminalErrorCode ? { "error.type": terminalErrorCode } : {}),
Expand All @@ -1679,8 +1689,7 @@ export async function createRuntimeAgentStreamResponse(
sawVisibleOutput: state.sawVisibleOutput,
sawTerminalError: state.sawTerminalError,
finishReason: state.metadata.finishReason,
toolErrorCount,
childRunErrorCount,
...runOutcome.logFields(),
};
if (terminalErrorCode) {
markSpanFailed(runSpan, terminalErrorCode);
Expand All @@ -1703,8 +1712,7 @@ export async function createRuntimeAgentStreamResponse(
status: "cancelled",
}),
"agent.run.final_status": "cancelled",
"agent.run.tool_error_count": toolErrorCount,
"agent.run.child_run_error_count": childRunErrorCount,
...runOutcome.spanAttributes(),
"error.type": "AgentRunCancelledError",
"error.message": error.message,
// The model call in flight at the abort may have been billed without
Expand Down Expand Up @@ -1736,8 +1744,7 @@ export async function createRuntimeAgentStreamResponse(
status: "failed",
}),
"agent.run.final_status": "failed",
"agent.run.tool_error_count": toolErrorCount,
"agent.run.child_run_error_count": childRunErrorCount,
...runOutcome.spanAttributes(),
"error.type": runErrorCode,
"error.cause.type": error instanceof Error ? error.name : "Error",
"error.message": errorMessage,
Expand Down
6 changes: 3 additions & 3 deletions src/metrics/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1114,18 +1114,18 @@ function isBoundedTenantSample(
): boolean {
if (!apply(regExpTest, DIRECT_METRIC_NAME, [name])) return false;
const entries = apply(objectEntries, Object, [attributes]) as Array<[string, AttributeValue]>;
let projectAttributes = 0;
let ownAttributes = 0;
for (let index = 0; index < entries.length; index++) {
const entry = entries[index];
if (entry === undefined) continue;
if (apply(arrayIncludes, PROJECT_LABEL_KEYS, [entry[0]])) continue;
projectAttributes++;
ownAttributes++;
if (!apply(regExpTest, DIRECT_LABEL_KEY, [entry[0]])) return false;
if (typeof entry[1] === "string" && entry[1].length > DIRECT_MAX_ATTRIBUTE_VALUE_LENGTH) {
return false;
}
}
return projectAttributes <= DIRECT_MAX_TENANT_ATTRIBUTES;
return ownAttributes <= DIRECT_MAX_TENANT_ATTRIBUTES;
}

// Bounds what one tenant can add to the shared metrics backend from one
Expand Down
19 changes: 10 additions & 9 deletions src/proxy/logger.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,22 +111,23 @@ interface LogEntry {
service: string;
veryfrontVersion: string;
message: string;
/** @deprecated Use `trace_id` instead. Kept for Grafana dashboard transition. */
// camelCase aliases of the snake_case fields below. Remove each one once no
// Grafana dashboard, alert rule or saved Loki query filters on it.
/** @deprecated Use `trace_id`. */
traceId?: string;
/** @deprecated Use `span_id` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `span_id`. */
spanId?: string;
// Request context fields (at top level for Grafana filtering)
/** @deprecated Use `request_id` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `request_id`. */
requestId?: string;
/** @deprecated Use `project_slug` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `project_slug`. */
projectSlug?: string;
/** @deprecated Use `project_id` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `project_id`. */
projectId?: string;
/** @deprecated Use `release_id` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `release_id`. */
releaseId?: string;
/** @deprecated Use `branch_id` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `branch_id`. */
branchId?: string;
/** @deprecated Use `branch_name` instead. Kept for Grafana dashboard transition. */
/** @deprecated Use `branch_name`. */
branchName?: string;
// Standard snake_case fields shared with the runtime logger and the API, so
// one Loki filter scopes lines from every service to a project.
Expand Down
26 changes: 12 additions & 14 deletions src/routing/registry/registry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,20 +31,18 @@ export function buildRouteRegistrySpanAttributes(
attributes["project.id"] = projectId;
}

if ((projectSlug || projectId) && ctx.environmentName) {
attributes["veryfront.environment_name"] = ctx.environmentName;
}

if (projectSlug || projectId) {
const signedBranchName = ctx.branchId ? ctx.branchName : ctx.defaultBranchName;
if (ctx.branchId || signedBranchName) {
if (ctx.branchId) attributes["branch.id"] = ctx.branchId;
if (signedBranchName) attributes["branch.name"] = signedBranchName;
} else if (ctx.resolvedEnvironment === "production") {
if (ctx.releaseId) attributes["release.id"] = ctx.releaseId;
} else if (ctx.requestContext?.branch) {
attributes["branch.name"] = ctx.requestContext.branch;
}
if (!projectSlug && !projectId) return attributes;

if (ctx.environmentName) attributes["veryfront.environment_name"] = ctx.environmentName;

const signedBranchName = ctx.branchId ? ctx.branchName : ctx.defaultBranchName;
if (ctx.branchId || signedBranchName) {
if (ctx.branchId) attributes["branch.id"] = ctx.branchId;
if (signedBranchName) attributes["branch.name"] = signedBranchName;
} else if (ctx.resolvedEnvironment === "production") {
if (ctx.releaseId) attributes["release.id"] = ctx.releaseId;
} else if (ctx.requestContext?.branch) {
attributes["branch.name"] = ctx.requestContext.branch;
}

return attributes;
Expand Down
Loading