fix(agent): mark failed agent.run spans as ERROR with their run error code

docs/api-reference/veryfront/observability.md

@@ -197,6 +197,7 @@ import {
 | `initializeOTLPWithApis`   | Initialize OTLP tracing with explicit API adapters.                                      | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
 | `injectContext`            | Context for inject.                                                                      | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
 | `isOTLPEnabled`            | Check whether OTLP export is enabled.                                                    | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
+| `markSpanFailed`           | Marks a span as failed with a stable error code.                                         | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
 | `setActiveSpanAttributes`  | Sets active span attributes.                                                             | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
 | `setActiveSpanErrorStatus` | Marks the active span as failed.                                                         | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |
 | `setSpanAttributes`        | Sets span attributes.                                                                    | [source](https://github.com/veryfront/veryfront-code/blob/main/src/observability/tracing/otlp-setup.ts) |

src/agent/hosted/agent-run-lifecycle.test.ts

@@ -20,6 +20,7 @@ class RecordingSpan implements HostedAgentRunSpan {
   attributes: Record<string, unknown> = {};
   finished = 0;
   withContextCalls = 0;
+  failedWith: string[] = [];
 
   setAttributes(attributes: Record<string, unknown>): void {
     this.attributes = { ...this.attributes, ...attributes };
@@ -29,6 +30,10 @@ class RecordingSpan implements HostedAgentRunSpan {
     this.finished += 1;
   }
 
+  markFailed(errorCode: string): void {
+    this.failedWith.push(errorCode);
+  }
+
   withContext<T>(fn: () => T): T {
     this.withContextCalls += 1;
     return fn();
@@ -146,6 +151,7 @@ describe("hosted-agent-run-lifecycle", () => {
     controller.finalize({ status: "failed", terminalErrorCode: "LATE" });
 
     assertEquals(span.finished, 1);
+    assertEquals(span.failedWith, []);
     assertEquals(span.attributes["message.id"], "message-2");
     assertEquals(span.attributes["agent.run.final_status"], "completed");
     assertEquals(span.attributes["gen_ai.provider.name"], "anthropic");
@@ -160,6 +166,61 @@ describe("hosted-agent-run-lifecycle", () => {
     assertEquals(span.attributes["gen_ai.usage.reasoning.output_tokens"], 1);
   });
 
+  it("marks a failed hosted run span failed with its terminal error code before finishing", () => {
+    const span = new RecordingSpan();
+    const finishedAfterFailure: number[] = [];
+    span.markFailed = (errorCode) => {
+      span.failedWith.push(errorCode);
+      finishedAfterFailure.push(span.finished);
+    };
+    const controller = createHostedAgentRunSpanController({
+      tracer: { startSpan: () => span },
+      operationName: "invoke_agent",
+      projectId: "project-1",
+      userId: "user-1",
+      agentId: "agent-1",
+    });
+
+    controller.finalize({ status: "failed", terminalErrorCode: "insufficient-credits" });
+
+    assertEquals(span.failedWith, ["insufficient-credits"]);
+    assertEquals(finishedAfterFailure, [0]);
+    assertEquals(span.finished, 1);
+    assertEquals(span.attributes["error.type"], "insufficient-credits");
+  });
+
+  it("marks a failed hosted run span without a stable terminal error code as STREAM_ERROR", () => {
+    const span = new RecordingSpan();
+    const controller = createHostedAgentRunSpanController({
+      tracer: { startSpan: () => span },
+      operationName: "chat",
+      projectId: "project-1",
+      userId: "user-1",
+      agentId: "agent-1",
+    });
+
+    controller.finalize({ status: "failed", terminalErrorCode: "/srv/app/secret.env" });
+
+    assertEquals(span.failedWith, ["STREAM_ERROR"]);
+    assertEquals(span.attributes["error.type"], "STREAM_ERROR");
+  });
+
+  it("leaves a cancelled hosted run span unmarked", () => {
+    const span = new RecordingSpan();
+    const controller = createHostedAgentRunSpanController({
+      tracer: { startSpan: () => span },
+      operationName: "chat",
+      projectId: "project-1",
+      userId: "user-1",
+      agentId: "agent-1",
+    });
+
+    controller.finalize({ status: "cancelled" });
+
+    assertEquals(span.failedWith, []);
+    assertEquals(span.finished, 1);
+  });
+
   // veryfront/veryfront-issue-inbox#1500: the hosted run span reported token counts
   // and no spend, on every status.
   //

src/agent/hosted/agent-run-lifecycle.ts

@@ -11,11 +11,13 @@ import {
   type AgentTraceUsage,
   buildAgentRunTraceAttributes,
   buildFinalizedAgentRunTraceAttributes,
+  resolveAgentRunErrorType,
 } from "./trace-attributes.ts";
 
 /** Public API contract for hosted agent run span. */
 export interface HostedAgentRunSpan {
   setAttributes: (attributes: AgentTraceAttributes) => void;
+  markFailed?: (errorCode: string) => void;
   finish: () => void;
   withContext: <T>(fn: () => T) => T;
 }
@@ -126,6 +128,9 @@ export function createHostedAgentRunSpanController(
 
       finalized = true;
       span.setAttributes(buildFinalizedAgentRunTraceAttributes(finalState));
+      if (finalState.status === "failed") {
+        span.markFailed?.(resolveAgentRunErrorType(finalState.terminalErrorCode));
+      }
       span.finish();
     },
   };

src/agent/hosted/trace-attributes.test.ts

@@ -234,4 +234,21 @@ describe("agent/agent-trace-attributes", () => {
       },
     );
   });
+
+  it("keeps only classification-shaped terminal error codes as the failed run error type", () => {
+    assertEquals(
+      buildFinalizedAgentRunTraceAttributes({
+        status: "failed",
+        terminalErrorCode: "insufficient-credits",
+      })["error.type"],
+      "insufficient-credits",
+    );
+    assertEquals(
+      buildFinalizedAgentRunTraceAttributes({
+        status: "failed",
+        terminalErrorCode: "postgres://app:secret@db.internal/prod",
+      })["error.type"],
+      "STREAM_ERROR",
+    );
+  });
 });

src/agent/hosted/trace-attributes.ts

@@ -287,6 +287,21 @@ export function buildInvokeAgentTraceAttributes(input: {
   });
 }
 
+const STABLE_RUN_ERROR_CODE_PATTERN = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/;
+
+/**
+ * Keeps a run error code only when it has the shape of a classification. A runtime
+ * can put any string there, and it becomes a span status message and a log field.
+ */
+export function toStableRunErrorCode(code: unknown): string | undefined {
+  return typeof code === "string" && STABLE_RUN_ERROR_CODE_PATTERN.test(code) ? code : undefined;
+}
+
+/** Stable `error.type` of a failed agent run, also used as its span status message. */
+export function resolveAgentRunErrorType(terminalErrorCode?: string | null): string {
+  return toStableRunErrorCode(terminalErrorCode) ?? "STREAM_ERROR";
+}
+
 /** Builds finalized agent run trace attributes. */
 export function buildFinalizedAgentRunTraceAttributes(input: {
   status: "completed" | "failed" | "cancelled";
@@ -309,7 +324,7 @@ export function buildFinalizedAgentRunTraceAttributes(input: {
     ...(finishReason ? { "gen_ai.response.finish_reasons": [finishReason] } : {}),
     ...(input.status === "failed"
       ? {
-        "error.type": input.terminalErrorCode ?? "STREAM_ERROR",
+        "error.type": resolveAgentRunErrorType(input.terminalErrorCode),
         "error.message": input.terminalErrorMessage,
       }
       : {}),

src/agent/runtime/agent-delegation.test.ts

@@ -5,6 +5,7 @@ import {
   AGENT_DELEGATE_TOOL_PREFIX,
   buildAgentDelegateTools,
   createInvokeAgentTool,
+  isFrameworkChildRunTool,
   isProviderSafeDelegateId,
 } from "./agent-delegation.ts";
 import type { Agent } from "../types.ts";
@@ -352,3 +353,18 @@ it("delegate agent execution preserves an explicit process-boundary restriction"
   assertEquals(observedPolicy, policy);
   assertEquals(observedDuringStreamConsumption, policy);
 });
+
+it("isFrameworkChildRunTool recognizes framework invoke_agent and delegate tools only", () => {
+  const delegateTools = buildAgentDelegateTools({
+    delegates: ["researcher"],
+    resolveAgent: () => undefined,
+  });
+
+  assertEquals(isFrameworkChildRunTool(createInvokeAgentTool()), true);
+  assertEquals(isFrameworkChildRunTool(delegateTools.agent_researcher), true);
+  assertEquals(
+    isFrameworkChildRunTool({ id: "agent_researcher", type: "function", execute: () => ({}) }),
+    false,
+  );
+  assertEquals(isFrameworkChildRunTool(true), false);
+});

src/agent/runtime/agent-delegation.ts

@@ -13,12 +13,19 @@ export const INVOKE_AGENT_TOOL_ID = "invoke_agent";
 const applyIntrinsic = Reflect.apply;
 const stringTrim = String.prototype.trim;
 const frameworkInvokeAgentTools = new WeakSet<object>();
+const frameworkDelegateTools = new WeakSet<object>();
 
 /** Whether a tool is the framework-created invoke_agent from {@link createInvokeAgentTool}. */
 export function isFrameworkInvokeAgentTool(value: unknown): boolean {
   return value !== null && typeof value === "object" && frameworkInvokeAgentTools.has(value);
 }
 
+/** Whether a tool is a framework tool whose calls run a child agent (invoke_agent or `agent_{id}`). */
+export function isFrameworkChildRunTool(value: unknown): boolean {
+  return value !== null && typeof value === "object" &&
+    (frameworkInvokeAgentTools.has(value) || frameworkDelegateTools.has(value));
+}
+
 const getInvokeAgentInputSchema = defineSchema((v) =>
   v.object({
     agent_id: v.string()
@@ -122,7 +129,7 @@ function createLazyDelegateTool(
   resolveAgent: DelegateAgentResolver,
   executeDelegate?: DelegateAgentExecutor,
 ): Tool {
-  return markRuntimeLocalTool({
+  const tool = markRuntimeLocalTool({
     id: `${AGENT_DELEGATE_TOOL_PREFIX}${delegateId}`,
     type: "function",
     description: `Delegate a self-contained subtask to the "${delegateId}" specialist agent, ` +
@@ -150,6 +157,8 @@ function createLazyDelegateTool(
       return agentAsTool(target, `Delegate to ${delegateId}`).execute(input, context);
     },
   });
+  frameworkDelegateTools.add(tool);
+  return tool;
 }
 
 /**