feat(budget): enforce durable governed-inference reservations

.github/workflows/ci.yml

@@ -491,6 +491,37 @@ jobs:
             --config "$GITHUB_WORKSPACE/cli/vitest.config.ts" \
             --root deploy/helm/kars/tests
 
+  inference-budget-api:
+    name: Governed inference API and CEL (Kind v1.31)
+    needs: changes
+    if: needs.changes.outputs.code == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 12
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
+        with:
+          node-version: "22"
+      - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
+      - uses: azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v4
+        with:
+          version: v1.30.5
+      - uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
+        with:
+          install_only: true
+          version: v0.24.0
+      - name: Restore existing YAML and test dependencies
+        run: npm ci --prefix cli
+      - name: Create disposable supported apiserver before any Rust image build
+        run: kind create cluster --name kars-budget-api --image kindest/node:v1.31.0 --config tests/e2e/kind-config.yaml
+      - name: Validate real CRD/CEL and Pod audience identity
+        run: node tests/e2e/inference-budget-api.mjs
+      - name: Remove only the disposable API fixture cluster
+        if: always()
+        run: kind delete cluster --name kars-budget-api
+
   security-scan:
     name: Security Scan
     runs-on: ubuntu-latest
@@ -636,7 +667,7 @@ jobs:
           # Fetch enough history to diff.
           git fetch --no-tags --depth=50 origin "$base" "$head" 2>/dev/null || true
           if git diff --name-only "$base" "$head" 2>/dev/null \
-             | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|shared/|runtimes/hermes/src/kars_runtime_hermes/plugin/sre|cli/src/(commands/sre|lib/sre|lib/namespace-ownership)|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then
+             | grep -E '^(controller/|inference-router/|a2a-gateway/|kars-a2a-core/|deploy/helm/|sandbox-images/|tests/e2e/|shared/|runtimes/hermes/src/kars_runtime_hermes/plugin/sre|cli/src/(commands/(sre|budget)|lib/sre|lib/namespace-ownership)|Cargo\.toml|Cargo\.lock|Makefile)' >/dev/null; then
             echo "run=true" >> "$GITHUB_OUTPUT"
           else
             echo "run=false" >> "$GITHUB_OUTPUT"

.gitignore

@@ -10,6 +10,8 @@
 *.userosscache
 *.sln.docstates
 *.env
+.budget-kind-*/
+**/.budget-client-test-*/
 
 # User-specific files (MonoDevelop/Xamarin Studio)
 *.userprefs

cli/src/cli.ts

@@ -11,6 +11,7 @@ import { upgradeCommand } from "./commands/upgrade.js";
 import { devCommand } from "./commands/dev.js";
 import { addCommand } from "./commands/add.js";
 import { credentialsCommand } from "./commands/credentials.js";
+import { budgetCommand } from "./commands/budget.js";
 import { namespaceCommand } from "./commands/namespace.js";
 import { configCommand } from "./commands/config.js";
 import { connectCommand } from "./commands/connect.js";
@@ -73,6 +74,7 @@ export function createCli(): Command {
   // Configuration
   program.addCommand(credentialsCommand());
   program.addCommand(namespaceCommand());
+  program.addCommand(budgetCommand());
   program.addCommand(configCommand());
   program.addCommand(modelCommand());
   program.addCommand(policyCommand());
@@ -116,7 +118,7 @@ export function createCli(): Command {
 Command groups:
   Lifecycle       up, dev, add, push, destroy
   Operations      connect, status, list, logs, inspect
-  Configuration   credentials, model, policy, egress, config
+  Configuration   credentials, budget, model, policy, egress, config
   Observability   trace, eval, operator, audit, headlamp
   Agent mobility  handoff, mesh, pair
   Interop         convert, a2a, a2a-agent, migrate

cli/src/commands/budget.test.ts

@@ -0,0 +1,59 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { describe, expect, it } from "vitest";
+import { budgetStatus, governedPlan } from "./budget.js";
+
+const plan = {
+  apiVersion: "kars.azure.com/v1alpha1", kind: "KarsTask",
+  metadata: { name: "mission", namespace: "workspace" },
+  spec: { objective: "Build", envelope: { tier: 3, authorityCeiling: 3, delegationDepth: 2,
+    budget: { tokens: 500, usdMicros: 0 } }, execution: { launch: false } },
+};
+
+describe("explicit governed inference CLI", () => {
+  it("creates a scoped plan without mapping aggregate limits into daily budgets", () => {
+    const result = governedPlan(JSON.stringify(plan), "workspace");
+    expect(result).toEqual({ ...plan, spec: { ...plan.spec,
+      envelope: { ...plan.spec.envelope, budget: { ...plan.spec.envelope.budget, scope: "GovernedInference" } } } });
+    expect(JSON.stringify(result)).not.toContain("daily");
+  });
+
+  it.each([-1, 0.5, Number.MAX_SAFE_INTEGER + 1, "10"])("rejects inexact or negative cap %s", (tokens) => {
+    const bad = structuredClone(plan) as Record<string, any>;
+    bad.spec.envelope.budget.tokens = tokens;
+    expect(() => governedPlan(JSON.stringify(bad), "workspace")).toThrow(/exact nonnegative/);
+  });
+
+  it("rejects cross-workspace creation, existing UID imports and fully unbounded plans", () => {
+    expect(() => governedPlan(JSON.stringify(plan), "other")).toThrow(/workspace/);
+    expect(() => governedPlan(JSON.stringify({ ...plan, metadata: { ...plan.metadata, uid: "old" } }), "workspace")).toThrow(/new UID/);
+    const unbounded = structuredClone(plan);
+    unbounded.spec.envelope.budget.tokens = 0;
+    expect(() => governedPlan(JSON.stringify(unbounded), "workspace")).toThrow(/positive/);
+  });
+
+  it("reports only a pinned account and refuses replacement/missing ledger instead of zeroing", async () => {
+    const root = { kind: "KarsTask", resource: { namespace: "workspace", name: "mission", uid: "task-uid" },
+      workspaceUid: "workspace-uid", clusterUid: "cluster-uid" };
+    const account = { namespace: "accounting", name: "root-account", uid: "account-uid" };
+    const task = { metadata: { ...plan.metadata, uid: "task-uid" }, status: {
+      phase: "Ready", inferenceBudget: { root, account, taskUid: "task-uid" },
+    } };
+    const ledger = { version: "governed-inference/v1", scope: "GovernedInference", phase: "Active",
+      accountUid: account.uid, limits: { tokens: 500 }, attempts: {}, meters: {
+        reserved: { tokens: 40, usdMicros: 0 }, settled: { tokens: 30, usdMicros: 0 },
+        uncertain: { tokens: 20, usdMicros: 0 }, unpricedAttempts: 2,
+      } };
+    const stored = { metadata: { uid: account.uid }, spec: { scope: "GovernedInference", root }, status: { ledger } };
+    const execute = async (args: string[]) => JSON.stringify(args[1] === "karstask" ? task : stored);
+    const result = await budgetStatus(execute, "task", "mission", "workspace");
+    expect(result.meters).toMatchObject({ uncertain: { tokens: "20", usdMicros: "0" } });
+    expect(result.priceCoverage).toContain("unknown");
+    stored.metadata.uid = "replacement";
+    await expect(budgetStatus(execute, "task", "mission", "workspace")).rejects.toThrow(/replaced/);
+    stored.metadata.uid = account.uid;
+    delete (stored.status as Record<string, unknown>).ledger;
+    await expect(budgetStatus(execute, "task", "mission", "workspace")).rejects.toThrow(/missing/);
+  });
+});

cli/src/commands/budget.ts

@@ -0,0 +1,157 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { Command } from "commander";
+import { readFile } from "node:fs/promises";
+import { parseAllDocuments } from "yaml";
+
+type ObjectValue = Record<string, unknown>;
+type Execute = (args: string[], input?: string) => Promise<string>;
+const scope = "GovernedInference";
+
+function object(value: unknown, message: string): ObjectValue {
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(message);
+  return value as ObjectValue;
+}
+
+function name(value: unknown): string {
+  if (typeof value !== "string" || !/^[a-z0-9](?:[a-z0-9.-]{0,251}[a-z0-9])?$/.test(value)) {
+    throw new Error("A valid Kubernetes resource name is required");
+  }
+  return value;
+}
+
+function amount(value: unknown): number {
+  if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) {
+    throw new Error("Budget amounts must be exact nonnegative safe integers; no floating prices");
+  }
+  return value;
+}
+
+/** Explicit opt-in on CREATE only: never silently convert a running legacy UID. */
+export function governedPlan(text: string, namespace: string): ObjectValue {
+  const documents = parseAllDocuments(text);
+  if (documents.length !== 1 || documents[0].errors.length) throw new Error("Expected one valid Task or Team manifest");
+  const plan = object(documents[0].toJSON(), "Expected a Task or Team manifest");
+  if (plan.apiVersion !== "kars.azure.com/v1alpha1" || !["KarsTask", "KarsTeam"].includes(String(plan.kind))) {
+    throw new Error("Governed inference plans must be KarsTask or KarsTeam v1alpha1");
+  }
+  const metadata = object(plan.metadata, "Manifest metadata is required");
+  name(metadata.name);
+  if (metadata.uid || metadata.resourceVersion || plan.status) throw new Error("Create a new UID; existing state cannot be imported or reset");
+  if (metadata.namespace !== undefined && metadata.namespace !== namespace) {
+    throw new Error("Manifest workspace differs from --namespace");
+  }
+  metadata.namespace = name(namespace);
+  const spec = object(plan.spec, "Manifest spec is required");
+  const envelope = object(spec.envelope, "Manifest envelope is required");
+  const budget = object(envelope.budget, "Set envelope.budget.tokens and/or usdMicros explicitly");
+  if (budget.scope !== undefined && budget.scope !== scope) throw new Error("Unsupported budget scope");
+  const tokens = amount(budget.tokens ?? 0);
+  const usdMicros = amount(budget.usdMicros ?? 0);
+  if (tokens === 0 && usdMicros === 0) throw new Error("At least one positive governed-inference cap is required; zero means unbounded");
+  budget.scope = scope;
+  return plan;
+}
+
+function resource(kind: string): string {
+  if (kind === "task") return "karstask";
+  if (kind === "team") return "karsteam";
+  throw new Error("--kind must be task or team");
+}
+
+function readObject(text: string): ObjectValue {
+  try { return object(JSON.parse(text), "Invalid API object"); }
+  catch { throw new Error("Kubernetes returned an invalid budget object"); }
+}
+
+function metadata(value: ObjectValue): ObjectValue { return object(value.metadata, "Object identity is missing"); }
+
+export async function budgetStatus(
+  execute: Execute, kind: string, target: string, namespace: string,
+): Promise<ObjectValue> {
+  const owner = readObject(await execute(["get", resource(kind), name(target), "-n", name(namespace), "-o", "json"]));
+  const ownerMeta = metadata(owner);
+  const status = object(owner.status, "Task/Team has no controller status yet");
+  const binding = kind === "task" ? object(status.inferenceBudget, "Task has no governed-inference account binding") : undefined;
+  const reference = object(binding?.account ?? status.inferenceBudgetAccount, "No lifetime account has been bound");
+  if (kind === "task" && binding?.taskUid !== ownerMeta.uid) throw new Error("Task UID binding is stale");
+  const account = readObject(await execute(["get", "karsbudgetaccount", name(reference.name),
+    "-n", name(reference.namespace), "-o", "json"]));
+  if (metadata(account).uid !== reference.uid) throw new Error("Budget account was replaced; refusing to report a fresh zero balance");
+  const spec = object(account.spec, "Account spec is missing");
+  const root = object(spec.root, "Account root is missing");
+  const identity = object(root.resource, "Account root identity is missing");
+  if (identity.namespace !== namespace || spec.scope !== scope) throw new Error("Budget scope/workspace mismatch");
+  if (kind === "team" && (root.kind !== "KarsTeam" || identity.uid !== ownerMeta.uid || identity.name !== ownerMeta.name)) {
+    throw new Error("Team lifetime UID binding is stale");
+  }
+  if (binding && JSON.stringify(binding.root) !== JSON.stringify(root)) {
+    // JSON object ordering is not authority; compare exact identity fields.
+    const pinned = object(binding.root, "Task root binding is missing");
+    const pinnedIdentity = object(pinned.resource, "Task root identity is missing");
+    if (["kind", "workspaceUid", "clusterUid"].some((key) => pinned[key] !== root[key])
+      || ["namespace", "name", "uid"].some((key) => pinnedIdentity[key] !== identity[key])) {
+      throw new Error("Task/account root UID binding differs");
+    }
+  }
+  const accountStatus = object(account.status, "Budget account is uninitialized");
+  const ledger = object(accountStatus.ledger, "Budget ledger is missing; no zero fallback");
+  if (ledger.accountUid !== reference.uid || ledger.scope !== scope || ledger.version !== "governed-inference/v1") {
+    throw new Error("Ledger version or identity differs");
+  }
+  const totals = object(ledger.meters, "Budget meters are missing");
+  const meters: ObjectValue = {};
+  for (const group of ["reserved", "settled", "uncertain"]) {
+    const values = object(totals[group], "Budget meters are incomplete");
+    meters[group] = { tokens: String(amount(values.tokens)), usdMicros: String(amount(values.usdMicros)) };
+  }
+  const unpriced = amount(totals.unpricedAttempts);
+  const pendingUnpriced = Object.values(object(ledger.attempts, "Budget attempt ledger is missing"))
+    .some((entry) => {
+      const attempt = object(entry, "Malformed budget attempt");
+      return ["Reserved", "InFlight"].includes(String(attempt.phase))
+        && object(attempt.quote, "Budget quote is missing").priceCovered === false;
+    });
+  return {
+    scope, root, account: reference, accountPhase: ledger.phase,
+    taskOrTeamPhase: status.phase, limits: ledger.limits, meters,
+    unpricedAttempts: String(unpriced),
+    priceCoverage: unpriced > 0 || pendingUnpriced ? "incomplete — monetary cost is unknown" : "configured maxima only — not an invoice",
+    meaning: "Governed inference tokens and configured maximum prices only; compute, tools, storage and invoice costs are excluded",
+    dispatch: "Every provider send still requires live router identity, full task authorization, valid bounds/tariffs and an atomic broker grant",
+  };
+}
+
+async function kubectl(args: string[], input?: string): Promise<string> {
+  const { execa } = await import("execa");
+  try {
+    return (await execa("kubectl", [...args, "--request-timeout=20s"], {
+      input, stdio: ["pipe", "pipe", "pipe"],
+    })).stdout;
+  } catch {
+    throw new Error("Kubernetes budget operation failed; check context, API readiness and operator RBAC");
+  }
+}
+
+export function budgetCommand(): Command {
+  const command = new Command("budget")
+    .description("Governed inference token/configured-maximum-price accounts (not total task spend)");
+  command.command("create")
+    .description("Opt in a NEW Task/Team manifest; controller configuration and launch gates still apply")
+    .requiredOption("-f, --file <path>", "One KarsTask or KarsTeam YAML/JSON manifest")
+    .option("-n, --namespace <workspace>", "Task/Team workspace", "kars-system")
+    .action(async (options: { file: string; namespace: string }) => {
+      const plan = governedPlan(await readFile(options.file, "utf8"), options.namespace);
+      await kubectl(["create", "-f", "-", "-o", "name"], JSON.stringify(plan));
+      console.log("Governed-inference plan created. This is not an enforcement/readiness assertion; inspect controller status before launch.");
+    });
+  command.command("status <name>")
+    .description("Read the pinned durable account; never substitute absent accounting with zero")
+    .option("--kind <task|team>", "Resource owning the binding", "task")
+    .option("-n, --namespace <workspace>", "Task/Team workspace", "kars-system")
+    .action(async (target: string, options: { kind: string; namespace: string }) => {
+      console.log(JSON.stringify(await budgetStatus(kubectl, options.kind, target, options.namespace), null, 2));
+    });
+  return command;
+}